From d9d6a949f1116f3e327530581395f549cb06a33c Mon Sep 17 00:00:00 2001 From: Kevin Zakka Date: Fri, 22 Oct 2021 20:14:55 -0700 Subject: [PATCH 001/491] Gracefully catch empty file docstring. (#1) * Gracefully catch empty file docstring. * Nits! Co-authored-by: Brent Yi --- dcargs/_parse.py | 5 ++++- tests/test_dcargs.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/dcargs/_parse.py b/dcargs/_parse.py index 0e0894ae1..521fbde53 100644 --- a/dcargs/_parse.py +++ b/dcargs/_parse.py @@ -10,11 +10,14 @@ def parse( cls: Type[DataclassType], - description: str = "", + description: Optional[str] = None, args: Optional[Sequence[str]] = None, ) -> DataclassType: """Populate a dataclass via CLI args.""" + if description is None: + description = "" + parser_definition, construction_metadata = _parsers.ParserDefinition.from_dataclass( cls, parent_dataclasses=set(), # Used for recursive calls. diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 5d877c078..c7a59918f 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -398,3 +398,13 @@ class OptionalSubparser: dcargs.parse(OptionalSubparser, args=["--x", "1", "B", "--z", "3"]) with pytest.raises(SystemExit): dcargs.parse(OptionalSubparser, args=["--x", "1", "C", "--y", "3"]) + + +def test_parse_empty_description(): + """If the file has no dosctring, it should be treated as an empty string.""" + + @dataclasses.dataclass + class A: + x: int = 0 + + assert dcargs.parse(A, description=None) == A(x=0) From 541bcd1812953ee2714a044700e17a729d58601e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 22 Oct 2021 20:42:43 -0700 Subject: [PATCH 002/491] Support default values for nested dataclass fields --- dcargs/_arguments.py | 2 ++ dcargs/_parse.py | 1 + dcargs/_parsers.py | 21 ++++++++++-- tests/test_dcargs.py | 17 ---------- tests/test_nested.py | 79 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 20 deletions(-) create mode 100644 tests/test_nested.py diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 6896caa68..f7ccd8d74 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -54,6 +54,7 @@ def make_from_field( parent_class: Type, field: dataclasses.Field, type_from_typevar: Dict[TypeVar, Type], + default_override: Optional[Any], ) -> Tuple["ArgumentDefinition", _construction.FieldRole]: """Create an argument definition from a field. Also returns a field role, which specifies special instructions for reconstruction.""" @@ -66,6 +67,7 @@ def make_from_field( field=field, parent_class=parent_class, type=field.type, + default=default_override, ) # Propagate argument through transforms until stable. diff --git a/dcargs/_parse.py b/dcargs/_parse.py index 521fbde53..e5ffe7891 100644 --- a/dcargs/_parse.py +++ b/dcargs/_parse.py @@ -23,6 +23,7 @@ def parse( parent_dataclasses=set(), # Used for recursive calls. subparser_name_from_type={}, # Aliases for subparsers; this is working, but not yet exposed. parent_type_from_typevar=None, # Used for recursive calls. + default_instance=None, # Overrides for default values. This could also be exposed. ) parser = argparse.ArgumentParser( diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index f830c3ef8..8128e1dc3 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -1,11 +1,13 @@ import argparse import dataclasses -from typing import Any, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union +from typing import Dict, List, Optional, Set, Tuple, Type, TypeVar, Union from typing_extensions import _GenericAlias # type: ignore from . import _arguments, _construction, _docstrings, _resolver, _strings +T = TypeVar("T") + @dataclasses.dataclass class ParserDefinition: @@ -41,10 +43,11 @@ def apply(self, parser: argparse.ArgumentParser) -> None: @staticmethod def from_dataclass( - cls: Union[Type[Any], _GenericAlias], + cls: Union[Type[T], _GenericAlias], parent_dataclasses: Optional[Set[Type]], subparser_name_from_type: Dict[Type, str], parent_type_from_typevar: Optional[Dict[TypeVar, Type]], + default_instance: Optional[T], ) -> Tuple["ParserDefinition", _construction.ConstructionMetadata]: """Create a parser definition from a dataclass.""" @@ -81,6 +84,9 @@ def from_dataclass( parent_dataclasses | {cls}, subparser_name_from_type=subparser_name_from_type, parent_type_from_typevar=type_from_typevar, + default_instance=field.default + if field.default is not dataclasses.MISSING + else None, ) metadata.update(child_metadata) @@ -112,6 +118,9 @@ def from_dataclass( assert ( subparsers is None ), "Only one subparser group is supported per dataclass" + assert ( + field.default == dataclasses.MISSING + ), "Default dataclass value not yet supported for subparser definitions" parsers: Dict[str, ParserDefinition] = {} for option in options_no_none: @@ -132,6 +141,7 @@ def from_dataclass( parent_dataclasses | {cls}, subparser_name_from_type, parent_type_from_typevar=type_from_typevar, + default_instance=None, ) metadata.update(child_metadata) @@ -149,7 +159,12 @@ def from_dataclass( # Make an argument! if arg_from_field: arg, role = _arguments.ArgumentDefinition.make_from_field( - cls, field, type_from_typevar + cls, + field, + type_from_typevar, + default_override=getattr(default_instance, field.name) + if default_instance is not None + else None, ) args.append(arg) metadata.role_from_field[field] = role diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index c7a59918f..bbd64c92c 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -301,23 +301,6 @@ class A: assert dcargs.parse(A, args=[]) == A() -def test_nested(): - @dataclasses.dataclass - class B: - y: int - - @dataclasses.dataclass - class Nested: - x: int - b: B - - assert dcargs.parse(Nested, args=["--x", "1", "--b.y", "3"]) == Nested( - x=1, b=B(y=3) - ) - with pytest.raises(SystemExit): - dcargs.parse(Nested, args=["--x", "1"]) - - # TODO: implement this! # def test_optional_nested(): # @dataclasses.dataclass diff --git a/tests/test_nested.py b/tests/test_nested.py new file mode 100644 index 000000000..049b958de --- /dev/null +++ b/tests/test_nested.py @@ -0,0 +1,79 @@ +import dataclasses + +import pytest + +import dcargs + + +def test_nested(): + @dataclasses.dataclass + class B: + y: int + + @dataclasses.dataclass + class Nested: + x: int + b: B + + assert dcargs.parse(Nested, args=["--x", "1", "--b.y", "3"]) == Nested( + x=1, b=B(y=3) + ) + with pytest.raises(SystemExit): + dcargs.parse(Nested, args=["--x", "1"]) + + +def test_nested_default(): + @dataclasses.dataclass + class B: + y: int = 3 + + @dataclasses.dataclass + class Nested: + x: int + b: B + + assert dcargs.parse(Nested, args=["--x", "1", "--b.y", "3"]) == Nested( + x=1, b=B(y=3) + ) + assert dcargs.parse(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=3)) + + +def test_default_nested(): + @dataclasses.dataclass + class B: + y: int = 3 + + @dataclasses.dataclass + class Nested: + x: int + b: B = B(y=5) + + assert dcargs.parse(Nested, args=["--x", "1", "--b.y", "3"]) == Nested( + x=1, b=B(y=3) + ) + assert dcargs.parse(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) + + +# TODO: implement this! +# def test_optional_nested(): +# @dataclasses.dataclass +# class OptionalNestedChild: +# y: int +# z: int +# +# @dataclasses.dataclass +# class OptionalNested: +# x: int +# b: Optional[OptionalNestedChild] +# +# assert dcargs.parse(OptionalNested, args=["--x", "1"]) == OptionalNested( +# x=1, b=None +# ) +# with pytest.raises(SystemExit): +# dcargs.parse(OptionalNested, args=["--x", "1", "--b.y", "3"]) +# with pytest.raises(SystemExit): +# dcargs.parse(OptionalNested, args=["--x", "1", "--b.z", "3"]) +# +# assert dcargs.parse( +# OptionalNested, args=["--x", "1", "--b.y", "2", "--b.z", "3"] +# ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) From e1e1fb39f7a4d9b90ce1a69f13dd682dc0721c54 Mon Sep 17 00:00:00 2001 From: Kevin Zakka Date: Fri, 22 Oct 2021 22:31:15 -0700 Subject: [PATCH 003/491] Fill default value for Optional args. (#2) * First attempt at docstring for Optional args. * Grammar fix. * Minor: else if -> elif * Syntax Co-authored-by: Brent Yi --- dcargs/_arguments.py | 2 +- tests/test_docstrings.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index f7ccd8d74..8f7b6178e 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -330,7 +330,7 @@ def _generate_helptext(arg: ArgumentDefinition) -> _ArgumentTransformOutput: if arg.default is not None and hasattr(arg.default, "name"): # Special case for enums. help_parts.append(f"(default: {arg.default.name})") - elif arg.default is not None: + elif not arg.required: # General case. help_parts.append("(default: %(default)s)") diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index a5a1f6cd7..bcce44bac 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -1,3 +1,4 @@ +import typing import contextlib import dataclasses import io @@ -55,3 +56,18 @@ class HelptextMultiline: " --z INT Documentation 3\n Next line of documentation 3 (default: 3)\n" in helptext ) + + +def test_none_default_value_helptext(): + @dataclasses.dataclass + class Config: + x: typing.Optional[int] = None + """An optional variable.""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(Config, args=["--help"]) + helptext = f.getvalue() + print(helptext) + assert " --x INT An optional variable. (default: None)\n" in helptext From e58e0d8dbd9dfb74b0b4060635bd206805e8102d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 22 Oct 2021 23:04:07 -0700 Subject: [PATCH 004/491] Fix double nested dataclass defaults, bump version --- dcargs/_parsers.py | 10 +++++++--- setup.py | 2 +- tests/test_nested.py | 22 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 8128e1dc3..e26825b75 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -79,14 +79,18 @@ def from_dataclass( # Add arguments for nested dataclasses. if _resolver.is_dataclass(field.type): + default = None + if default_instance is not None: + default = getattr(default_instance, field.name) + elif field.default is not dataclasses.MISSING: + default = field.default + child_definition, child_metadata = ParserDefinition.from_dataclass( field.type, parent_dataclasses | {cls}, subparser_name_from_type=subparser_name_from_type, parent_type_from_typevar=type_from_typevar, - default_instance=field.default - if field.default is not dataclasses.MISSING - else None, + default_instance=default, ) metadata.update(child_metadata) diff --git a/setup.py b/setup.py index 113e7bd29..d97363481 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.4", + version="0.0.5", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_nested.py b/tests/test_nested.py index 049b958de..afbf105ba 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -54,6 +54,28 @@ class Nested: assert dcargs.parse(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) +def test_double_default_nested(): + @dataclasses.dataclass + class Child: + y: int + + @dataclasses.dataclass + class Parent: + c: Child + + @dataclasses.dataclass + class Grandparent: + x: int + b: Parent = Parent(Child(y=5)) + + assert dcargs.parse(Grandparent, args=["--x", "1", "--b.c.y", "3"]) == Grandparent( + x=1, b=Parent(Child(y=3)) + ) + assert dcargs.parse(Grandparent, args=["--x", "1"]) == Grandparent( + x=1, b=Parent(Child(y=5)) + ) + + # TODO: implement this! # def test_optional_nested(): # @dataclasses.dataclass From a32ee3b94b520fcf3d253b6be1f202ddf47239db Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 25 Oct 2021 12:52:55 -0700 Subject: [PATCH 005/491] Nested dataclass & subparser support for generics --- dcargs/_arguments.py | 3 +++ dcargs/_construction.py | 21 ++++++++++++++----- dcargs/_parsers.py | 22 ++++++++++++++++---- tests/test_docstrings.py | 2 +- tests/test_generics.py | 44 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 81 insertions(+), 11 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 8f7b6178e..ba6878bbe 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -75,6 +75,9 @@ def make_from_field( role: _construction.FieldRole = _construction.FieldRole.VANILLA_FIELD def _handle_generics(arg: ArgumentDefinition) -> _ArgumentTransformOutput: + """Handle generic arguments. Note that this needs to be a transform -- if we + only checked field.type before running transforms, we wouldn't be able to + handle cases like Optional[T].""" if isinstance(arg.type, TypeVar): assert arg.type in type_from_typevar, "TypeVar not bounded" return ( diff --git a/dcargs/_construction.py b/dcargs/_construction.py index 1c264914f..b6eaa08d0 100644 --- a/dcargs/_construction.py +++ b/dcargs/_construction.py @@ -45,7 +45,7 @@ def construct_dataclass( assert _resolver.is_dataclass(cls) - cls, _type_from_typevar = _resolver.resolve_generic_dataclasses(cls) + cls, type_from_typevar = _resolver.resolve_generic_dataclasses(cls) kwargs: Dict[str, Any] = {} consumed_keywords: Set[str] = set() @@ -67,13 +67,20 @@ def get_value_from_arg(arg: str) -> Any: prefixed_field_name = field_name_prefix + field.name + # Resolve field type + field_type = ( + type_from_typevar[field.type] # type: ignore + if field.type in type_from_typevar + else field.type + ) + if role is FieldRole.ENUM: # Handle enums. - value = field.type[get_value_from_arg(prefixed_field_name)] + value = field_type[get_value_from_arg(prefixed_field_name)] elif role is FieldRole.NESTED_DATACLASS: # Nested dataclasses. value, consumed_keywords_child = construct_dataclass( - field.type, + field_type, value_from_arg, metadata, field_name_prefix=prefixed_field_name @@ -89,10 +96,14 @@ def get_value_from_arg(arg: str) -> Any: if subparser_name is None: # No subparser selected -- this should only happen when we do either # Optional[Union[A, B, ...]] or Union[A, B, None]. - assert type(None) in field.type.__args__ + assert type(None) in field_type.__args__ value = None else: - options = field.type.__args__ + options = field_type.__args__ + options = map( + lambda x: x if x not in type_from_typevar else type_from_typevar[x], + field_type.__args__, + ) chosen_cls = None for option in options: if metadata.subparser_name_from_type[option] == subparser_name: diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index e26825b75..f6672b79f 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -51,6 +51,9 @@ def from_dataclass( ) -> Tuple["ParserDefinition", _construction.ConstructionMetadata]: """Create a parser definition from a dataclass.""" + # TODO: this function is getting bloated and can probably be refactored. There's + # also some repeated logic between here and _construct.py; worth revisiting. + if parent_dataclasses is None: parent_dataclasses = set() @@ -77,8 +80,15 @@ def from_dataclass( # If set to False, we don't directly create an argument from this field. arg_from_field: bool = True + # Resolve field type + field_type = ( + type_from_typevar[field.type] # type: ignore + if field.type in type_from_typevar + else field.type + ) + # Add arguments for nested dataclasses. - if _resolver.is_dataclass(field.type): + if _resolver.is_dataclass(field_type): default = None if default_instance is not None: default = getattr(default_instance, field.name) @@ -86,7 +96,7 @@ def from_dataclass( default = field.default child_definition, child_metadata = ParserDefinition.from_dataclass( - field.type, + field_type, parent_dataclasses | {cls}, subparser_name_from_type=subparser_name_from_type, parent_type_from_typevar=type_from_typevar, @@ -112,9 +122,13 @@ def from_dataclass( arg_from_field = False # Union of dataclasses should create subparsers. - if hasattr(field.type, "__origin__") and field.type.__origin__ is Union: + if hasattr(field_type, "__origin__") and field_type.__origin__ is Union: # We don't use sets here to retain order of subcommands. - options = field.type.__args__ + options = field_type.__args__ + options = map( + lambda x: x if x not in type_from_typevar else type_from_typevar[x], + field_type.__args__, + ) options_no_none = [o for o in options if o != type(None)] # noqa if len(options_no_none) >= 2 and all( map(_resolver.is_dataclass, options_no_none) diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index bcce44bac..8aa847059 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -1,7 +1,7 @@ -import typing import contextlib import dataclasses import io +import typing import pytest diff --git a/tests/test_generics.py b/tests/test_generics.py index 5d275dcd6..6b792a7ed 100644 --- a/tests/test_generics.py +++ b/tests/test_generics.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Generic, TypeVar +from typing import Generic, TypeVar, Union import pytest @@ -112,3 +112,45 @@ class Triangle(Generic[ScalarType]): Point3(1.0, 1.2, 1.3, "world"), Point3(1.0, 1.2, 1.3, "world"), ) + + +def test_generic_nested_dataclass(): + @dataclasses.dataclass + class Child: + a: int + b: int + + T = TypeVar("T") + + @dataclasses.dataclass + class DataclassGeneric(Generic[T]): + child: T + + assert dcargs.parse( + DataclassGeneric[Child], args=["--child.a", "5", "--child.b", "7"] + ) == DataclassGeneric(Child(5, 7)) + + +def test_generic_subparsers(): + @dataclasses.dataclass + class CommandOne: + a: int + + @dataclasses.dataclass + class CommandTwo: + b: int + + T1 = TypeVar("T1") + T2 = TypeVar("T2") + + @dataclasses.dataclass + class Subparser(Generic[T1, T2]): + command: Union[T1, T2] + + assert dcargs.parse( + Subparser[CommandOne, CommandTwo], args="command-one --a 5".split(" ") + ) == Subparser(CommandOne(5)) + + assert dcargs.parse( + Subparser[CommandOne, CommandTwo], args="command-two --b 7".split(" ") + ) == Subparser(CommandTwo(7)) From 4bad48e5aa3cc68fe60079ff8520656ccfecaa79 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 26 Oct 2021 02:46:55 -0700 Subject: [PATCH 006/491] Fix docstring parsing edge case --- dcargs/_docstrings.py | 67 ++++++++++++++++++++++++++++------------ dcargs/_parse.py | 1 + tests/test_docstrings.py | 21 +++++++++++++ 3 files changed, 70 insertions(+), 19 deletions(-) diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index b90baecb2..5370bbec2 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -13,7 +13,7 @@ @dataclasses.dataclass class _Token: token_type: int - token: str + content: str line_number: int @@ -46,7 +46,7 @@ def make(cls) -> "_Tokenization": line_number += 1 tokens_from_line[line_number] = [] elif toktype is not tokenize.INDENT: - token = _Token(token_type=toktype, token=tok, line_number=line_number) + token = _Token(token_type=toktype, content=tok, line_number=line_number) tokens.append(token) tokens_from_line[line_number].append(token) @@ -54,12 +54,12 @@ def make(cls) -> "_Tokenization": for i, token in enumerate(tokens[:-1]): if token.token_type == tokenize.NAME: # Naive heuristic for field names - # This will currently catch variable/argument annotations as well if ( - tokens[i + 1].token == ":" - and token.token not in field_data_from_name + tokens[i + 1].content == ":" + and tokens[i] == tokens_from_line[tokens[i].line_number][0] + and token.content not in field_data_from_name ): - field_data_from_name[token.token] = _FieldData( + field_data_from_name[token.content] = _FieldData( index=i, line_number=token.line_number, prev_field_line_number=prev_field_line_number, @@ -96,22 +96,51 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: field_data = tokenization.field_data_from_name[field_name] # Check for docstring-style comment. - if ( - field_data.line_number + 1 in tokenization.tokens_from_line - and len(tokenization.tokens_from_line[field_data.line_number + 1]) > 0 + line_number = field_data.line_number + 1 + while ( + line_number in tokenization.tokens_from_line + and len(tokenization.tokens_from_line[line_number]) > 0 ): - first_token_on_next_line = tokenization.tokens_from_line[ - field_data.line_number + 1 - ][0] - if first_token_on_next_line.token_type == tokenize.STRING: - docstring = first_token_on_next_line.token.strip() - assert docstring.endswith('"""') and docstring.startswith('"""') - return _strings.dedent(docstring[3:-3]) + first_token = tokenization.tokens_from_line[line_number][0] + first_token_content = first_token.content.strip() + + # Found a docstring! + if ( + first_token.token_type == tokenize.STRING + and first_token_content.startswith('"""') + and first_token_content.endswith('"""') + ): + return _strings.dedent(first_token_content[3:-3]) + + # Found the next field. + if ( + first_token.token_type == tokenize.NAME + and len(tokenization.tokens_from_line[line_number]) >= 2 + and tokenization.tokens_from_line[line_number][1].content == ":" + ): + break + + # Found a method. + if first_token.content == "def": + break + + line_number += 1 + # if ( + # field_data.line_number + 1 in tokenization.tokens_from_line + # and len(tokenization.tokens_from_line[field_data.line_number + 1]) > 0 + # ): + # first_token_on_next_line = tokenization.tokens_from_line[ + # field_data.line_number + 1 + # ][0] + # if first_token_on_next_line.token_type == tokenize.STRING: + # docstring = first_token_on_next_line.token.strip() + # assert docstring.endswith('"""') and docstring.startswith('"""') + # return _strings.dedent(docstring[3:-3]) # Check for comment on the same line as the field. final_token_on_line = tokenization.tokens_from_line[field_data.line_number][-1] if final_token_on_line.token_type == tokenize.COMMENT: - comment: str = final_token_on_line.token + comment: str = final_token_on_line.content assert comment.startswith("#") return comment[1:].strip() @@ -125,8 +154,8 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: comment_token.token_type == tokenize.COMMENT and comment_token.line_number > field_data.prev_field_line_number ): - assert comment_token.token.startswith("#") - comments.append(comment_token.token[1:].strip()) + assert comment_token.content.startswith("#") + comments.append(comment_token.content[1:].strip()) else: break if len(comments) > 0: diff --git a/dcargs/_parse.py b/dcargs/_parse.py index e5ffe7891..3e0b8f98b 100644 --- a/dcargs/_parse.py +++ b/dcargs/_parse.py @@ -10,6 +10,7 @@ def parse( cls: Type[DataclassType], + *, description: Optional[str] = None, args: Optional[Sequence[str]] = None, ) -> DataclassType: diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 8aa847059..a8ef74de0 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -71,3 +71,24 @@ class Config: helptext = f.getvalue() print(helptext) assert " --x INT An optional variable. (default: None)\n" in helptext + + +def test_helptext_hard_string(): + @dataclasses.dataclass + class HelptextHardString: + # fmt: off + x: str = ( + "This docstring may be tougher to parse!" + ) + """Helptext.""" + # fmt: on + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(HelptextHardString, args=["--help"]) + helptext = f.getvalue() + assert ( + "--x STR Helptext. (default: This docstring may be tougher to parse!)\n" + in helptext + ) From 8cf1b2a041c810b73cc77bb68a1587a3e2b5cc1a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 26 Oct 2021 13:48:05 -0700 Subject: [PATCH 007/491] Version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d97363481..1b06dca70 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.5", + version="0.0.6", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", From 94617e98240c666a822a005edf7dd83a6492c050 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 27 Oct 2021 15:53:06 -0700 Subject: [PATCH 008/491] Minor refactor --- README.md | 26 ++--- dcargs/_parse.py | 1 - dcargs/_parsers.py | 259 ++++++++++++++++++++++++++------------------- 3 files changed, 164 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index ca1eb20b1..92d506b5d 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,9 @@ `dcargs` is a tool for generating portable, reusable, and strongly typed CLI interfaces from dataclass definitions. -We expose one function, `parse(Type[T]) -> T`, which takes a dataclass type and -instantiates it via an argparse-style CLI interface. If we create a script -called `simple.py`: +We expose one function, `parse()`, which takes a dataclass type and instantiates +it via an argparse-style CLI interface. If we create a script called +`simple.py`: ```python import dataclasses @@ -99,15 +99,17 @@ in the [tests](./tests/). There are several alternative libraries to `dcargs`; here's a rough summary of some of them: -| | Parsers from dataclasses | Parsers from attrs | Nested dataclasses | Subparsers (via Unions) | Containers | Choices from literals | Docstrings as helptext | -| ----------------------------------------------------------------------------------------------- | ------------------------ | ------------------ | ------------------ | ----------------------- | ---------- | -------------------------------------------------------- | ---------------------- | -| **dcargs** | ✓ | | ✓ | ✓ | ✓ | ✓ | ✓ | -| **[datargs](https://github.com/roee30/datargs)** | ✓ | ✓ | | ✓ | ✓ | ✓ | | -| **[simple-parsing](https://github.com/lebrice/SimpleParsing)** | ✓ | | ✓ | ✓ | ✓ | [soon](https://github.com/lebrice/SimpleParsing/pull/86) | ✓ | -| **[argparse-dataclass](https://pypi.org/project/argparse-dataclass/)** | ✓ | | | | | | | -| **[argparse-dataclasses](https://pypi.org/project/argparse-dataclasses/)** | ✓ | | | | | | | -| **[dataclass-cli](https://github.com/malte-soe/dataclass-cli)** | ✓ | | | | | | | -| **[hf_argparser](https://huggingface.co/transformers/_modules/transformers/hf_argparser.html)** | ✓ | | | | ✓ | | | +| | dataclasses | attrs | Nesting | Subparsers | Containers | Choices from literals | Docstrings as helptext | Generics | +| ----------------------------------------------------------------------------------------------- | ----------- | ----- | ------- | ---------- | ---------- | -------------------------------------------------------- | ---------------------- | -------- | +| **dcargs** | ✓ | | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| **[datargs](https://github.com/roee30/datargs)** | ✓ | ✓ | | ✓ | ✓ | ✓ | | | +| **[typed-argument-parser](https://github.com/swansonk14/typed-argument-parser)** | | | | ✓ | ✓ | ✓ | ✓ | | +| **[simple-parsing](https://github.com/lebrice/SimpleParsing)** | ✓ | | ✓ | ✓ | ✓ | [soon](https://github.com/lebrice/SimpleParsing/pull/86) | ✓ | | +| **[argparse-dataclass](https://pypi.org/project/argparse-dataclass/)** | ✓ | | | | | | | | +| **[argparse-dataclasses](https://pypi.org/project/argparse-dataclasses/)** | ✓ | | | | | | | | +| **[dataclass-cli](https://github.com/malte-soe/dataclass-cli)** | ✓ | | | | | | | | +| **[clout](https://github.com/python-clout/clout)** | | ✓ | ✓ | | | | | | +| **[hf_argparser](https://huggingface.co/transformers/_modules/transformers/hf_argparser.html)** | ✓ | | | | ✓ | | | | Some other distinguishing factors that `dcargs` has put effort into: diff --git a/dcargs/_parse.py b/dcargs/_parse.py index 3e0b8f98b..a16b221ab 100644 --- a/dcargs/_parse.py +++ b/dcargs/_parse.py @@ -22,7 +22,6 @@ def parse( parser_definition, construction_metadata = _parsers.ParserDefinition.from_dataclass( cls, parent_dataclasses=set(), # Used for recursive calls. - subparser_name_from_type={}, # Aliases for subparsers; this is working, but not yet exposed. parent_type_from_typevar=None, # Used for recursive calls. default_instance=None, # Overrides for default values. This could also be exposed. ) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index f6672b79f..1708e2e73 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -1,6 +1,6 @@ import argparse import dataclasses -from typing import Dict, List, Optional, Set, Tuple, Type, TypeVar, Union +from typing import Any, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union from typing_extensions import _GenericAlias # type: ignore @@ -13,7 +13,7 @@ class ParserDefinition: """Each parser contains a list of arguments and optionally a subparser.""" - args: List["_arguments.ArgumentDefinition"] + args: List[_arguments.ArgumentDefinition] subparsers: Optional["SubparsersDefinition"] def apply(self, parser: argparse.ArgumentParser) -> None: @@ -45,15 +45,11 @@ def apply(self, parser: argparse.ArgumentParser) -> None: def from_dataclass( cls: Union[Type[T], _GenericAlias], parent_dataclasses: Optional[Set[Type]], - subparser_name_from_type: Dict[Type, str], parent_type_from_typevar: Optional[Dict[TypeVar, Type]], default_instance: Optional[T], ) -> Tuple["ParserDefinition", _construction.ConstructionMetadata]: """Create a parser definition from a dataclass.""" - # TODO: this function is getting bloated and can probably be refactored. There's - # also some repeated logic between here and _construct.py; worth revisiting. - if parent_dataclasses is None: parent_dataclasses = set() @@ -69,123 +65,56 @@ def from_dataclass( assert ( cls not in parent_dataclasses ), f"Found a cyclic dataclass dependency with type {cls}" + parent_dataclasses = parent_dataclasses | {cls} args = [] subparsers = None metadata = _construction.ConstructionMetadata() for field in _resolver.resolved_fields(cls): # type: ignore + + # Ignore fields not in constructor if not field.init: continue - # If set to False, we don't directly create an argument from this field. - arg_from_field: bool = True - - # Resolve field type - field_type = ( - type_from_typevar[field.type] # type: ignore - if field.type in type_from_typevar - else field.type + field_parser = _NestedDataclassHandler( + cls, + field, + type_from_typevar, + parent_dataclasses, + default_instance, ) - # Add arguments for nested dataclasses. - if _resolver.is_dataclass(field_type): - default = None - if default_instance is not None: - default = getattr(default_instance, field.name) - elif field.default is not dataclasses.MISSING: - default = field.default - - child_definition, child_metadata = ParserDefinition.from_dataclass( - field_type, - parent_dataclasses | {cls}, - subparser_name_from_type=subparser_name_from_type, - parent_type_from_typevar=type_from_typevar, - default_instance=default, - ) - metadata.update(child_metadata) + # Try to create subparsers from this field. + subparsers_out = field_parser.handle_unions_over_dataclasses() + if subparsers_out is not None: + assert ( + subparsers is None + ), "Only one subparser (union over dataclasses) is allowed per class" + subparsers, subparsers_metadata = subparsers_out + metadata.update(subparsers_metadata) - child_args = child_definition.args - for i, arg in enumerate(child_args): - child_args[i] = arg.prefix( - field.name + _strings.NESTED_DATACLASS_DELIMETER - ) + continue + # Try to interpret field as a nested dataclass. + nested_out = field_parser.handle_nested_dataclasses() + if nested_out is not None: + child_args, child_metadata = nested_out args.extend(child_args) + metadata.update(child_metadata) + + continue - if child_definition.subparsers is not None: - assert subparsers is None - subparsers = child_definition.subparsers - - metadata.role_from_field[ - field - ] = _construction.FieldRole.NESTED_DATACLASS - arg_from_field = False - - # Union of dataclasses should create subparsers. - if hasattr(field_type, "__origin__") and field_type.__origin__ is Union: - # We don't use sets here to retain order of subcommands. - options = field_type.__args__ - options = map( - lambda x: x if x not in type_from_typevar else type_from_typevar[x], - field_type.__args__, - ) - options_no_none = [o for o in options if o != type(None)] # noqa - if len(options_no_none) >= 2 and all( - map(_resolver.is_dataclass, options_no_none) - ): - assert ( - subparsers is None - ), "Only one subparser group is supported per dataclass" - assert ( - field.default == dataclasses.MISSING - ), "Default dataclass value not yet supported for subparser definitions" - - parsers: Dict[str, ParserDefinition] = {} - for option in options_no_none: - subparser_name = ( - subparser_name_from_type[option] - if option in subparser_name_from_type - else _strings.hyphen_separated_from_camel_case( - option.__name__ - ) - ) - metadata.subparser_name_from_type[option] = subparser_name - - ( - parsers[subparser_name], - child_metadata, - ) = ParserDefinition.from_dataclass( - option, - parent_dataclasses | {cls}, - subparser_name_from_type, - parent_type_from_typevar=type_from_typevar, - default_instance=None, - ) - metadata.update(child_metadata) - - subparsers = SubparsersDefinition( - name=field.name, - description=_docstrings.get_field_docstring(cls, field.name), - parsers=parsers, - required=( - options == options_no_none - ), # Not required if no options. - ) - metadata.role_from_field[field] = _construction.FieldRole.SUBPARSERS - arg_from_field = False - - # Make an argument! - if arg_from_field: - arg, role = _arguments.ArgumentDefinition.make_from_field( - cls, - field, - type_from_typevar, - default_override=getattr(default_instance, field.name) - if default_instance is not None - else None, - ) - args.append(arg) - metadata.role_from_field[field] = role + # Handle simple fields! + arg, role = _arguments.ArgumentDefinition.make_from_field( + cls, + field, + type_from_typevar, + default_override=getattr(default_instance, field.name) + if default_instance is not None + else None, + ) + args.append(arg) + metadata.role_from_field[field] = role return ( ParserDefinition( @@ -204,3 +133,115 @@ class SubparsersDefinition: description: Optional[str] parsers: Dict[str, ParserDefinition] required: bool + + +@dataclasses.dataclass +class _NestedDataclassHandler: + """Helper functions for handling nested dataclasses, which are converted to either + subparsers or prefixed fields.""" + + cls: Type + field: dataclasses.Field + type_from_typevar: Dict[TypeVar, Type] + parent_dataclasses: Set[Type] + default_instance: Any + + def handle_unions_over_dataclasses( + self, + ) -> Optional[Tuple["SubparsersDefinition", _construction.ConstructionMetadata]]: + """Handle unions over dataclasses, which are converted to subparsers.. Returns + `None` if not applicable.""" + + metadata = _construction.ConstructionMetadata() + + # Union of dataclasses should create subparsers. + if ( + not hasattr(self.field.type, "__origin__") + or self.field.type.__origin__ is not Union + ): + return None + + # We don't use sets here to retain order of subcommands. + options = self.field.type.__args__ + options = map( + lambda x: x + if x not in self.type_from_typevar + else self.type_from_typevar[x], + self.field.type.__args__, + ) + options_no_none = [o for o in options if o != type(None)] # noqa + if len(options_no_none) < 2 or not all( + map(_resolver.is_dataclass, options_no_none) + ): + return None + + assert ( + self.field.default == dataclasses.MISSING + ), "Default dataclass value not yet supported for subparser definitions" + + parsers: Dict[str, ParserDefinition] = {} + for option in options_no_none: + subparser_name = _strings.hyphen_separated_from_camel_case(option.__name__) + metadata.subparser_name_from_type[option] = subparser_name + + ( + parsers[subparser_name], + child_metadata, + ) = ParserDefinition.from_dataclass( + option, + self.parent_dataclasses, + parent_type_from_typevar=self.type_from_typevar, + default_instance=None, + ) + metadata.update(child_metadata) + + subparsers = SubparsersDefinition( + name=self.field.name, + description=_docstrings.get_field_docstring(self.cls, self.field.name), + parsers=parsers, + required=(options == options_no_none), # Not required if no options. + ) + metadata.role_from_field[self.field] = _construction.FieldRole.SUBPARSERS + + return subparsers, metadata + + def handle_nested_dataclasses( + self, + ) -> Optional[ + Tuple[List[_arguments.ArgumentDefinition], _construction.ConstructionMetadata] + ]: + """Handle nested dataclasses. Returns `None` if not applicable.""" + # Resolve field type + field_type = ( + self.type_from_typevar[self.field.type] # type: ignore + if self.field.type in self.type_from_typevar # type: ignore + else self.field.type + ) + if not _resolver.is_dataclass(field_type): + return None + + # Add arguments for nested dataclasses. + default = None + if self.default_instance is not None: + default = getattr(self.default_instance, self.field.name) + elif self.field.default is not dataclasses.MISSING: + default = self.field.default + + child_definition, child_metadata = ParserDefinition.from_dataclass( + field_type, + self.parent_dataclasses, + parent_type_from_typevar=self.type_from_typevar, + default_instance=default, + ) + + child_args = child_definition.args + for i, arg in enumerate(child_args): + child_args[i] = arg.prefix( + self.field.name + _strings.NESTED_DATACLASS_DELIMETER + ) + + child_metadata.role_from_field[ + self.field + ] = _construction.FieldRole.NESTED_DATACLASS + + return child_args, child_metadata From 3c438da26ae9b7fe0f98d1f019db92827ab942e2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 27 Oct 2021 17:00:22 -0700 Subject: [PATCH 009/491] Support multi-type tuples, sets --- README.md | 4 ++- dcargs/_arguments.py | 76 ++++++++++++++++++++++++++-------------- dcargs/_construction.py | 34 +++++++++--------- dcargs/_parsers.py | 4 +-- tests/test_dcargs.py | 26 +++++++++++++- tests/test_docstrings.py | 17 +++++++-- 6 files changed, 110 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 92d506b5d..203767124 100644 --- a/README.md +++ b/README.md @@ -77,9 +77,11 @@ containing: - `typing.Literal` types (populates argparse's `choices`) - `typing.Sequence` types (populates argparse's `nargs`) - `typing.List` types (populates argparse's `nargs`) - - `typing.Tuple` types, such as `typing.Tuple[T, T, T]` or + - `typing.Tuple` types, such as `typing.Tuple[T1, T2, T3]` or `typing.Tuple[T, ...]` (populates argparse's `nargs`, and converts automatically) + - `typing.Set` types (populates argparse's `nargs`, and converts + automatically) - `typing.Final` types and `typing.Annotated` (for parsing, these are effectively no-ops) - Nested combinations of the above: `Optional[Literal[T]]`, diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index ba6878bbe..224c43571 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -26,7 +26,7 @@ class ArgumentDefinition: nargs: Optional[Union[int, str]] = None default: Optional[Any] = None choices: Optional[Set[Any]] = None - metavar: Optional[str] = None + metavar: Optional[Union[str, Tuple[str, ...]]] = None help: Optional[str] = None dest: Optional[str] = None @@ -72,7 +72,7 @@ def make_from_field( # Propagate argument through transforms until stable. prev_arg = arg - role: _construction.FieldRole = _construction.FieldRole.VANILLA_FIELD + role: _construction.FieldRole = _construction.FieldRoleEnum.VANILLA_FIELD def _handle_generics(arg: ArgumentDefinition) -> _ArgumentTransformOutput: """Handle generic arguments. Note that this needs to be a transform -- if we @@ -97,7 +97,7 @@ def _handle_generics(arg: ArgumentDefinition) -> _ArgumentTransformOutput: # Update field role. if new_role is not None: assert ( - role == _construction.FieldRole.VANILLA_FIELD + role == _construction.FieldRoleEnum.VANILLA_FIELD ), "Something went wrong -- only one field role can be specified per argument!" role = new_role @@ -226,16 +226,20 @@ def _bool_flags(arg: ArgumentDefinition) -> _ArgumentTransformOutput: assert False, "Invalid default" -def _nargs_from_sequences_and_lists( +def _nargs_from_sequences_lists_and_sets( arg: ArgumentDefinition, ) -> _ArgumentTransformOutput: """Transform for handling Sequence[T] and list types.""" if hasattr(arg.type, "__origin__") and arg.type.__origin__ in ( # type: ignore collections.abc.Sequence, # different from typing.Sequence! list, # different from typing.List! + set, # different from typing.Set! ): assert len(arg.type.__args__) == 1 # type: ignore (typ,) = arg.type.__args__ # type: ignore + role = arg.type.__origin__ # type: ignore + if role is collections.abc.Sequence: + role = list return ( dataclasses.replace( @@ -246,7 +250,7 @@ def _nargs_from_sequences_and_lists( # input, they can use Optional[Tuple[...]] nargs="+", ), - None, + role, ) else: return arg, None @@ -254,28 +258,46 @@ def _nargs_from_sequences_and_lists( def _nargs_from_tuples(arg: ArgumentDefinition) -> _ArgumentTransformOutput: """Transform for handling Tuple[T, T, ...] types.""" - if hasattr(arg.type, "__origin__") and arg.type.__origin__ == tuple: # type: ignore - argset = set(arg.type.__args__) # type: ignore - argset_no_ellipsis = argset - {Ellipsis} # - assert len(argset_no_ellipsis) == 1, "Tuples must be of a single type!" - - if argset != argset_no_ellipsis: - # `*` is >=0 values, `+` is >=1 values. - # We're going to require at least 1 value; if a user wants to accept no - # input, they can use Optional[Tuple[...]]. - nargs = "+" + + if arg.nargs is None and hasattr(arg.type, "__origin__") and arg.type.__origin__ == tuple: # type: ignore + types = arg.type.__args__ # type: ignore + typeset = set(types) + typeset_no_ellipsis = typeset - {Ellipsis} # + + if typeset_no_ellipsis != typeset: + # Ellipsis: variable argument counts + assert ( + len(typeset_no_ellipsis) == 1 + ), "If ellipsis is used, tuples must contain only one type." + (typ,) = typeset_no_ellipsis + + return ( + dataclasses.replace( + arg, + # `*` is >=0 values, `+` is >=1 values. + # We're going to require at least 1 value; if a user wants to accept no + # input, they can use Optional[Tuple[...]]. + nargs="+", + type=typ, + ), + tuple, + ) else: - nargs = len(arg.type.__args__) # type: ignore - (typ,) = argset_no_ellipsis + # Tuples with more than one type + assert arg.metavar is None + return ( + dataclasses.replace( + arg, + nargs=len(types), + type=str, # Types will be converted in the dataclass reconstruction step + metavar=tuple( + t.__name__.upper() if hasattr(t, "__name__") else "X" + for t in types + ), + ), + lambda str_list: tuple(typ(x) for typ, x in zip(types, str_list)), + ) - return ( - dataclasses.replace( - arg, - nargs=nargs, - type=typ, - ), - _construction.FieldRole.TUPLE, - ) else: return arg, None @@ -314,7 +336,7 @@ def _enums_as_strings(arg: ArgumentDefinition) -> _ArgumentTransformOutput: type=str, default=None if arg.default is None else arg.default.name, ), - _construction.FieldRole.ENUM, + lambda enum_name: arg.type[enum_name], # type: ignore ) else: return arg, None @@ -359,7 +381,7 @@ def _use_type_as_metavar(arg: ArgumentDefinition) -> _ArgumentTransformOutput: _handle_optionals, _populate_defaults, _bool_flags, - _nargs_from_sequences_and_lists, + _nargs_from_sequences_lists_and_sets, _nargs_from_tuples, _choices_from_literals, _enums_as_strings, diff --git a/dcargs/_construction.py b/dcargs/_construction.py index b6eaa08d0..5289a4948 100644 --- a/dcargs/_construction.py +++ b/dcargs/_construction.py @@ -1,6 +1,6 @@ import dataclasses import enum -from typing import Any, Dict, Set, Tuple, Type, TypeVar, Union +from typing import Any, Callable, Dict, Set, Tuple, Type, TypeVar, Union from typing_extensions import _GenericAlias # type: ignore @@ -9,16 +9,18 @@ DataclassType = TypeVar("DataclassType", bound=Union[Type, _GenericAlias]) -class FieldRole(enum.Enum): - """Enum for specifying special behaviors for .""" - +# Each dataclass field is assigned a role, which is either taken from an enum or a +# callable type that converts raw values from the argparse namespace to their final +# values in the dataclass. +class FieldRoleEnum(enum.Enum): VANILLA_FIELD = enum.auto() - TUPLE = enum.auto() - ENUM = enum.auto() NESTED_DATACLASS = enum.auto() # Singular nested dataclass. SUBPARSERS = enum.auto() # Unions over dataclasses. +FieldRole = Union[FieldRoleEnum, Callable[[Any], Any]] + + @dataclasses.dataclass class ConstructionMetadata: """Metadata recorded during parsing that's needed for reconstructing dataclasses.""" @@ -74,10 +76,7 @@ def get_value_from_arg(arg: str) -> Any: else field.type ) - if role is FieldRole.ENUM: - # Handle enums. - value = field_type[get_value_from_arg(prefixed_field_name)] - elif role is FieldRole.NESTED_DATACLASS: + if role is FieldRoleEnum.NESTED_DATACLASS: # Nested dataclasses. value, consumed_keywords_child = construct_dataclass( field_type, @@ -87,7 +86,7 @@ def get_value_from_arg(arg: str) -> Any: + _strings.NESTED_DATACLASS_DELIMETER, ) consumed_keywords |= consumed_keywords_child - elif role is FieldRole.SUBPARSERS: + elif role is FieldRoleEnum.SUBPARSERS: # Unions over dataclasses (subparsers). subparser_dest = _strings.SUBPARSER_DEST_FMT.format( name=prefixed_field_name @@ -116,15 +115,14 @@ def get_value_from_arg(arg: str) -> Any: metadata, ) consumed_keywords |= consumed_keywords_child - elif role is FieldRole.TUPLE: - # For sequences, argparse always gives us lists. Sometimes we want tuples. - value = get_value_from_arg(prefixed_field_name) - if value is not None: - value = tuple(value) - - elif role is FieldRole.VANILLA_FIELD: + elif role is FieldRoleEnum.VANILLA_FIELD: # General case. value = get_value_from_arg(prefixed_field_name) + elif callable(role): + # Callable roles. Used for tuples, lists, sets, etc. + value = get_value_from_arg(prefixed_field_name) + if value is not None: + value = role(value) else: assert False diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 1708e2e73..741593c6b 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -201,7 +201,7 @@ def handle_unions_over_dataclasses( parsers=parsers, required=(options == options_no_none), # Not required if no options. ) - metadata.role_from_field[self.field] = _construction.FieldRole.SUBPARSERS + metadata.role_from_field[self.field] = _construction.FieldRoleEnum.SUBPARSERS return subparsers, metadata @@ -242,6 +242,6 @@ def handle_nested_dataclasses( child_metadata.role_from_field[ self.field - ] = _construction.FieldRole.NESTED_DATACLASS + ] = _construction.FieldRoleEnum.NESTED_DATACLASS return child_args, child_metadata diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index bbd64c92c..fc7dcd8eb 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -1,7 +1,7 @@ import dataclasses import enum import pathlib -from typing import ClassVar, List, Optional, Sequence, Tuple, Union +from typing import ClassVar, List, Optional, Sequence, Set, Tuple, Union import pytest from typing_extensions import Annotated, Final, Literal # Backward compatibility. @@ -140,6 +140,18 @@ class A: dcargs.parse(A, args=[]) +def test_sets(): + @dataclasses.dataclass + class A: + x: Set[int] + + assert dcargs.parse(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + def test_optional_sequences(): @dataclasses.dataclass class A: @@ -174,6 +186,18 @@ class A: dcargs.parse(A, args=[]) +def test_tuples_fixed_multitype(): + @dataclasses.dataclass + class A: + x: Tuple[int, str, float] + + assert dcargs.parse(A, args=["--x", "1", "2", "3.5"]) == A(x=(1, "2", 3.5)) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + def test_tuples_variable(): @dataclasses.dataclass class A: diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index a8ef74de0..b61a29c29 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -1,7 +1,7 @@ import contextlib import dataclasses import io -import typing +from typing import Optional, Tuple import pytest @@ -61,7 +61,7 @@ class HelptextMultiline: def test_none_default_value_helptext(): @dataclasses.dataclass class Config: - x: typing.Optional[int] = None + x: Optional[int] = None """An optional variable.""" f = io.StringIO() @@ -92,3 +92,16 @@ class HelptextHardString: "--x STR Helptext. (default: This docstring may be tougher to parse!)\n" in helptext ) + + +def test_tuple_helptext(): + @dataclasses.dataclass + class TupleHelptext: + x: Tuple[int, str, float] + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(TupleHelptext, args=["--help"]) + helptext = f.getvalue() + assert "--x INT STR FLOAT\n" in helptext From 5c6b610b7e3de7137f6f3e563537a524a815ee02 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 13 Nov 2021 17:33:54 -0800 Subject: [PATCH 010/491] Hide default values for boolean flags --- dcargs/_arguments.py | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 224c43571..71de23191 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -193,6 +193,14 @@ def _bool_flags(arg: ArgumentDefinition) -> _ArgumentTransformOutput: if arg.type != bool: return arg, None + # Populate helptext for boolean flags => don't show default value, which can be + # confusing. + helptext = (_docstrings.get_field_docstring(arg.parent_class, arg.field.name),) + arg = dataclasses.replace( + arg, + help=helptext if helptext is not None else "", + ) + if arg.default is None: return ( dataclasses.replace( diff --git a/setup.py b/setup.py index 1b06dca70..02ba050d4 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.6", + version="0.0.7", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", From 03257fb143956b33c911d5871186e35e90f5ecbf Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 16 Nov 2021 13:30:47 -0800 Subject: [PATCH 011/491] Remove accidental tuple --- dcargs/_arguments.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 71de23191..4c4579999 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -195,7 +195,7 @@ def _bool_flags(arg: ArgumentDefinition) -> _ArgumentTransformOutput: # Populate helptext for boolean flags => don't show default value, which can be # confusing. - helptext = (_docstrings.get_field_docstring(arg.parent_class, arg.field.name),) + helptext = _docstrings.get_field_docstring(arg.parent_class, arg.field.name) arg = dataclasses.replace( arg, help=helptext if helptext is not None else "", diff --git a/setup.py b/setup.py index 02ba050d4..55e787b76 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.7", + version="0.0.8", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", From 06467664ee5ee6a69e12c2c82385ed70de5469da Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 18 Nov 2021 15:12:09 -0800 Subject: [PATCH 012/491] Bump typing_extensions; remove GenericAlias dep --- dcargs/_construction.py | 4 +--- dcargs/_docstrings.py | 9 ++++----- dcargs/_parse.py | 4 +--- dcargs/_parsers.py | 4 +--- dcargs/_resolver.py | 29 +++++++++++++++++------------ setup.py | 2 +- 6 files changed, 25 insertions(+), 27 deletions(-) diff --git a/dcargs/_construction.py b/dcargs/_construction.py index 5289a4948..83764ecd1 100644 --- a/dcargs/_construction.py +++ b/dcargs/_construction.py @@ -2,11 +2,9 @@ import enum from typing import Any, Callable, Dict, Set, Tuple, Type, TypeVar, Union -from typing_extensions import _GenericAlias # type: ignore - from . import _resolver, _strings -DataclassType = TypeVar("DataclassType", bound=Union[Type, _GenericAlias]) +DataclassType = TypeVar("DataclassType") # Each dataclass field is assigned a role, which is either taken from an enum or a diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 5370bbec2..70995eb56 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -5,9 +5,7 @@ import tokenize from typing import Dict, List, Optional, Type -from typing_extensions import _GenericAlias # type: ignore - -from . import _strings +from . import _resolver, _strings @dataclasses.dataclass @@ -76,8 +74,9 @@ def make(cls) -> "_Tokenization": def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: """Get docstring for a field in a class.""" - if isinstance(cls, _GenericAlias): - cls = cls.__origin__ + origin_cls = _resolver.unwrap_generic(cls) + if origin_cls is not None: + cls = origin_cls assert dataclasses.is_dataclass(cls) try: diff --git a/dcargs/_parse.py b/dcargs/_parse.py index a16b221ab..c02382d52 100644 --- a/dcargs/_parse.py +++ b/dcargs/_parse.py @@ -1,11 +1,9 @@ import argparse from typing import Optional, Sequence, Type, TypeVar, Union -from typing_extensions import _GenericAlias # type: ignore - from . import _construction, _parsers, _strings -DataclassType = TypeVar("DataclassType", bound=Union[Type, _GenericAlias]) +DataclassType = TypeVar("DataclassType") def parse( diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 741593c6b..909ae867f 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -2,8 +2,6 @@ import dataclasses from typing import Any, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union -from typing_extensions import _GenericAlias # type: ignore - from . import _arguments, _construction, _docstrings, _resolver, _strings T = TypeVar("T") @@ -43,7 +41,7 @@ def apply(self, parser: argparse.ArgumentParser) -> None: @staticmethod def from_dataclass( - cls: Union[Type[T], _GenericAlias], + cls: Type[T], parent_dataclasses: Optional[Set[Type]], parent_type_from_typevar: Optional[Dict[TypeVar, Type]], default_instance: Optional[T], diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 9687ba0a1..f2192aecd 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -1,36 +1,41 @@ import copy import dataclasses import functools -from typing import Dict, List, Tuple, Type, TypeVar, Union +from typing import Dict, List, Optional, Tuple, Type, TypeVar, Union -from typing_extensions import _GenericAlias, get_type_hints # type: ignore +from typing_extensions import get_type_hints -def is_dataclass(cls: Union[Type, _GenericAlias]) -> bool: +def unwrap_generic(cls: Type) -> Optional[Type]: + """Returns the origin of a generic type; or None if not a generic.""" + # Note that isinstance(cls, GenericAlias) breaks in Python >= 3.9 + return cls.__origin__ if hasattr(cls, "__origin__") else None + + +def is_dataclass(cls: Type) -> bool: """Same as `dataclasses.is_dataclass`, but also handles generic aliases.""" - return dataclasses.is_dataclass(cls) or ( - isinstance(cls, _GenericAlias) and dataclasses.is_dataclass(cls.__origin__) - ) + origin_cls = unwrap_generic(cls) + return dataclasses.is_dataclass(cls if origin_cls is None else origin_cls) def resolve_generic_dataclasses( - cls: Union[Type, _GenericAlias], + cls: Type, ) -> Tuple[Type, Dict[TypeVar, Type]]: """If the input is a dataclass: no-op. If it's a generic alias: returns the root dataclass, and a mapping from typevars to concrete types.""" - if isinstance(cls, _GenericAlias): - typevars = cls.__origin__.__parameters__ + origin_cls = unwrap_generic(cls) + if origin_cls is not None: + typevars = origin_cls.__parameters__ typevar_values = cls.__args__ assert len(typevars) == len(typevar_values) - cls = cls.__origin__ - return cls, dict(zip(typevars, typevar_values)) + return origin_cls, dict(zip(typevars, typevar_values)) else: return cls, {} @functools.lru_cache(maxsize=16) -def resolved_fields(cls: Union[Type, _GenericAlias]) -> List[dataclasses.Field]: +def resolved_fields(cls: Type) -> List[dataclasses.Field]: """Similar to dataclasses.fields, but resolves forward references.""" assert dataclasses.is_dataclass(cls) diff --git a/setup.py b/setup.py index 55e787b76..d4edb4320 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ packages=find_packages(), package_data={"dcargs": ["py.typed"]}, python_requires=">=3.7", - install_requires=["typing_extensions"], + install_requires=["typing_extensions>=4.0.0"], extras_require={ "testing": [ "pytest", From 1d789d53a4348a58eba0dbd59fafce811668bfdd Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 18 Nov 2021 15:45:41 -0800 Subject: [PATCH 013/491] Use more standard typing helpers --- dcargs/_arguments.py | 31 +++++++++++++++---------------- dcargs/_construction.py | 7 ++++--- dcargs/_docstrings.py | 6 ++++-- dcargs/_parse.py | 2 +- dcargs/_parsers.py | 12 +++++------- dcargs/_resolver.py | 16 +++++----------- 6 files changed, 34 insertions(+), 40 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 4c4579999..db5900be6 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -4,7 +4,7 @@ import enum from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union -from typing_extensions import Final, Literal, _AnnotatedAlias # Backward compatibility. +from typing_extensions import Final, Literal, _AnnotatedAlias, get_args, get_origin from . import _construction, _docstrings, _strings @@ -117,8 +117,8 @@ def _handle_generics(arg: ArgumentDefinition) -> _ArgumentTransformOutput: def _unwrap_final(arg: ArgumentDefinition) -> _ArgumentTransformOutput: """Treat Final[T] as just T.""" - if hasattr(arg.type, "__origin__") and arg.type.__origin__ is Final: # type: ignore - (typ,) = arg.type.__args__ # type: ignore + if get_origin(arg.type) is Final: + (typ,) = get_args(arg.type) return ( dataclasses.replace( arg, @@ -133,7 +133,7 @@ def _unwrap_final(arg: ArgumentDefinition) -> _ArgumentTransformOutput: def _unwrap_annotated(arg: ArgumentDefinition) -> _ArgumentTransformOutput: """Treat Annotated[T, annotation] as just T.""" if hasattr(arg.type, "__class__") and arg.type.__class__ == _AnnotatedAlias: - typ = arg.type.__origin__ # type: ignore + typ = get_origin(arg.type) return ( dataclasses.replace( arg, @@ -148,8 +148,8 @@ def _unwrap_annotated(arg: ArgumentDefinition) -> _ArgumentTransformOutput: def _handle_optionals(arg: ArgumentDefinition) -> _ArgumentTransformOutput: """Transform for handling Optional[T] types. Sets default to None and marks arg as not required.""" - if hasattr(arg.type, "__origin__") and arg.type.__origin__ is Union: # type: ignore - options = set(arg.type.__args__) # type: ignore + if get_origin(arg.type) is Union: + options = set(get_args(arg.type)) assert ( len(options) == 2 and type(None) in options ), "Union must be either over dataclasses (for subparsers) or Optional" @@ -238,14 +238,13 @@ def _nargs_from_sequences_lists_and_sets( arg: ArgumentDefinition, ) -> _ArgumentTransformOutput: """Transform for handling Sequence[T] and list types.""" - if hasattr(arg.type, "__origin__") and arg.type.__origin__ in ( # type: ignore + if get_origin(arg.type) in ( collections.abc.Sequence, # different from typing.Sequence! list, # different from typing.List! set, # different from typing.Set! ): - assert len(arg.type.__args__) == 1 # type: ignore - (typ,) = arg.type.__args__ # type: ignore - role = arg.type.__origin__ # type: ignore + (typ,) = get_args(arg.type) + role = get_origin(arg.type) if role is collections.abc.Sequence: role = list @@ -267,10 +266,10 @@ def _nargs_from_sequences_lists_and_sets( def _nargs_from_tuples(arg: ArgumentDefinition) -> _ArgumentTransformOutput: """Transform for handling Tuple[T, T, ...] types.""" - if arg.nargs is None and hasattr(arg.type, "__origin__") and arg.type.__origin__ == tuple: # type: ignore - types = arg.type.__args__ # type: ignore + if arg.nargs is None and get_origin(arg.type) is tuple: + types = get_args(arg.type) typeset = set(types) - typeset_no_ellipsis = typeset - {Ellipsis} # + typeset_no_ellipsis = typeset - {Ellipsis} if typeset_no_ellipsis != typeset: # Ellipsis: variable argument counts @@ -312,15 +311,15 @@ def _nargs_from_tuples(arg: ArgumentDefinition) -> _ArgumentTransformOutput: def _choices_from_literals(arg: ArgumentDefinition) -> _ArgumentTransformOutput: """For literal types, set choices.""" - if hasattr(arg.type, "__origin__") and arg.type.__origin__ is Literal: # type: ignore - choices = set(arg.type.__args__) # type: ignore + if get_origin(arg.type) is Literal: + choices = set(get_args(arg.type)) assert ( len(set(map(type, choices))) == 1 ), "All choices in literal must have the same type!" return ( dataclasses.replace( arg, - type=type(arg.type.__args__[0]), # type: ignore + type=type(next(iter(choices))), choices=choices, ), None, diff --git a/dcargs/_construction.py b/dcargs/_construction.py index 83764ecd1..bf2f7fb73 100644 --- a/dcargs/_construction.py +++ b/dcargs/_construction.py @@ -2,6 +2,8 @@ import enum from typing import Any, Callable, Dict, Set, Tuple, Type, TypeVar, Union +from typing_extensions import get_args + from . import _resolver, _strings DataclassType = TypeVar("DataclassType") @@ -93,13 +95,12 @@ def get_value_from_arg(arg: str) -> Any: if subparser_name is None: # No subparser selected -- this should only happen when we do either # Optional[Union[A, B, ...]] or Union[A, B, None]. - assert type(None) in field_type.__args__ + assert type(None) in get_args(field_type) value = None else: - options = field_type.__args__ options = map( lambda x: x if x not in type_from_typevar else type_from_typevar[x], - field_type.__args__, + get_args(field_type), ) chosen_cls = None for option in options: diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 70995eb56..16e302178 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -5,7 +5,9 @@ import tokenize from typing import Dict, List, Optional, Type -from . import _resolver, _strings +from typing_extensions import get_origin + +from . import _strings @dataclasses.dataclass @@ -74,7 +76,7 @@ def make(cls) -> "_Tokenization": def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: """Get docstring for a field in a class.""" - origin_cls = _resolver.unwrap_generic(cls) + origin_cls = get_origin(cls) if origin_cls is not None: cls = origin_cls diff --git a/dcargs/_parse.py b/dcargs/_parse.py index c02382d52..861fbe716 100644 --- a/dcargs/_parse.py +++ b/dcargs/_parse.py @@ -1,5 +1,5 @@ import argparse -from typing import Optional, Sequence, Type, TypeVar, Union +from typing import Optional, Sequence, Type, TypeVar from . import _construction, _parsers, _strings diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 909ae867f..a62c1eb61 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -2,6 +2,8 @@ import dataclasses from typing import Any, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union +from typing_extensions import get_args, get_origin + from . import _arguments, _construction, _docstrings, _resolver, _strings T = TypeVar("T") @@ -153,19 +155,15 @@ def handle_unions_over_dataclasses( metadata = _construction.ConstructionMetadata() # Union of dataclasses should create subparsers. - if ( - not hasattr(self.field.type, "__origin__") - or self.field.type.__origin__ is not Union - ): + if get_origin(self.field.type) is not Union: return None - # We don't use sets here to retain order of subcommands. - options = self.field.type.__args__ + # We don't use sets here to retain order of subcommands. options = map( lambda x: x if x not in self.type_from_typevar else self.type_from_typevar[x], - self.field.type.__args__, + get_args(self.field.type), ) options_no_none = [o for o in options if o != type(None)] # noqa if len(options_no_none) < 2 or not all( diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index f2192aecd..e09fbf812 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -1,20 +1,14 @@ import copy import dataclasses import functools -from typing import Dict, List, Optional, Tuple, Type, TypeVar, Union +from typing import Dict, List, Tuple, Type, TypeVar -from typing_extensions import get_type_hints - - -def unwrap_generic(cls: Type) -> Optional[Type]: - """Returns the origin of a generic type; or None if not a generic.""" - # Note that isinstance(cls, GenericAlias) breaks in Python >= 3.9 - return cls.__origin__ if hasattr(cls, "__origin__") else None +from typing_extensions import get_args, get_origin, get_type_hints def is_dataclass(cls: Type) -> bool: """Same as `dataclasses.is_dataclass`, but also handles generic aliases.""" - origin_cls = unwrap_generic(cls) + origin_cls = get_origin(cls) return dataclasses.is_dataclass(cls if origin_cls is None else origin_cls) @@ -24,10 +18,10 @@ def resolve_generic_dataclasses( """If the input is a dataclass: no-op. If it's a generic alias: returns the root dataclass, and a mapping from typevars to concrete types.""" - origin_cls = unwrap_generic(cls) + origin_cls = get_origin(cls) if origin_cls is not None: typevars = origin_cls.__parameters__ - typevar_values = cls.__args__ + typevar_values = get_args(cls) assert len(typevars) == len(typevar_values) return origin_cls, dict(zip(typevars, typevar_values)) else: From ec216b870babb784c51975fcd336e6a07f5d0d86 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 21 Nov 2021 18:36:45 -0800 Subject: [PATCH 014/491] Add serialization API --- .github/workflows/mypy.yml | 2 +- README.md | 52 +++++-- dcargs/__init__.py | 7 +- dcargs/_serialization.py | 142 ++++++++++++++++++ examples/generics.py | 6 + examples/generics_alt.py | 34 +++++ examples/simple.py | 2 + setup.py | 4 +- ....py => test_generics_and_serialization.py} | 80 ++++++---- 9 files changed, 280 insertions(+), 49 deletions(-) create mode 100644 dcargs/_serialization.py create mode 100644 examples/generics_alt.py rename tests/{test_generics.py => test_generics_and_serialization.py} (62%) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 7ea44da67..7e98a8a7b 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -26,4 +26,4 @@ jobs: pip install -e .[type-checking] - name: Test with mypy run: | - mypy . + mypy --install-types --non-interactive . diff --git a/README.md b/README.md index 203767124..a9ed2c0e7 100644 --- a/README.md +++ b/README.md @@ -6,18 +6,39 @@ +* [Simple example](#simple-example) * [Feature list](#feature-list) * [Comparisons to alternative tools](#comparisons-to-alternative-tools) -* [Example usage](#example-usage) +* [Nested example](#nested-example) -`dcargs` is a tool for generating portable, reusable, and strongly typed CLI -interfaces from dataclass definitions. +**dcargs** is a library for building dataclass-based argument parsers and +configuration objects. -We expose one function, `parse()`, which takes a dataclass type and instantiates -it via an argparse-style CLI interface. If we create a script called -`simple.py`: +The vision: we use (potentially nested or generic) dataclasses to define +configuration objects that can be (a) populated via a CLI interface without +additional effort and (b) robustly and human-readably serialized. The result is +a statically typed replacement for not only `argparse`, but libraries likes +[YACS](https://github.com/rbgirshick/yacs) and +[ml_collections](https://github.com/google/ml_collections). + +We expose a one-function argument parsing API: + +- dcargs.parse(cls: Type[T], \*, description: + Optional[str]) -> T takes a dataclass type and instantiates it via an + argparse-style CLI interface. + +And two functions for dataclass serialization: + +- dcargs.from_yaml(cls: Type[T], stream: Union[str, + IO[str], bytes, IO[bytes]]) -> T and + dcargs.to_yaml(instance: T) -> str convert + between YAML-style strings and dataclass instances. In contrast to naively + dumping or loading (via pickle, PyYAML, etc), explicit type references enable + robustness against code reorganization and refactor. + +### Simple example ```python import dataclasses @@ -34,6 +55,8 @@ class Args: if __name__ == "__main__": args = dcargs.parse(Args) print(args) + print() + print(dcargs.to_yaml(args)) ``` Running `python simple.py --help` would print: @@ -53,13 +76,17 @@ And, from `python simple.py --field1 string --field2 4`: ``` Args(field1='string', field2=4) + +!Args +field1: string +field2: 4 ``` ### Feature list The parse function supports a wide range of dataclass definitions, while automatically generating helptext from comments/docstrings. Some of the basic -features are shown in the [example below](#example-usage). +features are shown in the [nesting example below](#nested-example). Our unit tests cover many more complex type annotations, including classes containing: @@ -93,13 +120,10 @@ containing: - Generic dataclasses (including nested generics, see [./examples/generics.py](./examples/generics.py)) -A usage example is available below. Examples of additional features can be found -in the [tests](./tests/). - ### Comparisons to alternative tools -There are several alternative libraries to `dcargs`; here's a rough summary of -some of them: +There are several alternative libraries to the parsing functionality of +`dcargs`; here's a rough summary of some of them: | | dataclasses | attrs | Nesting | Subparsers | Containers | Choices from literals | Docstrings as helptext | Generics | | ----------------------------------------------------------------------------------------------- | ----------- | ----- | ------- | ---------- | ---------- | -------------------------------------------------------- | ---------------------- | -------- | @@ -126,7 +150,7 @@ Some other distinguishing factors that `dcargs` has put effort into: on dataclass field metadata, or on the more extreme end inheritance+decorators to make parsing-specific dataclasses) -### Example usage +### Nested example This code: @@ -203,3 +227,5 @@ required arguments: --optimizer.type {ADAM,SGD} Variant of SGD to use. ``` + +Examples of additional features can be found in our [unit tests](./tests/). diff --git a/dcargs/__init__.py b/dcargs/__init__.py index a784b4a67..1883e95df 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,3 +1,8 @@ from ._parse import parse +from ._serialization import from_yaml, to_yaml -__all__ = ["parse"] +__all__ = [ + "parse", + "from_yaml", + "to_yaml", +] diff --git a/dcargs/_serialization.py b/dcargs/_serialization.py new file mode 100644 index 000000000..25098e5ed --- /dev/null +++ b/dcargs/_serialization.py @@ -0,0 +1,142 @@ +import dataclasses +from typing import IO, Any, Optional, Set, Type, TypeVar, Union + +import yaml +from typing_extensions import get_origin + +from . import _resolver + +DATACLASS_YAML_TAG_PREFIX = "!" + +DataclassType = TypeVar("DataclassType") + + +def _get_contained_dataclass_types_from_instance(instance: Any) -> Set[Type]: + """Takes a dataclass instance, and recursively searches its cihldren for dataclass + types.""" + if not dataclasses.is_dataclass(instance): + return set() + out = {type(instance)} + for v in vars(instance).values(): + out |= _get_contained_dataclass_types_from_instance(v) + return out + + +def _get_contained_dataclass_types_from_type( + cls: Type, + _parent_contained_dataclasses: Optional[Set[Type]] = None, +) -> Set[Type]: + """Takes a dataclass type, and recursively searches its fields for dataclass + types.""" + assert _resolver.is_dataclass(cls) + parent_contained_dataclasses = ( + set() + if _parent_contained_dataclasses is None + else _parent_contained_dataclasses + ) + + cls, type_from_typevar = _resolver.resolve_generic_dataclasses(cls) + + contained_dataclasses = {cls} + + def handle_type(typ) -> Set[Type]: + if _resolver.is_dataclass(typ) and typ not in parent_contained_dataclasses: + return _get_contained_dataclass_types_from_type( + typ, + _parent_contained_dataclasses=contained_dataclasses + | parent_contained_dataclasses, + ) + return set() + + # Handle generics. + for typ in type_from_typevar.values(): + contained_dataclasses |= handle_type(typ) + + if cls in parent_contained_dataclasses: + return contained_dataclasses + + # Handle fields. + for field in _resolver.resolved_fields(cls): # type: ignore + contained_dataclasses |= handle_type(field.type) + + return contained_dataclasses + + +def _make_loader(cls: Type) -> Type[yaml.Loader]: + class DataclassLoader(yaml.Loader): + pass + + # Design Q: do we want to support multiple dataclass types with the same name? + # - Why yes: rudimentary support for this is easy. + # - Why no: might cause confusing error messages? Using all possible context cues + # for this is also hard. + # + # => let's just keep things simple, assert uniqueness for now. Easier to add new + # features later than remove them. + + contained_types = list(_get_contained_dataclass_types_from_type(cls)) + contained_type_names = list(map(lambda cls: cls.__name__, contained_types)) + assert len(set(contained_type_names)) == len( + contained_type_names + ), f"Contained dataclass type names must all be unique, but got {contained_type_names}" + + loader: yaml.Loader + node: yaml.Node + + def make_constructor(typ: Type): + return lambda loader, node: typ(**loader.construct_mapping(node)) + + for typ, name in zip(contained_types, contained_type_names): + DataclassLoader.add_constructor( + tag=DATACLASS_YAML_TAG_PREFIX + name, + constructor=make_constructor(typ), + ) + + return DataclassLoader + + +def _make_dumper(instance: Any) -> Type[yaml.Dumper]: + class DataclassDumper(yaml.Dumper): + pass + + contained_types = list(_get_contained_dataclass_types_from_instance(instance)) + contained_type_names = list(map(lambda cls: cls.__name__, contained_types)) + assert len(set(contained_type_names)) == len( + contained_type_names + ), f"Contained dataclass type names must all be unique, but got {contained_type_names}" + + dumper: yaml.Dumper + data: Any + field: dataclasses.Field + + def make_representer(name: str): + return lambda dumper, data: dumper.represent_mapping( + tag=DATACLASS_YAML_TAG_PREFIX + name, + mapping={ + field.name: getattr(data, field.name) + for field in dataclasses.fields(data) + if field.init + }, + ) + + for typ, name in zip(contained_types, contained_type_names): + DataclassDumper.add_representer(typ, make_representer(name)) + return DataclassDumper + + +def from_yaml( + cls: Type[DataclassType], + stream: Union[str, IO[str], bytes, IO[bytes]], +) -> DataclassType: + """Re-construct a dataclass instance from a yaml-compatible string, which should be + generated from `dcargs.to_yaml()`.""" + out = yaml.load(stream, Loader=_make_loader(cls)) + origin_cls = get_origin(cls) + assert isinstance(out, origin_cls if origin_cls is not None else cls) + return out + + +def to_yaml(instance: Any) -> str: + """Serialize a dataclass; returns a yaml-compatible string that can be deserialized + via `dcargs.from_yaml()`.""" + return yaml.dump(instance, Dumper=_make_dumper(instance)) diff --git a/examples/generics.py b/examples/generics.py index 9fb1f675d..ecb22ce08 100644 --- a/examples/generics.py +++ b/examples/generics.py @@ -31,6 +31,12 @@ class Args: triangle_optional_coords: Triangle[Optional[float]] + triangle_with_default: Triangle[int] = Triangle( + a=Point3(1, 2, 3, "world"), + b=Point3(1, 2, 3, "world"), + c=Point3(1, 2, 3, "world"), + ) + if __name__ == "__main__": args = dcargs.parse(Args) diff --git a/examples/generics_alt.py b/examples/generics_alt.py new file mode 100644 index 000000000..bfe0a6c57 --- /dev/null +++ b/examples/generics_alt.py @@ -0,0 +1,34 @@ +import dataclasses +from typing import Generic, TypeVar + +import dcargs + +ScalarType = TypeVar("ScalarType") +ShapeType = TypeVar("ShapeType") + + +@dataclasses.dataclass +class Point3(Generic[ScalarType]): + x: ScalarType + y: ScalarType + z: ScalarType + frame_id: str + + +@dataclasses.dataclass +class Triangle: + a: Point3[float] + b: Point3[float] + c: Point3[float] + + +@dataclasses.dataclass +class Args(Generic[ShapeType]): + point_continuous: Point3[float] + point_discrete: Point3[int] + shape: ShapeType + + +if __name__ == "__main__": + args = dcargs.parse(Args[Triangle]) + print(args) diff --git a/examples/simple.py b/examples/simple.py index ae067b638..eebcca04f 100644 --- a/examples/simple.py +++ b/examples/simple.py @@ -12,3 +12,5 @@ class Args: if __name__ == "__main__": args = dcargs.parse(Args) print(args) + print() + print(dcargs.to_yaml(args)) diff --git a/setup.py b/setup.py index d4edb4320..826b6166d 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.8", + version="0.0.9", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", @@ -16,7 +16,7 @@ packages=find_packages(), package_data={"dcargs": ["py.typed"]}, python_requires=">=3.7", - install_requires=["typing_extensions>=4.0.0"], + install_requires=["typing_extensions>=4.0.0", "pyyaml"], extras_require={ "testing": [ "pytest", diff --git a/tests/test_generics.py b/tests/test_generics_and_serialization.py similarity index 62% rename from tests/test_generics.py rename to tests/test_generics_and_serialization.py index 6b792a7ed..4611b09b4 100644 --- a/tests/test_generics.py +++ b/tests/test_generics_and_serialization.py @@ -1,10 +1,17 @@ import dataclasses -from typing import Generic, TypeVar, Union +from typing import Generic, Type, TypeVar, Union import pytest import dcargs +T = TypeVar("T") + + +def _check_serialization_identity(cls: Type[T], instance: T) -> None: + assert dcargs.from_yaml(cls, dcargs.to_yaml(instance)) == instance + + ScalarType = TypeVar("ScalarType") @@ -22,30 +29,31 @@ class SimpleGeneric: point_continuous: Point3[float] point_discrete: Point3[int] - assert ( - dcargs.parse( - SimpleGeneric, - args=[ - "--point-continuous.x", - "1.2", - "--point-continuous.y", - "2.2", - "--point-continuous.z", - "3.2", - "--point-continuous.frame-id", - "world", - "--point-discrete.x", - "1", - "--point-discrete.y", - "2", - "--point-discrete.z", - "3", - "--point-discrete.frame-id", - "world", - ], - ) - == SimpleGeneric(Point3(1.2, 2.2, 3.2, "world"), Point3(1, 2, 3, "world")) + parsed_instance = dcargs.parse( + SimpleGeneric, + args=[ + "--point-continuous.x", + "1.2", + "--point-continuous.y", + "2.2", + "--point-continuous.z", + "3.2", + "--point-continuous.frame-id", + "world", + "--point-discrete.x", + "1", + "--point-discrete.y", + "2", + "--point-discrete.z", + "3", + "--point-discrete.frame-id", + "world", + ], + ) + assert parsed_instance == SimpleGeneric( + Point3(1.2, 2.2, 3.2, "world"), Point3(1, 2, 3, "world") ) + _check_serialization_identity(SimpleGeneric, parsed_instance) with pytest.raises(SystemExit): # Accidentally pass in floats instead of ints for discrete @@ -79,7 +87,7 @@ class Triangle(Generic[ScalarType]): b: Point3[ScalarType] c: Point3[ScalarType] - dcargs.parse( + parsed_instance = dcargs.parse( Triangle[float], args=[ "--a.x", @@ -107,11 +115,13 @@ class Triangle(Generic[ScalarType]): "--c.frame-id", "world", ], - ) == Triangle( + ) + assert parsed_instance == Triangle( Point3(1.0, 1.2, 1.3, "world"), Point3(1.0, 1.2, 1.3, "world"), Point3(1.0, 1.2, 1.3, "world"), ) + _check_serialization_identity(Triangle[float], parsed_instance) def test_generic_nested_dataclass(): @@ -126,9 +136,11 @@ class Child: class DataclassGeneric(Generic[T]): child: T - assert dcargs.parse( + parsed_instance = dcargs.parse( DataclassGeneric[Child], args=["--child.a", "5", "--child.b", "7"] - ) == DataclassGeneric(Child(5, 7)) + ) + assert parsed_instance == DataclassGeneric(Child(5, 7)) + _check_serialization_identity(DataclassGeneric[Child], parsed_instance) def test_generic_subparsers(): @@ -147,10 +159,14 @@ class CommandTwo: class Subparser(Generic[T1, T2]): command: Union[T1, T2] - assert dcargs.parse( + parsed_instance = dcargs.parse( Subparser[CommandOne, CommandTwo], args="command-one --a 5".split(" ") - ) == Subparser(CommandOne(5)) + ) + assert parsed_instance == Subparser(CommandOne(5)) + _check_serialization_identity(Subparser[CommandOne, CommandTwo], parsed_instance) - assert dcargs.parse( + parsed_instance = dcargs.parse( Subparser[CommandOne, CommandTwo], args="command-two --b 7".split(" ") - ) == Subparser(CommandTwo(7)) + ) + assert parsed_instance == Subparser(CommandTwo(7)) + _check_serialization_identity(Subparser[CommandOne, CommandTwo], parsed_instance) From 7ff0e007a374ed8f112071035b29ef8d1f7fcbbf Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 23 Nov 2021 19:55:33 -0800 Subject: [PATCH 015/491] More robust enum serialization --- README.md | 2 +- dcargs/_serialization.py | 82 +++++++++++++++++------- examples/example.py | 1 + tests/test_generics_and_serialization.py | 45 +++++++------ 4 files changed, 86 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index a9ed2c0e7..59ac8c776 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ And, from `python simple.py --field1 string --field2 4`: ``` Args(field1='string', field2=4) -!Args +!dataclass:Args field1: string field2: 4 ``` diff --git a/dcargs/_serialization.py b/dcargs/_serialization.py index 25098e5ed..44d6824e9 100644 --- a/dcargs/_serialization.py +++ b/dcargs/_serialization.py @@ -1,4 +1,6 @@ import dataclasses +import datetime +import enum from typing import IO, Any, Optional, Set, Type, TypeVar, Union import yaml @@ -6,27 +8,31 @@ from . import _resolver -DATACLASS_YAML_TAG_PREFIX = "!" +ENUM_YAML_TAG_PREFIX = "!enum:" +DATACLASS_YAML_TAG_PREFIX = "!dataclass:" DataclassType = TypeVar("DataclassType") -def _get_contained_dataclass_types_from_instance(instance: Any) -> Set[Type]: - """Takes a dataclass instance, and recursively searches its cihldren for dataclass +def _get_contained_special_types_from_instance(instance: Any) -> Set[Type]: + """Takes an object and recursively searches its cihldren for dataclass or enum types.""" - if not dataclasses.is_dataclass(instance): + if issubclass(type(instance), enum.Enum): + return {type(instance)} + elif not dataclasses.is_dataclass(instance): return set() + out = {type(instance)} for v in vars(instance).values(): - out |= _get_contained_dataclass_types_from_instance(v) + out |= _get_contained_special_types_from_instance(v) return out -def _get_contained_dataclass_types_from_type( +def _get_contained_special_types_from_type( cls: Type, _parent_contained_dataclasses: Optional[Set[Type]] = None, ) -> Set[Type]: - """Takes a dataclass type, and recursively searches its fields for dataclass + """Takes a dataclass type, and recursively searches its fields for dataclass or enum types.""" assert _resolver.is_dataclass(cls) parent_contained_dataclasses = ( @@ -41,11 +47,13 @@ def _get_contained_dataclass_types_from_type( def handle_type(typ) -> Set[Type]: if _resolver.is_dataclass(typ) and typ not in parent_contained_dataclasses: - return _get_contained_dataclass_types_from_type( + return _get_contained_special_types_from_type( typ, _parent_contained_dataclasses=contained_dataclasses | parent_contained_dataclasses, ) + elif type(typ) is enum.EnumMeta: + return {typ} return set() # Handle generics. @@ -74,7 +82,7 @@ class DataclassLoader(yaml.Loader): # => let's just keep things simple, assert uniqueness for now. Easier to add new # features later than remove them. - contained_types = list(_get_contained_dataclass_types_from_type(cls)) + contained_types = list(_get_contained_special_types_from_type(cls)) contained_type_names = list(map(lambda cls: cls.__name__, contained_types)) assert len(set(contained_type_names)) == len( contained_type_names @@ -83,14 +91,25 @@ class DataclassLoader(yaml.Loader): loader: yaml.Loader node: yaml.Node - def make_constructor(typ: Type): + def make_dataclass_constructor(typ: Type): return lambda loader, node: typ(**loader.construct_mapping(node)) + def make_enum_constructor(typ: Type): + return lambda loader, node: typ[loader.construct_python_str(node)] + for typ, name in zip(contained_types, contained_type_names): - DataclassLoader.add_constructor( - tag=DATACLASS_YAML_TAG_PREFIX + name, - constructor=make_constructor(typ), - ) + if dataclasses.is_dataclass(typ): + DataclassLoader.add_constructor( + tag=DATACLASS_YAML_TAG_PREFIX + name, + constructor=make_dataclass_constructor(typ), + ) + elif issubclass(typ, enum.Enum): + DataclassLoader.add_constructor( + tag=ENUM_YAML_TAG_PREFIX + name, + constructor=make_enum_constructor(typ), + ) + else: + assert False return DataclassLoader @@ -99,7 +118,7 @@ def _make_dumper(instance: Any) -> Type[yaml.Dumper]: class DataclassDumper(yaml.Dumper): pass - contained_types = list(_get_contained_dataclass_types_from_instance(instance)) + contained_types = list(_get_contained_special_types_from_instance(instance)) contained_type_names = list(map(lambda cls: cls.__name__, contained_types)) assert len(set(contained_type_names)) == len( contained_type_names @@ -110,14 +129,22 @@ class DataclassDumper(yaml.Dumper): field: dataclasses.Field def make_representer(name: str): - return lambda dumper, data: dumper.represent_mapping( - tag=DATACLASS_YAML_TAG_PREFIX + name, - mapping={ - field.name: getattr(data, field.name) - for field in dataclasses.fields(data) - if field.init - }, - ) + def representer(dumper, data): + if dataclasses.is_dataclass(data): + return dumper.represent_mapping( + tag=DATACLASS_YAML_TAG_PREFIX + name, + mapping={ + field.name: getattr(data, field.name) + for field in dataclasses.fields(data) + if field.init + }, + ) + elif isinstance(data, enum.Enum): + return dumper.represent_scalar( + tag=ENUM_YAML_TAG_PREFIX + name, value=data.name + ) + + return representer for typ, name in zip(contained_types, contained_type_names): DataclassDumper.add_representer(typ, make_representer(name)) @@ -136,7 +163,14 @@ def from_yaml( return out +def _timestamp() -> str: + """Get a current timestamp as a string. Example format: `2021-11-05-15:46:32`.""" + return datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + + def to_yaml(instance: Any) -> str: """Serialize a dataclass; returns a yaml-compatible string that can be deserialized via `dcargs.from_yaml()`.""" - return yaml.dump(instance, Dumper=_make_dumper(instance)) + return f"# YAML generated via dcargs, at {_timestamp()}.\n" + yaml.dump( + instance, Dumper=_make_dumper(instance) + ) diff --git a/examples/example.py b/examples/example.py index c9c909bfe..11b355a48 100644 --- a/examples/example.py +++ b/examples/example.py @@ -41,3 +41,4 @@ class ExperimentConfig: if __name__ == "__main__": config = dcargs.parse(ExperimentConfig, description=__doc__) print(config) + print(dcargs.to_yaml(config)) diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 4611b09b4..7bc4760b9 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -1,4 +1,5 @@ import dataclasses +import enum from typing import Generic, Type, TypeVar, Union import pytest @@ -15,12 +16,17 @@ def _check_serialization_identity(cls: Type[T], instance: T) -> None: ScalarType = TypeVar("ScalarType") +class CoordinateFrame(enum.Enum): + WORLD = enum.auto() + CAMERA = enum.auto() + + @dataclasses.dataclass class Point3(Generic[ScalarType]): x: ScalarType y: ScalarType z: ScalarType - frame_id: str + frame: CoordinateFrame def test_simple_generic(): @@ -38,20 +44,21 @@ class SimpleGeneric: "2.2", "--point-continuous.z", "3.2", - "--point-continuous.frame-id", - "world", + "--point-continuous.frame", + "WORLD", "--point-discrete.x", "1", "--point-discrete.y", "2", "--point-discrete.z", "3", - "--point-discrete.frame-id", - "world", + "--point-discrete.frame", + "WORLD", ], ) assert parsed_instance == SimpleGeneric( - Point3(1.2, 2.2, 3.2, "world"), Point3(1, 2, 3, "world") + Point3(1.2, 2.2, 3.2, CoordinateFrame.WORLD), + Point3(1, 2, 3, CoordinateFrame.WORLD), ) _check_serialization_identity(SimpleGeneric, parsed_instance) @@ -66,16 +73,16 @@ class SimpleGeneric: "2.2", "--point-continuous.z", "3.2", - "--point-continuous.frame-id", - "world", + "--point-continuous.frame", + "WORLD", "--point-discrete.x", "1.5", "--point-discrete.y", "2.5", "--point-discrete.z", "3.5", - "--point-discrete.frame-id", - "world", + "--point-discrete.frame", + "WORLD", ], ) @@ -96,30 +103,30 @@ class Triangle(Generic[ScalarType]): "1.2", "--a.z", "1.3", - "--a.frame-id", - "world", + "--a.frame", + "WORLD", "--b.x", "1.0", "--b.y", "1.2", "--b.z", "1.3", - "--b.frame-id", - "world", + "--b.frame", + "WORLD", "--c.x", "1.0", "--c.y", "1.2", "--c.z", "1.3", - "--c.frame-id", - "world", + "--c.frame", + "WORLD", ], ) assert parsed_instance == Triangle( - Point3(1.0, 1.2, 1.3, "world"), - Point3(1.0, 1.2, 1.3, "world"), - Point3(1.0, 1.2, 1.3, "world"), + Point3(1.0, 1.2, 1.3, CoordinateFrame.WORLD), + Point3(1.0, 1.2, 1.3, CoordinateFrame.WORLD), + Point3(1.0, 1.2, 1.3, CoordinateFrame.WORLD), ) _check_serialization_identity(Triangle[float], parsed_instance) From 311a66baaa0e4ccde16c4466096097eaf5d86dd6 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 26 Nov 2021 23:57:22 -0800 Subject: [PATCH 016/491] Docstring fix for inherited fields + test --- dcargs/_docstrings.py | 74 +++++++++++++++++++++++----------------- dcargs/_serialization.py | 4 ++- tests/test_docstrings.py | 55 +++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 32 deletions(-) diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 16e302178..7757991a3 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -25,14 +25,14 @@ class _FieldData: @dataclasses.dataclass -class _Tokenization: +class _ClassTokenization: tokens: List[_Token] tokens_from_line: Dict[int, List[_Token]] field_data_from_name: Dict[str, _FieldData] @staticmethod - @functools.lru_cache(maxsize=4) - def make(cls) -> "_Tokenization": + @functools.lru_cache(maxsize=8) + def make(cls) -> "_ClassTokenization": """Parse the source code of a class, and cache some tokenization information.""" readline = io.BytesIO(inspect.getsource(cls).encode("utf-8")).readline @@ -66,34 +66,57 @@ def make(cls) -> "_Tokenization": ) prev_field_line_number = token.line_number - return _Tokenization( + return _ClassTokenization( tokens=tokens, tokens_from_line=tokens_from_line, field_data_from_name=field_data_from_name, ) +def get_class_tokenization_with_field( + cls: Type, field_name: str +) -> Optional[_ClassTokenization]: + # Search for token in this class + all parents. + found_field: bool = False + classes_to_search = cls.mro() + for search_cls in classes_to_search: + # Unwrap generics. + origin_cls = get_origin(search_cls) + if origin_cls is not None: + search_cls = origin_cls + + # Skip parent classes that aren't dataclasses. + if not dataclasses.is_dataclass(search_cls): + continue + + try: + tokenization = _ClassTokenization.make(search_cls) # type: ignore + except OSError as e: + # Dynamic dataclasses will result in an OSError -- this is fine, we just assume + # there's no docstring. + assert "could not find class definition" in e.args[0] + return None + + # Grab field-specific tokenization data. + if field_name in tokenization.field_data_from_name: + found_field = True + break + + assert ( + found_field + ), "Docstring parsing error -- this usually means that there are multiple \ + dataclasses in the same file with the same name but different scopes." + + return tokenization + + def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: """Get docstring for a field in a class.""" - origin_cls = get_origin(cls) - if origin_cls is not None: - cls = origin_cls - - assert dataclasses.is_dataclass(cls) - try: - tokenization = _Tokenization.make(cls) # type: ignore - except OSError as e: - # Dynamic dataclasses will result in an OSError -- this is fine, we just assume - # there's no docstring. - assert "could not find class definition" in e.args[0] + tokenization = get_class_tokenization_with_field(cls, field_name) + if tokenization is None: # Currently only happens for dynamic dataclasses. return None - # Grab field-specific tokenization data. - assert ( - field_name in tokenization.field_data_from_name - ), "Docstring parsing error -- this usually means that there are multiple \ - dataclasses in the same file with the same name but different scopes." field_data = tokenization.field_data_from_name[field_name] # Check for docstring-style comment. @@ -126,17 +149,6 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: break line_number += 1 - # if ( - # field_data.line_number + 1 in tokenization.tokens_from_line - # and len(tokenization.tokens_from_line[field_data.line_number + 1]) > 0 - # ): - # first_token_on_next_line = tokenization.tokens_from_line[ - # field_data.line_number + 1 - # ][0] - # if first_token_on_next_line.token_type == tokenize.STRING: - # docstring = first_token_on_next_line.token.strip() - # assert docstring.endswith('"""') and docstring.startswith('"""') - # return _strings.dedent(docstring[3:-3]) # Check for comment on the same line as the field. final_token_on_line = tokenization.tokens_from_line[field_data.line_number][-1] diff --git a/dcargs/_serialization.py b/dcargs/_serialization.py index 44d6824e9..f91cd771c 100644 --- a/dcargs/_serialization.py +++ b/dcargs/_serialization.py @@ -120,9 +120,11 @@ class DataclassDumper(yaml.Dumper): contained_types = list(_get_contained_special_types_from_instance(instance)) contained_type_names = list(map(lambda cls: cls.__name__, contained_types)) + + # Note: this is currently a stricter than necessary assert. assert len(set(contained_type_names)) == len( contained_type_names - ), f"Contained dataclass type names must all be unique, but got {contained_type_names}" + ), f"Contained dataclass/enum names must all be unique, but got {contained_type_names}" dumper: yaml.Dumper data: Any diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index b61a29c29..3741ad210 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -94,6 +94,61 @@ class HelptextHardString: ) +def test_helptext_with_inheritance(): + @dataclasses.dataclass + class Parent: + # fmt: off + x: str = ( + "This docstring may be tougher to parse!" + ) + """Helptext.""" + # fmt: on + + @dataclasses.dataclass + class Child(Parent): + pass + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(Child, args=["--help"]) + helptext = f.getvalue() + assert ( + "--x STR Helptext. (default: This docstring may be tougher to parse!)\n" + in helptext + ) + + +def test_helptext_with_inheritance_overriden(): + @dataclasses.dataclass + class Parent2: + # fmt: off + x: str = ( + "This docstring may be tougher to parse!" + ) + """Helptext.""" + # fmt: on + + @dataclasses.dataclass + class Child2(Parent2): + # fmt: off + x: str = ( + "This docstring may be tougher to parse?" + ) + """Helptext.""" + # fmt: on + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(Child2, args=["--help"]) + helptext = f.getvalue() + assert ( + "--x STR Helptext. (default: This docstring may be tougher to parse?)\n" + in helptext + ) + + def test_tuple_helptext(): @dataclasses.dataclass class TupleHelptext: From d5da406ef7f8684115ef69acd85f973bef7a11aa Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 2 Jan 2022 02:58:36 -0800 Subject: [PATCH 017/491] Fix `%` in docstrings, version bump --- dcargs/_arguments.py | 3 +++ setup.py | 2 +- tests/test_docstrings.py | 7 +++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index db5900be6..8ae91b73b 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -357,6 +357,9 @@ def _generate_helptext(arg: ArgumentDefinition) -> _ArgumentTransformOutput: arg.parent_class, arg.field.name ) if docstring_help is not None: + # Note that the percent symbol needs some extra handling in argparse. + # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string + docstring_help = docstring_help.replace("%", "%%") help_parts.append(docstring_help) if arg.default is not None and hasattr(arg.default, "name"): diff --git a/setup.py b/setup.py index 826b6166d..cffd2ca8b 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.9", + version="0.0.10", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 3741ad210..9fac2831a 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -80,16 +80,19 @@ class HelptextHardString: x: str = ( "This docstring may be tougher to parse!" ) - """Helptext.""" + """Helptext. 2% milk.""" # fmt: on + # Note that the percent symbol needs some extra handling in argparse. + # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string + f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.parse(HelptextHardString, args=["--help"]) helptext = f.getvalue() assert ( - "--x STR Helptext. (default: This docstring may be tougher to parse!)\n" + "--x STR Helptext. 2% milk. (default: This docstring may be tougher to parse!)\n" in helptext ) From e2dd680f91695481fd86dcbd8fd8cb9be51e8a7e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 7 Jan 2022 17:32:02 -0800 Subject: [PATCH 018/491] Fix `%` for docstrings in bool flags --- dcargs/_arguments.py | 19 ++++++++++++++----- setup.py | 2 +- tests/test_docstrings.py | 11 ++++------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 8ae91b73b..7ee834233 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -195,11 +195,20 @@ def _bool_flags(arg: ArgumentDefinition) -> _ArgumentTransformOutput: # Populate helptext for boolean flags => don't show default value, which can be # confusing. - helptext = _docstrings.get_field_docstring(arg.parent_class, arg.field.name) - arg = dataclasses.replace( - arg, - help=helptext if helptext is not None else "", - ) + docstring_help = _docstrings.get_field_docstring(arg.parent_class, arg.field.name) + if docstring_help is not None: + # Note that the percent symbol needs some extra handling in argparse. + # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string + docstring_help = docstring_help.replace("%", "%%") + arg = dataclasses.replace( + arg, + help=docstring_help, + ) + else: + arg = dataclasses.replace( + arg, + help="", + ) if arg.default is None: return ( diff --git a/setup.py b/setup.py index cffd2ca8b..fcf9e4a05 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.10", + version="0.0.11", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 9fac2831a..0993dbb6c 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -73,12 +73,12 @@ class Config: assert " --x INT An optional variable. (default: None)\n" in helptext -def test_helptext_hard_string(): +def test_helptext_hard_bool(): @dataclasses.dataclass class HelptextHardString: # fmt: off - x: str = ( - "This docstring may be tougher to parse!" + x: bool = ( + False ) """Helptext. 2% milk.""" # fmt: on @@ -91,10 +91,7 @@ class HelptextHardString: with contextlib.redirect_stdout(f): dcargs.parse(HelptextHardString, args=["--help"]) helptext = f.getvalue() - assert ( - "--x STR Helptext. 2% milk. (default: This docstring may be tougher to parse!)\n" - in helptext - ) + assert "--x Helptext. 2% milk.\n" in helptext def test_helptext_with_inheritance(): From 191d57bd9461b7985a203729580dfff266417b75 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 13 Jan 2022 19:12:17 -0800 Subject: [PATCH 019/491] Support booleans in fixed-length tuples --- dcargs/_arguments.py | 44 ++++++++++- setup.py | 2 +- tests/test_collections.py | 153 ++++++++++++++++++++++++++++++++++++++ tests/test_dcargs.py | 107 +------------------------- tests/test_docstrings.py | 4 +- 5 files changed, 198 insertions(+), 112 deletions(-) create mode 100644 tests/test_collections.py diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 7ee834233..f1cedc074 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -8,6 +8,31 @@ from . import _construction, _docstrings, _strings +T = TypeVar("T") + + +def _instance_from_string(typ: Type[T], arg: str) -> T: + """Given a type and and a string from the command-line, reconstruct an object. Not + intended to deal with generic types or containers; these are handled in the + "argument transformations" below. + + This is intended to replace all calls to `type(string)`, which can cause unexpected + behavior. As an example, note that the following argparse code will always print + `True`, because `bool("True") == bool("False") == bool("0") == True`. + ``` + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--flag", type=bool) + + print(parser.parse_args().flag) + ``` + """ + if typ is bool: + return _strings.bool_from_string(arg) # type: ignore + else: + return typ(arg) # type: ignore + @dataclasses.dataclass(frozen=True) class ArgumentDefinition: @@ -38,6 +63,16 @@ def add_argument( name = "--" + kwargs.pop("name").replace("_", "-") kwargs.pop("field") kwargs.pop("parent_class") + + # Wrap the raw type with handling for special types. (currently only booleans) + # This feels a like a bit of band-aid; in the future, we may want to always set + # the argparse type to str and fold this logic into a more general version of + # what we currently call the "field role" (callables used for reconstructing + # lists, tuples, sets, etc). + if "type" in kwargs: + raw_type = kwargs["type"] + kwargs["type"] = lambda arg: _instance_from_string(raw_type, arg) + parser.add_argument(name, **kwargs) def prefix(self, prefix: str) -> "ArgumentDefinition": @@ -214,7 +249,6 @@ def _bool_flags(arg: ArgumentDefinition) -> _ArgumentTransformOutput: return ( dataclasses.replace( arg, - type=_strings.bool_from_string, # type: ignore metavar="{True,False}", ), None, @@ -301,17 +335,21 @@ def _nargs_from_tuples(arg: ArgumentDefinition) -> _ArgumentTransformOutput: else: # Tuples with more than one type assert arg.metavar is None + return ( dataclasses.replace( arg, nargs=len(types), - type=str, # Types will be converted in the dataclass reconstruction step + type=str, # Types will be converted in the dataclass reconstruction step. metavar=tuple( t.__name__.upper() if hasattr(t, "__name__") else "X" for t in types ), ), - lambda str_list: tuple(typ(x) for typ, x in zip(types, str_list)), + # Field role: convert lists of strings to tuples of the correct types. + lambda str_list: tuple( + _instance_from_string(typ, x) for typ, x in zip(types, str_list) + ), ) else: diff --git a/setup.py b/setup.py index fcf9e4a05..e1e9ef99c 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.11", + version="0.0.12", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_collections.py b/tests/test_collections.py new file mode 100644 index 000000000..e6a3e192d --- /dev/null +++ b/tests/test_collections.py @@ -0,0 +1,153 @@ +import dataclasses +from typing import List, Optional, Sequence, Set, Tuple + +import pytest + +import dcargs + + +def test_tuples_fixed(): + @dataclasses.dataclass + class A: + x: Tuple[int, int, int] + + assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + +def test_tuples_fixed_multitype(): + @dataclasses.dataclass + class A: + x: Tuple[int, str, float] + + assert dcargs.parse(A, args=["--x", "1", "2", "3.5"]) == A(x=(1, "2", 3.5)) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + +def test_tuples_fixed_bool(): + @dataclasses.dataclass + class A: + x: Tuple[bool, bool, bool] + + assert dcargs.parse(A, args=["--x", "True", "True", "False"]) == A( + x=(True, True, False) + ) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + +def test_tuples_variable(): + @dataclasses.dataclass + class A: + x: Tuple[int, ...] + + assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + +def test_tuples_variable_bool(): + @dataclasses.dataclass + class A: + x: Tuple[bool, ...] + + assert dcargs.parse(A, args=["--x", "True", "True", "False"]) == A( + x=(True, True, False) + ) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + +def test_tuples_variable_optional(): + @dataclasses.dataclass + class A: + x: Optional[Tuple[int, ...]] + + assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + assert dcargs.parse(A, args=[]) == A(x=None) + + +def test_sequences(): + @dataclasses.dataclass + class A: + x: Sequence[int] + + assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + +def test_lists(): + @dataclasses.dataclass + class A: + x: List[int] + + assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + +def test_lists_bool(): + @dataclasses.dataclass + class A: + x: List[bool] + + assert dcargs.parse(A, args=["--x", "True", "False", "True"]) == A( + x=[True, False, True] + ) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + +def test_sets(): + @dataclasses.dataclass + class A: + x: Set[int] + + assert dcargs.parse(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + +def test_optional_sequences(): + @dataclasses.dataclass + class A: + x: Optional[Sequence[int]] + + assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + assert dcargs.parse(A, args=[]) == A(x=None) + + +def test_optional_lists(): + @dataclasses.dataclass + class A: + x: Optional[List[int]] + + assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + assert dcargs.parse(A, args=[]) == A(x=None) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index fc7dcd8eb..fca079216 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -1,7 +1,7 @@ import dataclasses import enum import pathlib -from typing import ClassVar, List, Optional, Sequence, Set, Tuple, Union +from typing import ClassVar, Optional, Union import pytest from typing_extensions import Annotated, Final, Literal # Backward compatibility. @@ -116,111 +116,6 @@ class A: assert dcargs.parse(A, args=[]) == A(x=None) -def test_sequences(): - @dataclasses.dataclass - class A: - x: Sequence[int] - - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) - with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) - with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) - - -def test_lists(): - @dataclasses.dataclass - class A: - x: List[int] - - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) - with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) - with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) - - -def test_sets(): - @dataclasses.dataclass - class A: - x: Set[int] - - assert dcargs.parse(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) - with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) - with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) - - -def test_optional_sequences(): - @dataclasses.dataclass - class A: - x: Optional[Sequence[int]] - - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) - with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) - assert dcargs.parse(A, args=[]) == A(x=None) - - -def test_optional_lists(): - @dataclasses.dataclass - class A: - x: Optional[List[int]] - - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) - with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) - assert dcargs.parse(A, args=[]) == A(x=None) - - -def test_tuples_fixed(): - @dataclasses.dataclass - class A: - x: Tuple[int, int, int] - - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) - with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) - with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) - - -def test_tuples_fixed_multitype(): - @dataclasses.dataclass - class A: - x: Tuple[int, str, float] - - assert dcargs.parse(A, args=["--x", "1", "2", "3.5"]) == A(x=(1, "2", 3.5)) - with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) - with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) - - -def test_tuples_variable(): - @dataclasses.dataclass - class A: - x: Tuple[int, ...] - - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) - with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) - with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) - - -def test_tuples_variable_optional(): - @dataclasses.dataclass - class A: - x: Optional[Tuple[int, ...]] - - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) - with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) - assert dcargs.parse(A, args=[]) == A(x=None) - - def test_enum(): class Color(enum.Enum): RED = enum.auto() diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 0993dbb6c..625a3cfac 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -135,7 +135,7 @@ class Child2(Parent2): x: str = ( "This docstring may be tougher to parse?" ) - """Helptext.""" + """Helptext!""" # fmt: on f = io.StringIO() @@ -144,7 +144,7 @@ class Child2(Parent2): dcargs.parse(Child2, args=["--help"]) helptext = f.getvalue() assert ( - "--x STR Helptext. (default: This docstring may be tougher to parse?)\n" + "--x STR Helptext! (default: This docstring may be tougher to parse?)\n" in helptext ) From 455fbe9e490250a1642e42ae891a2d68a42761a3 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 15 Jan 2022 17:54:04 -0800 Subject: [PATCH 020/491] Minor refactor; fix some corner cases for generics --- README.md | 13 +- dcargs/_arguments.py | 357 +++++++++++------------ dcargs/_construction.py | 119 +++++--- dcargs/_parse.py | 18 +- dcargs/_parsers.py | 73 +++-- dcargs/_resolver.py | 6 +- dcargs/_serialization.py | 2 +- dcargs/_strings.py | 17 ++ examples/simple.py | 1 + examples/subparsers.py | 2 +- tests/test_docstrings.py | 1 - tests/test_generics_and_serialization.py | 33 ++- 12 files changed, 353 insertions(+), 289 deletions(-) diff --git a/README.md b/README.md index 59ac8c776..45cc41d56 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ import dcargs class Args: field1: str # A string field. field2: int # A numeric field. + flag: bool = False # A boolean flag. if __name__ == "__main__": @@ -62,24 +63,26 @@ if __name__ == "__main__": Running `python simple.py --help` would print: ``` -usage: simple.py [-h] --field1 STR --field2 INT - -optional arguments: - -h, --help show this help message and exit +usage: simple.py [-h] --field1 STR --field2 INT [--flag] required arguments: --field1 STR A string field. --field2 INT A numeric field. + +optional arguments: + -h, --help show this help message and exit + --flag A boolean flag. ``` And, from `python simple.py --field1 string --field2 4`: ``` -Args(field1='string', field2=4) +Args(field1='string', field2=4, flag=False) !dataclass:Args field1: string field2: 4 +flag: false ``` ### Feature list diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index f1cedc074..ac72a1eba 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -2,36 +2,27 @@ import collections.abc import dataclasses import enum -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, + cast, +) from typing_extensions import Final, Literal, _AnnotatedAlias, get_args, get_origin -from . import _construction, _docstrings, _strings +from . import _construction, _docstrings -T = TypeVar("T") - -def _instance_from_string(typ: Type[T], arg: str) -> T: - """Given a type and and a string from the command-line, reconstruct an object. Not - intended to deal with generic types or containers; these are handled in the - "argument transformations" below. - - This is intended to replace all calls to `type(string)`, which can cause unexpected - behavior. As an example, note that the following argparse code will always print - `True`, because `bool("True") == bool("False") == bool("0") == True`. - ``` - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--flag", type=bool) - - print(parser.parse_args().flag) - ``` - """ - if typ is bool: - return _strings.bool_from_string(arg) # type: ignore - else: - return typ(arg) # type: ignore +def _no_op_action(x): + return x @dataclasses.dataclass(frozen=True) @@ -39,17 +30,25 @@ class ArgumentDefinition: """Options for defining arguments. Contains all necessary arguments for argparse's add_argument() method.""" + prefix: str # Prefix for nesting. + field: dataclasses.Field # Corresponding dataclass field. + parent_class: Type # Class that this field belongs to. + + # Action that is called on parsed arguments. This handles conversions from strings + # to our desired types. + field_action: _construction.FieldAction + # Fields that will be populated initially. + # Important: from here on out, all fields correspond 1:1 to inputs to argparse's + # add_argument() method. name: str - field: dataclasses.Field - parent_class: Type type: Optional[Union[Type, TypeVar]] + default: Optional[Any] # Fields that will be handled by argument transformations. required: Optional[bool] = None action: Optional[str] = None nargs: Optional[Union[int, str]] = None - default: Optional[Any] = None choices: Optional[Set[Any]] = None metavar: Optional[Union[str, Tuple[str, ...]]] = None help: Optional[str] = None @@ -60,29 +59,29 @@ def add_argument( ) -> None: """Add a defined argument to a parser.""" kwargs = {k: v for k, v in vars(self).items() if v is not None} - name = "--" + kwargs.pop("name").replace("_", "-") + + # Important: as far as argparse is concerned, all inputs are strings. + # Conversions from strings to our desired types happen in the "field action"; + # this is a bit more flexible, and lets us handle more complex types like enums + # and multi-type tuples. + if "type" in kwargs: + kwargs["type"] = str + + # Don't pass field action into argparse. + if "dest" in kwargs: + kwargs["dest"] = self.prefix + kwargs["dest"] + kwargs.pop("field") kwargs.pop("parent_class") + kwargs.pop("prefix") + kwargs.pop("field_action") + kwargs.pop("name") - # Wrap the raw type with handling for special types. (currently only booleans) - # This feels a like a bit of band-aid; in the future, we may want to always set - # the argparse type to str and fold this logic into a more general version of - # what we currently call the "field role" (callables used for reconstructing - # lists, tuples, sets, etc). - if "type" in kwargs: - raw_type = kwargs["type"] - kwargs["type"] = lambda arg: _instance_from_string(raw_type, arg) - - parser.add_argument(name, **kwargs) - - def prefix(self, prefix: str) -> "ArgumentDefinition": - """Prefix an argument's name and destination. Used for nested dataclasses.""" - _strings.NESTED_DATACLASS_DELIMETER - arg = self - arg = dataclasses.replace(arg, name=prefix + arg.name) - if arg.dest is not None: - arg = dataclasses.replace(arg, dest=prefix + arg.dest) - return arg + # Note that the name must be passed in as a position argument. + parser.add_argument(self.get_flag(), **kwargs) + + def get_flag(self) -> str: + return "--" + (self.prefix + self.name).replace("_", "-") @staticmethod def make_from_field( @@ -90,97 +89,93 @@ def make_from_field( field: dataclasses.Field, type_from_typevar: Dict[TypeVar, Type], default_override: Optional[Any], - ) -> Tuple["ArgumentDefinition", _construction.FieldRole]: - """Create an argument definition from a field. Also returns a field role, which + ) -> "ArgumentDefinition": + """Create an argument definition from a field. Also returns a field action, which specifies special instructions for reconstruction.""" assert field.init, "Field must be in class constructor" # Create initial argument. arg = ArgumentDefinition( - name=field.name, + prefix="", field=field, parent_class=parent_class, + field_action=_no_op_action, + name=field.name, type=field.type, default=default_override, ) # Propagate argument through transforms until stable. prev_arg = arg - role: _construction.FieldRole = _construction.FieldRoleEnum.VANILLA_FIELD - def _handle_generics(arg: ArgumentDefinition) -> _ArgumentTransformOutput: + def _handle_generics(arg: ArgumentDefinition) -> ArgumentDefinition: """Handle generic arguments. Note that this needs to be a transform -- if we only checked field.type before running transforms, we wouldn't be able to handle cases like Optional[T].""" if isinstance(arg.type, TypeVar): assert arg.type in type_from_typevar, "TypeVar not bounded" - return ( - dataclasses.replace( - arg, type=type_from_typevar[arg.type] # type:ignore - ), - None, + return dataclasses.replace( + arg, type=type_from_typevar[arg.type] # type:ignore ) else: - return arg, None + return arg while True: for transform in [_handle_generics] + _argument_transforms: # type: ignore # Apply transform. - arg, new_role = transform(arg) - - # Update field role. - if new_role is not None: - assert ( - role == _construction.FieldRoleEnum.VANILLA_FIELD - ), "Something went wrong -- only one field role can be specified per argument!" - role = new_role + arg = transform(arg) # Stability check. if arg == prev_arg: break prev_arg = arg - return arg, role + if arg.field_action is _no_op_action and arg.type is not None: + cast_type = cast(Type, arg.type) + arg = dataclasses.replace( + arg, + field_action=lambda x: _construction.instance_from_string( + cast_type, + x, + ), + ) + elif arg.field_action is _no_op_action: + assert arg.action in ("store_true", "store_false") -# Argument transformations. -# Each transform returns an argument definition and (optionall) a special role for -# reconstruction -- note that a field can only ever have one role. + return arg -_ArgumentTransformOutput = Tuple[ArgumentDefinition, Optional[_construction.FieldRole]] + +# Argument transformations. +# Each transform returns an argument definition and (optionall) a special action for +# reconstruction -- note that a field can only ever have one action. -def _unwrap_final(arg: ArgumentDefinition) -> _ArgumentTransformOutput: +def _unwrap_final(arg: ArgumentDefinition) -> ArgumentDefinition: """Treat Final[T] as just T.""" if get_origin(arg.type) is Final: (typ,) = get_args(arg.type) - return ( - dataclasses.replace( - arg, - type=typ, - ), - None, + return dataclasses.replace( + arg, + type=typ, ) else: - return arg, None + return arg -def _unwrap_annotated(arg: ArgumentDefinition) -> _ArgumentTransformOutput: +def _unwrap_annotated(arg: ArgumentDefinition) -> ArgumentDefinition: """Treat Annotated[T, annotation] as just T.""" if hasattr(arg.type, "__class__") and arg.type.__class__ == _AnnotatedAlias: typ = get_origin(arg.type) - return ( - dataclasses.replace( - arg, - type=typ, - ), - None, + return dataclasses.replace( + arg, + type=typ, ) else: - return arg, None + return arg -def _handle_optionals(arg: ArgumentDefinition) -> _ArgumentTransformOutput: +def _handle_optionals(arg: ArgumentDefinition) -> ArgumentDefinition: """Transform for handling Optional[T] types. Sets default to None and marks arg as not required.""" if get_origin(arg.type) is Union: @@ -190,23 +185,20 @@ def _handle_optionals(arg: ArgumentDefinition) -> _ArgumentTransformOutput: ), "Union must be either over dataclasses (for subparsers) or Optional" (typ,) = options - {type(None)} required = False - return ( - dataclasses.replace( - arg, - type=typ, - required=required, - ), - None, + return dataclasses.replace( + arg, + type=typ, + required=required, ) else: - return arg, None + return arg -def _populate_defaults(arg: ArgumentDefinition) -> _ArgumentTransformOutput: +def _populate_defaults(arg: ArgumentDefinition) -> ArgumentDefinition: """Populate default values.""" if arg.default is not None: # Skip if another handler has already populated the default. - return arg, None + return arg default = None required = True @@ -220,13 +212,13 @@ def _populate_defaults(arg: ArgumentDefinition) -> _ArgumentTransformOutput: if arg.required is not None: required = arg.required - return dataclasses.replace(arg, default=default, required=required), None + return dataclasses.replace(arg, default=default, required=required) -def _bool_flags(arg: ArgumentDefinition) -> _ArgumentTransformOutput: +def _bool_flags(arg: ArgumentDefinition) -> ArgumentDefinition: """For booleans, we use a `store_true` action.""" if arg.type != bool: - return arg, None + return arg # Populate helptext for boolean flags => don't show default value, which can be # confusing. @@ -246,32 +238,23 @@ def _bool_flags(arg: ArgumentDefinition) -> _ArgumentTransformOutput: ) if arg.default is None: - return ( - dataclasses.replace( - arg, - metavar="{True,False}", - ), - None, + return dataclasses.replace( + arg, + metavar="{True,False}", ) elif arg.default is False: - return ( - dataclasses.replace( - arg, - action="store_true", - type=None, - ), - None, + return dataclasses.replace( + arg, + action="store_true", + type=None, ) elif arg.default is True: - return ( - dataclasses.replace( - arg, - dest=arg.name, - name="no_" + arg.name, - action="store_false", - type=None, - ), - None, + return dataclasses.replace( + arg, + dest=arg.name, + name="no_" + arg.name, + action="store_false", + type=None, ) else: assert False, "Invalid default" @@ -279,7 +262,7 @@ def _bool_flags(arg: ArgumentDefinition) -> _ArgumentTransformOutput: def _nargs_from_sequences_lists_and_sets( arg: ArgumentDefinition, -) -> _ArgumentTransformOutput: +) -> ArgumentDefinition: """Transform for handling Sequence[T] and list types.""" if get_origin(arg.type) in ( collections.abc.Sequence, # different from typing.Sequence! @@ -287,26 +270,26 @@ def _nargs_from_sequences_lists_and_sets( set, # different from typing.Set! ): (typ,) = get_args(arg.type) - role = get_origin(arg.type) - if role is collections.abc.Sequence: - role = list + container_type = get_origin(arg.type) + if container_type is collections.abc.Sequence: + container_type = list - return ( - dataclasses.replace( - arg, - type=typ, - # `*` is >=0 values, `+` is >=1 values - # We're going to require at least 1 value; if a user wants to accept no - # input, they can use Optional[Tuple[...]] - nargs="+", + return dataclasses.replace( + arg, + type=typ, + # `*` is >=0 values, `+` is >=1 values + # We're going to require at least 1 value; if a user wants to accept no + # input, they can use Optional[Tuple[...]] + nargs="+", + field_action=lambda str_list: container_type( # type: ignore + _construction.instance_from_string(typ, x) for x in str_list ), - role, ) else: - return arg, None + return arg -def _nargs_from_tuples(arg: ArgumentDefinition) -> _ArgumentTransformOutput: +def _nargs_from_tuples(arg: ArgumentDefinition) -> ArgumentDefinition: """Transform for handling Tuple[T, T, ...] types.""" if arg.nargs is None and get_origin(arg.type) is tuple: @@ -321,61 +304,56 @@ def _nargs_from_tuples(arg: ArgumentDefinition) -> _ArgumentTransformOutput: ), "If ellipsis is used, tuples must contain only one type." (typ,) = typeset_no_ellipsis - return ( - dataclasses.replace( - arg, - # `*` is >=0 values, `+` is >=1 values. - # We're going to require at least 1 value; if a user wants to accept no - # input, they can use Optional[Tuple[...]]. - nargs="+", - type=typ, + return dataclasses.replace( + arg, + # `*` is >=0 values, `+` is >=1 values. + # We're going to require at least 1 value; if a user wants to accept no + # input, they can use Optional[Tuple[...]]. + nargs="+", + type=typ, + field_action=lambda str_list: tuple( + _construction.instance_from_string(typ, x) for x in str_list ), - tuple, ) else: # Tuples with more than one type assert arg.metavar is None - return ( - dataclasses.replace( - arg, - nargs=len(types), - type=str, # Types will be converted in the dataclass reconstruction step. - metavar=tuple( - t.__name__.upper() if hasattr(t, "__name__") else "X" - for t in types - ), + return dataclasses.replace( + arg, + nargs=len(types), + type=str, # Types will be converted in the dataclass reconstruction step. + metavar=tuple( + t.__name__.upper() if hasattr(t, "__name__") else "X" for t in types ), - # Field role: convert lists of strings to tuples of the correct types. - lambda str_list: tuple( - _instance_from_string(typ, x) for typ, x in zip(types, str_list) + # Field action: convert lists of strings to tuples of the correct types. + field_action=lambda str_list: tuple( + _construction.instance_from_string(typ, x) + for typ, x in zip(types, str_list) ), ) else: - return arg, None + return arg -def _choices_from_literals(arg: ArgumentDefinition) -> _ArgumentTransformOutput: +def _choices_from_literals(arg: ArgumentDefinition) -> ArgumentDefinition: """For literal types, set choices.""" if get_origin(arg.type) is Literal: - choices = set(get_args(arg.type)) + choices = get_args(arg.type) assert ( len(set(map(type, choices))) == 1 ), "All choices in literal must have the same type!" - return ( - dataclasses.replace( - arg, - type=type(next(iter(choices))), - choices=choices, - ), - None, + return dataclasses.replace( + arg, + type=type(next(iter(choices))), + choices=set(map(str, choices)), ) else: - return arg, None + return arg -def _enums_as_strings(arg: ArgumentDefinition) -> _ArgumentTransformOutput: +def _enums_as_strings(arg: ArgumentDefinition) -> ArgumentDefinition: """For enums, use string representations.""" if isinstance(arg.type, type) and issubclass(arg.type, enum.Enum): if arg.choices is None: @@ -383,20 +361,18 @@ def _enums_as_strings(arg: ArgumentDefinition) -> _ArgumentTransformOutput: else: choices = set(x.name for x in arg.choices) - return ( - dataclasses.replace( - arg, - choices=choices, - type=str, - default=None if arg.default is None else arg.default.name, - ), - lambda enum_name: arg.type[enum_name], # type: ignore + return dataclasses.replace( + arg, + choices=choices, + type=str, + default=None if arg.default is None else arg.default.name, + field_action=lambda enum_name: arg.type[enum_name], # type: ignore ) else: - return arg, None + return arg -def _generate_helptext(arg: ArgumentDefinition) -> _ArgumentTransformOutput: +def _generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: """Generate helptext from docstring and argument name.""" if arg.help is None: help_parts = [] @@ -416,23 +392,22 @@ def _generate_helptext(arg: ArgumentDefinition) -> _ArgumentTransformOutput: # General case. help_parts.append("(default: %(default)s)") - return dataclasses.replace(arg, help=" ".join(help_parts)), None + return dataclasses.replace(arg, help=" ".join(help_parts)) else: - return arg, None + return arg -def _use_type_as_metavar(arg: ArgumentDefinition) -> _ArgumentTransformOutput: +def _use_type_as_metavar(arg: ArgumentDefinition) -> ArgumentDefinition: """Communicate the argument type using the metavar.""" if hasattr(arg.type, "__name__") and arg.choices is None and arg.metavar is None: - return ( - dataclasses.replace(arg, metavar=arg.type.__name__.upper()), # type: ignore - None, - ) + return dataclasses.replace( + arg, metavar=arg.type.__name__.upper() # type: ignore + ) # type: ignore else: - return arg, None + return arg -_argument_transforms: List[Callable[[ArgumentDefinition], _ArgumentTransformOutput]] = [ +_argument_transforms: List[Callable[[ArgumentDefinition], ArgumentDefinition]] = [ _unwrap_final, _unwrap_annotated, _handle_optionals, diff --git a/dcargs/_construction.py b/dcargs/_construction.py index bf2f7fb73..2788b7f7a 100644 --- a/dcargs/_construction.py +++ b/dcargs/_construction.py @@ -1,44 +1,72 @@ -import dataclasses -import enum -from typing import Any, Callable, Dict, Set, Tuple, Type, TypeVar, Union +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Set, + Tuple, + Type, + TypeVar, + Union, +) from typing_extensions import get_args from . import _resolver, _strings -DataclassType = TypeVar("DataclassType") +if TYPE_CHECKING: + from . import _parsers + +T = TypeVar("T") + + +def instance_from_string(typ: Type[T], arg: str) -> T: + """Given a type and and a string from the command-line, reconstruct an object. Not + intended to deal with generic types or containers; these are handled in the + "argument transformations". + This is intended to replace all calls to `type(string)`, which can cause unexpected + behavior. As an example, note that the following argparse code will always print + `True`, because `bool("True") == bool("False") == bool("0") == True`. + ``` + import argparse -# Each dataclass field is assigned a role, which is either taken from an enum or a -# callable type that converts raw values from the argparse namespace to their final -# values in the dataclass. -class FieldRoleEnum(enum.Enum): - VANILLA_FIELD = enum.auto() - NESTED_DATACLASS = enum.auto() # Singular nested dataclass. - SUBPARSERS = enum.auto() # Unions over dataclasses. + parser = argparse.ArgumentParser() + parser.add_argument("--flag", type=bool) + print(parser.parse_args().flag) + ``` + """ + if typ is bool: + return _strings.bool_from_string(arg) # type: ignore + else: + return typ(arg) # type: ignore -FieldRole = Union[FieldRoleEnum, Callable[[Any], Any]] +# Each argument is assigned an action, which determines how it's populated from the CLI +# string. +# +# There are 2 options: +FieldAction = Union[ + # Most standard fields: these are converted from strings from the CLI. + Callable[[str], Any], + # Sequence fields! This should be used whenever argparse's `nargs` field is set. + Callable[[List[str]], Any], +] -@dataclasses.dataclass -class ConstructionMetadata: - """Metadata recorded during parsing that's needed for reconstructing dataclasses.""" - role_from_field: Dict[dataclasses.Field, FieldRole] = dataclasses.field( - default_factory=dict - ) - subparser_name_from_type: Dict[Type, str] = dataclasses.field(default_factory=dict) +class FieldActionValueError(Exception): + """Exception raised when field actions fail; this is caused by""" - def update(self, other: "ConstructionMetadata") -> None: - self.role_from_field.update(other.role_from_field) - self.subparser_name_from_type.update(other.subparser_name_from_type) + +DataclassType = TypeVar("DataclassType") def construct_dataclass( cls: Type[DataclassType], + parser_definition: "_parsers.ParserDefinition", value_from_arg: Dict[str, Any], - metadata: ConstructionMetadata, field_name_prefix: str = "", ) -> Tuple[DataclassType, Set[str]]: """Construct a dataclass object from a dictionary of values from argparse. @@ -47,7 +75,7 @@ def construct_dataclass( assert _resolver.is_dataclass(cls) - cls, type_from_typevar = _resolver.resolve_generic_dataclasses(cls) + cls, type_from_typevar = _resolver.resolve_generic_classes(cls) kwargs: Dict[str, Any] = {} consumed_keywords: Set[str] = set() @@ -60,13 +88,15 @@ def get_value_from_arg(arg: str) -> Any: consumed_keywords.add(arg) return value_from_arg[arg] + arg_from_prefixed_field_name = {} + for arg in parser_definition.args: + arg_from_prefixed_field_name[arg.prefix + arg.field.name] = arg + for field in _resolver.resolved_fields(cls): # type: ignore if not field.init: continue value: Any - role = metadata.role_from_field[field] - prefixed_field_name = field_name_prefix + field.name # Resolve field type @@ -76,18 +106,33 @@ def get_value_from_arg(arg: str) -> Any: else field.type ) - if role is FieldRoleEnum.NESTED_DATACLASS: + if prefixed_field_name in arg_from_prefixed_field_name: + # Callable actions. Used for tuples, lists, sets, etc. + arg = arg_from_prefixed_field_name[prefixed_field_name] + action = arg.field_action + value = get_value_from_arg(prefixed_field_name) + if value is not None: + try: + value = action(value) + except ValueError as e: + raise FieldActionValueError( + f"Parsing error for {arg.get_flag()}: {e.args[0]}" + ) + elif prefixed_field_name in parser_definition.nested_dataclass_field_names: # Nested dataclasses. value, consumed_keywords_child = construct_dataclass( field_type, + parser_definition, value_from_arg, - metadata, field_name_prefix=prefixed_field_name + _strings.NESTED_DATACLASS_DELIMETER, ) consumed_keywords |= consumed_keywords_child - elif role is FieldRoleEnum.SUBPARSERS: - # Unions over dataclasses (subparsers). + else: + # Unions over dataclasses (subparsers). This is the only other option. + assert parser_definition.subparsers is not None + assert field.name == parser_definition.subparsers.name + subparser_dest = _strings.SUBPARSER_DEST_FMT.format( name=prefixed_field_name ) @@ -104,26 +149,16 @@ def get_value_from_arg(arg: str) -> Any: ) chosen_cls = None for option in options: - if metadata.subparser_name_from_type[option] == subparser_name: + if _strings.subparser_name_from_type(option) == subparser_name: chosen_cls = option break assert chosen_cls is not None value, consumed_keywords_child = construct_dataclass( chosen_cls, + parser_definition.subparsers.parsers[subparser_name], value_from_arg, - metadata, ) consumed_keywords |= consumed_keywords_child - elif role is FieldRoleEnum.VANILLA_FIELD: - # General case. - value = get_value_from_arg(prefixed_field_name) - elif callable(role): - # Callable roles. Used for tuples, lists, sets, etc. - value = get_value_from_arg(prefixed_field_name) - if value is not None: - value = role(value) - else: - assert False kwargs[field.name] = value diff --git a/dcargs/_parse.py b/dcargs/_parse.py index 861fbe716..84ff91b6a 100644 --- a/dcargs/_parse.py +++ b/dcargs/_parse.py @@ -17,7 +17,7 @@ def parse( if description is None: description = "" - parser_definition, construction_metadata = _parsers.ParserDefinition.from_dataclass( + parser_definition = _parsers.ParserDefinition.from_dataclass( cls, parent_dataclasses=set(), # Used for recursive calls. parent_type_from_typevar=None, # Used for recursive calls. @@ -33,9 +33,19 @@ def parse( namespace = parser.parse_args(args) value_from_arg = vars(namespace) - out, consumed_keywords = _construction.construct_dataclass( - cls, value_from_arg, construction_metadata - ) + + try: + out, consumed_keywords = _construction.construct_dataclass( + cls, + parser_definition, + value_from_arg, + ) + except _construction.FieldActionValueError as e: + parser.print_usage() + print() + print(e.args[0]) + raise SystemExit() + assert ( consumed_keywords == value_from_arg.keys() ), "Not all arguments were consumed!" diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index a62c1eb61..db5cb08b7 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -4,7 +4,7 @@ from typing_extensions import get_args, get_origin -from . import _arguments, _construction, _docstrings, _resolver, _strings +from . import _arguments, _docstrings, _resolver, _strings T = TypeVar("T") @@ -13,7 +13,9 @@ class ParserDefinition: """Each parser contains a list of arguments and optionally a subparser.""" + # Track a list of (argument def, action) pairs. args: List[_arguments.ArgumentDefinition] + nested_dataclass_field_names: List[str] subparsers: Optional["SubparsersDefinition"] def apply(self, parser: argparse.ArgumentParser) -> None: @@ -47,7 +49,7 @@ def from_dataclass( parent_dataclasses: Optional[Set[Type]], parent_type_from_typevar: Optional[Dict[TypeVar, Type]], default_instance: Optional[T], - ) -> Tuple["ParserDefinition", _construction.ConstructionMetadata]: + ) -> "ParserDefinition": """Create a parser definition from a dataclass.""" if parent_dataclasses is None: @@ -55,7 +57,7 @@ def from_dataclass( assert _resolver.is_dataclass(cls) - cls, type_from_typevar = _resolver.resolve_generic_dataclasses(cls) + cls, type_from_typevar = _resolver.resolve_generic_classes(cls) if parent_type_from_typevar is not None: for typevar, typ in type_from_typevar.items(): @@ -68,8 +70,8 @@ def from_dataclass( parent_dataclasses = parent_dataclasses | {cls} args = [] + nested_dataclass_field_names = [] subparsers = None - metadata = _construction.ConstructionMetadata() for field in _resolver.resolved_fields(cls): # type: ignore # Ignore fields not in constructor @@ -90,22 +92,22 @@ def from_dataclass( assert ( subparsers is None ), "Only one subparser (union over dataclasses) is allowed per class" - subparsers, subparsers_metadata = subparsers_out - metadata.update(subparsers_metadata) + subparsers = subparsers_out continue # Try to interpret field as a nested dataclass. nested_out = field_parser.handle_nested_dataclasses() if nested_out is not None: - child_args, child_metadata = nested_out - args.extend(child_args) - metadata.update(child_metadata) + child_args, child_nested_field_names = nested_out + args.extend(child_args) + nested_dataclass_field_names.extend(child_nested_field_names) + nested_dataclass_field_names.append(field.name) continue # Handle simple fields! - arg, role = _arguments.ArgumentDefinition.make_from_field( + arg = _arguments.ArgumentDefinition.make_from_field( cls, field, type_from_typevar, @@ -114,14 +116,11 @@ def from_dataclass( else None, ) args.append(arg) - metadata.role_from_field[field] = role - - return ( - ParserDefinition( - args=args, - subparsers=subparsers, - ), - metadata, + + return ParserDefinition( + args=args, + nested_dataclass_field_names=nested_dataclass_field_names, + subparsers=subparsers, ) @@ -148,12 +147,10 @@ class _NestedDataclassHandler: def handle_unions_over_dataclasses( self, - ) -> Optional[Tuple["SubparsersDefinition", _construction.ConstructionMetadata]]: + ) -> Optional["SubparsersDefinition"]: """Handle unions over dataclasses, which are converted to subparsers.. Returns `None` if not applicable.""" - metadata = _construction.ConstructionMetadata() - # Union of dataclasses should create subparsers. if get_origin(self.field.type) is not Union: return None @@ -177,19 +174,14 @@ def handle_unions_over_dataclasses( parsers: Dict[str, ParserDefinition] = {} for option in options_no_none: - subparser_name = _strings.hyphen_separated_from_camel_case(option.__name__) - metadata.subparser_name_from_type[option] = subparser_name + subparser_name = _strings.subparser_name_from_type(option) - ( - parsers[subparser_name], - child_metadata, - ) = ParserDefinition.from_dataclass( + parsers[subparser_name] = ParserDefinition.from_dataclass( option, self.parent_dataclasses, parent_type_from_typevar=self.type_from_typevar, default_instance=None, ) - metadata.update(child_metadata) subparsers = SubparsersDefinition( name=self.field.name, @@ -197,15 +189,12 @@ def handle_unions_over_dataclasses( parsers=parsers, required=(options == options_no_none), # Not required if no options. ) - metadata.role_from_field[self.field] = _construction.FieldRoleEnum.SUBPARSERS - return subparsers, metadata + return subparsers def handle_nested_dataclasses( self, - ) -> Optional[ - Tuple[List[_arguments.ArgumentDefinition], _construction.ConstructionMetadata] - ]: + ) -> Optional[Tuple[List[_arguments.ArgumentDefinition], List[str]]]: """Handle nested dataclasses. Returns `None` if not applicable.""" # Resolve field type field_type = ( @@ -223,7 +212,7 @@ def handle_nested_dataclasses( elif self.field.default is not dataclasses.MISSING: default = self.field.default - child_definition, child_metadata = ParserDefinition.from_dataclass( + child_definition = ParserDefinition.from_dataclass( field_type, self.parent_dataclasses, parent_type_from_typevar=self.type_from_typevar, @@ -232,12 +221,16 @@ def handle_nested_dataclasses( child_args = child_definition.args for i, arg in enumerate(child_args): - child_args[i] = arg.prefix( - self.field.name + _strings.NESTED_DATACLASS_DELIMETER + child_args[i] = dataclasses.replace( + arg, + prefix=self.field.name + + _strings.NESTED_DATACLASS_DELIMETER + + arg.prefix, ) - child_metadata.role_from_field[ - self.field - ] = _construction.FieldRoleEnum.NESTED_DATACLASS + nested_dataclass_field_names = [ + self.field.name + _strings.NESTED_DATACLASS_DELIMETER + x + for x in child_definition.nested_dataclass_field_names + ] - return child_args, child_metadata + return child_args, nested_dataclass_field_names diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index e09fbf812..5adab5af9 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -12,11 +12,11 @@ def is_dataclass(cls: Type) -> bool: return dataclasses.is_dataclass(cls if origin_cls is None else origin_cls) -def resolve_generic_dataclasses( +def resolve_generic_classes( cls: Type, ) -> Tuple[Type, Dict[TypeVar, Type]]: - """If the input is a dataclass: no-op. If it's a generic alias: returns the root - dataclass, and a mapping from typevars to concrete types.""" + """If the input is a class: no-op. If it's a generic alias: returns the origin + class, and a mapping from typevars to concrete types.""" origin_cls = get_origin(cls) if origin_cls is not None: diff --git a/dcargs/_serialization.py b/dcargs/_serialization.py index f91cd771c..0df45d6e3 100644 --- a/dcargs/_serialization.py +++ b/dcargs/_serialization.py @@ -41,7 +41,7 @@ def _get_contained_special_types_from_type( else _parent_contained_dataclasses ) - cls, type_from_typevar = _resolver.resolve_generic_dataclasses(cls) + cls, type_from_typevar = _resolver.resolve_generic_classes(cls) contained_dataclasses = {cls} diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 69bb2292c..cd1799c7e 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -1,6 +1,9 @@ import functools import re import textwrap +from typing import Type + +from . import _resolver NESTED_DATACLASS_DELIMETER: str = "." SUBPARSER_DEST_FMT: str = "{name} (positional)" @@ -23,6 +26,20 @@ def hyphen_separated_from_camel_case(name: str) -> str: return _camel_separator_pattern().sub(r"-\1", name).lower() +def subparser_name_from_type(cls: Type) -> str: + cls, type_from_typevar = _resolver.resolve_generic_classes(cls) + if len(type_from_typevar) == 0: + assert hasattr(cls, "__name__") + return hyphen_separated_from_camel_case(cls.__name__) # type: ignore + + return "-".join( + map( + subparser_name_from_type, + [cls] + list(type_from_typevar.values()), + ) + ) + + def bool_from_string(text: str) -> bool: text = text.lower() if text in ("true", "1"): diff --git a/examples/simple.py b/examples/simple.py index eebcca04f..06691177f 100644 --- a/examples/simple.py +++ b/examples/simple.py @@ -7,6 +7,7 @@ class Args: field1: str # A string field. field2: int # A numeric field. + flag: bool = False # A boolean flag. if __name__ == "__main__": diff --git a/examples/subparsers.py b/examples/subparsers.py index 1ba651671..aa4f1d2db 100644 --- a/examples/subparsers.py +++ b/examples/subparsers.py @@ -22,6 +22,6 @@ class Commit: all: bool = False -if __name__ == "__main": +if __name__ == "__main__": args = dcargs.parse(Args) print(args) diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index 625a3cfac..eea16fafd 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -69,7 +69,6 @@ class Config: with contextlib.redirect_stdout(f): dcargs.parse(Config, args=["--help"]) helptext = f.getvalue() - print(helptext) assert " --x INT An optional variable. (default: None)\n" in helptext diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 7bc4760b9..b3d714417 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -1,6 +1,6 @@ import dataclasses import enum -from typing import Generic, Type, TypeVar, Union +from typing import Generic, List, Type, TypeVar, Union import pytest @@ -177,3 +177,34 @@ class Subparser(Generic[T1, T2]): ) assert parsed_instance == Subparser(CommandTwo(7)) _check_serialization_identity(Subparser[CommandOne, CommandTwo], parsed_instance) + + +def test_generic_subparsers_in_container(): + T = TypeVar("T") + + @dataclasses.dataclass + class Command(Generic[T]): + a: List[int] + + T1 = TypeVar("T1") + T2 = TypeVar("T2") + + @dataclasses.dataclass + class Subparser(Generic[T1, T2]): + command: Union[T1, T2] + + parsed_instance = dcargs.parse( + Subparser[Command[int], Command[float]], args="command-int --a 5 3".split(" ") + ) + assert parsed_instance == Subparser(Command([5, 3])) + _check_serialization_identity( + Subparser[Command[int], Command[float]], parsed_instance + ) + + parsed_instance = dcargs.parse( + Subparser[Command[int], Command[float]], args="command-float --a 7 2".split(" ") + ) + assert parsed_instance == Subparser(Command([7.0, 2.0])) + _check_serialization_identity( + Subparser[Command[int], Command[float]], parsed_instance + ) From 6481ab668f62a2c99c87b8645c3f832888829398 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 15 Jan 2022 18:45:19 -0800 Subject: [PATCH 021/491] More general handling for generics --- dcargs/_arguments.py | 542 ++++++++++++----------- dcargs/_construction.py | 23 - dcargs/_parsers.py | 1 - tests/test_generics_and_serialization.py | 22 +- 4 files changed, 300 insertions(+), 288 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index ac72a1eba..d9b2cdc69 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -18,17 +18,23 @@ from typing_extensions import Final, Literal, _AnnotatedAlias, get_args, get_origin -from . import _construction, _docstrings +from . import _construction, _docstrings, _strings def _no_op_action(x): return x +T = TypeVar("T") + + @dataclasses.dataclass(frozen=True) class ArgumentDefinition: """Options for defining arguments. Contains all necessary arguments for argparse's - add_argument() method.""" + add_argument() method. + + TODO: this class (as well as major other parts of this library) has succumbed a bit + to entropy and could benefit from some refactoring.""" prefix: str # Prefix for nesting. field: dataclasses.Field # Corresponding dataclass field. @@ -109,20 +115,35 @@ def make_from_field( # Propagate argument through transforms until stable. prev_arg = arg - def _handle_generics(arg: ArgumentDefinition) -> ArgumentDefinition: - """Handle generic arguments. Note that this needs to be a transform -- if we - only checked field.type before running transforms, we wouldn't be able to - handle cases like Optional[T].""" - if isinstance(arg.type, TypeVar): - assert arg.type in type_from_typevar, "TypeVar not bounded" - return dataclasses.replace( - arg, type=type_from_typevar[arg.type] # type:ignore - ) + def instance_from_string(typ: Type[T], arg: str) -> T: + """Given a type and and a string from the command-line, reconstruct an object. Not + intended to deal with containers; these are handled in the argument + transformations. + + This is intended to replace all calls to `type(string)`, which can cause unexpected + behavior. As an example, note that the following argparse code will always print + `True`, because `bool("True") == bool("False") == bool("0") == True`. + ``` + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--flag", type=bool) + + print(parser.parse_args().flag) + ``` + """ + if typ in type_from_typevar: + # The Type vs TypeVar annotations could be cleaned up... + typ = type_from_typevar[typ] # type: ignore + + if typ is bool: + return _strings.bool_from_string(arg) # type: ignore else: - return arg + return typ(arg) # type: ignore + argument_transforms = _get_argument_transforms(instance_from_string) while True: - for transform in [_handle_generics] + _argument_transforms: # type: ignore + for transform in argument_transforms: # type: ignore # Apply transform. arg = transform(arg) @@ -135,7 +156,7 @@ def _handle_generics(arg: ArgumentDefinition) -> ArgumentDefinition: cast_type = cast(Type, arg.type) arg = dataclasses.replace( arg, - field_action=lambda x: _construction.instance_from_string( + field_action=lambda x: instance_from_string( cast_type, x, ), @@ -146,236 +167,78 @@ def _handle_generics(arg: ArgumentDefinition) -> ArgumentDefinition: return arg -# Argument transformations. -# Each transform returns an argument definition and (optionall) a special action for -# reconstruction -- note that a field can only ever have one action. - - -def _unwrap_final(arg: ArgumentDefinition) -> ArgumentDefinition: - """Treat Final[T] as just T.""" - if get_origin(arg.type) is Final: - (typ,) = get_args(arg.type) - return dataclasses.replace( - arg, - type=typ, - ) - else: - return arg - - -def _unwrap_annotated(arg: ArgumentDefinition) -> ArgumentDefinition: - """Treat Annotated[T, annotation] as just T.""" - if hasattr(arg.type, "__class__") and arg.type.__class__ == _AnnotatedAlias: - typ = get_origin(arg.type) - return dataclasses.replace( - arg, - type=typ, - ) - else: - return arg - - -def _handle_optionals(arg: ArgumentDefinition) -> ArgumentDefinition: - """Transform for handling Optional[T] types. Sets default to None and marks arg as - not required.""" - if get_origin(arg.type) is Union: - options = set(get_args(arg.type)) - assert ( - len(options) == 2 and type(None) in options - ), "Union must be either over dataclasses (for subparsers) or Optional" - (typ,) = options - {type(None)} - required = False - return dataclasses.replace( - arg, - type=typ, - required=required, - ) - else: - return arg - - -def _populate_defaults(arg: ArgumentDefinition) -> ArgumentDefinition: - """Populate default values.""" - if arg.default is not None: - # Skip if another handler has already populated the default. - return arg - - default = None - required = True - if arg.field.default is not dataclasses.MISSING: - default = arg.field.default - required = False - elif arg.field.default_factory is not dataclasses.MISSING: # type: ignore - default = arg.field.default_factory() # type: ignore - required = False - - if arg.required is not None: - required = arg.required - - return dataclasses.replace(arg, default=default, required=required) - - -def _bool_flags(arg: ArgumentDefinition) -> ArgumentDefinition: - """For booleans, we use a `store_true` action.""" - if arg.type != bool: - return arg - - # Populate helptext for boolean flags => don't show default value, which can be - # confusing. - docstring_help = _docstrings.get_field_docstring(arg.parent_class, arg.field.name) - if docstring_help is not None: - # Note that the percent symbol needs some extra handling in argparse. - # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string - docstring_help = docstring_help.replace("%", "%%") - arg = dataclasses.replace( - arg, - help=docstring_help, - ) - else: - arg = dataclasses.replace( - arg, - help="", - ) - - if arg.default is None: - return dataclasses.replace( - arg, - metavar="{True,False}", - ) - elif arg.default is False: - return dataclasses.replace( - arg, - action="store_true", - type=None, - ) - elif arg.default is True: - return dataclasses.replace( - arg, - dest=arg.name, - name="no_" + arg.name, - action="store_false", - type=None, - ) - else: - assert False, "Invalid default" - - -def _nargs_from_sequences_lists_and_sets( - arg: ArgumentDefinition, -) -> ArgumentDefinition: - """Transform for handling Sequence[T] and list types.""" - if get_origin(arg.type) in ( - collections.abc.Sequence, # different from typing.Sequence! - list, # different from typing.List! - set, # different from typing.Set! - ): - (typ,) = get_args(arg.type) - container_type = get_origin(arg.type) - if container_type is collections.abc.Sequence: - container_type = list - - return dataclasses.replace( - arg, - type=typ, - # `*` is >=0 values, `+` is >=1 values - # We're going to require at least 1 value; if a user wants to accept no - # input, they can use Optional[Tuple[...]] - nargs="+", - field_action=lambda str_list: container_type( # type: ignore - _construction.instance_from_string(typ, x) for x in str_list - ), - ) - else: - return arg - - -def _nargs_from_tuples(arg: ArgumentDefinition) -> ArgumentDefinition: - """Transform for handling Tuple[T, T, ...] types.""" - - if arg.nargs is None and get_origin(arg.type) is tuple: - types = get_args(arg.type) - typeset = set(types) - typeset_no_ellipsis = typeset - {Ellipsis} - - if typeset_no_ellipsis != typeset: - # Ellipsis: variable argument counts - assert ( - len(typeset_no_ellipsis) == 1 - ), "If ellipsis is used, tuples must contain only one type." - (typ,) = typeset_no_ellipsis +def _get_argument_transforms( + instance_from_string: Callable[[Type[T], str], T] +) -> List[Callable[[ArgumentDefinition], ArgumentDefinition]]: + """Get a list of argument transformations.""" + def unwrap_final(arg: ArgumentDefinition) -> ArgumentDefinition: + """Treat Final[T] as just T.""" + if get_origin(arg.type) is Final: + (typ,) = get_args(arg.type) return dataclasses.replace( arg, - # `*` is >=0 values, `+` is >=1 values. - # We're going to require at least 1 value; if a user wants to accept no - # input, they can use Optional[Tuple[...]]. - nargs="+", type=typ, - field_action=lambda str_list: tuple( - _construction.instance_from_string(typ, x) for x in str_list - ), ) else: - # Tuples with more than one type - assert arg.metavar is None + return arg + def unwrap_annotated(arg: ArgumentDefinition) -> ArgumentDefinition: + """Treat Annotated[T, annotation] as just T.""" + if hasattr(arg.type, "__class__") and arg.type.__class__ == _AnnotatedAlias: + typ = get_origin(arg.type) return dataclasses.replace( arg, - nargs=len(types), - type=str, # Types will be converted in the dataclass reconstruction step. - metavar=tuple( - t.__name__.upper() if hasattr(t, "__name__") else "X" for t in types - ), - # Field action: convert lists of strings to tuples of the correct types. - field_action=lambda str_list: tuple( - _construction.instance_from_string(typ, x) - for typ, x in zip(types, str_list) - ), + type=typ, ) - - else: - return arg - - -def _choices_from_literals(arg: ArgumentDefinition) -> ArgumentDefinition: - """For literal types, set choices.""" - if get_origin(arg.type) is Literal: - choices = get_args(arg.type) - assert ( - len(set(map(type, choices))) == 1 - ), "All choices in literal must have the same type!" - return dataclasses.replace( - arg, - type=type(next(iter(choices))), - choices=set(map(str, choices)), - ) - else: - return arg - - -def _enums_as_strings(arg: ArgumentDefinition) -> ArgumentDefinition: - """For enums, use string representations.""" - if isinstance(arg.type, type) and issubclass(arg.type, enum.Enum): - if arg.choices is None: - choices = set(x.name for x in arg.type) else: - choices = set(x.name for x in arg.choices) - - return dataclasses.replace( - arg, - choices=choices, - type=str, - default=None if arg.default is None else arg.default.name, - field_action=lambda enum_name: arg.type[enum_name], # type: ignore - ) - else: - return arg - + return arg -def _generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: - """Generate helptext from docstring and argument name.""" - if arg.help is None: - help_parts = [] + def handle_optionals(arg: ArgumentDefinition) -> ArgumentDefinition: + """Transform for handling Optional[T] types. Sets default to None and marks arg as + not required.""" + if get_origin(arg.type) is Union: + options = set(get_args(arg.type)) + assert ( + len(options) == 2 and type(None) in options + ), "Union must be either over dataclasses (for subparsers) or Optional" + (typ,) = options - {type(None)} + required = False + return dataclasses.replace( + arg, + type=typ, + required=required, + ) + else: + return arg + + def populate_defaults(arg: ArgumentDefinition) -> ArgumentDefinition: + """Populate default values.""" + if arg.default is not None: + # Skip if another handler has already populated the default. + return arg + + default = None + required = True + if arg.field.default is not dataclasses.MISSING: + default = arg.field.default + required = False + elif arg.field.default_factory is not dataclasses.MISSING: # type: ignore + default = arg.field.default_factory() # type: ignore + required = False + + if arg.required is not None: + required = arg.required + + return dataclasses.replace(arg, default=default, required=required) + + def bool_flags(arg: ArgumentDefinition) -> ArgumentDefinition: + """For booleans, we use a `store_true` action.""" + if arg.type != bool: + return arg + + # Populate helptext for boolean flags => don't show default value, which can be + # confusing. docstring_help = _docstrings.get_field_docstring( arg.parent_class, arg.field.name ) @@ -383,40 +246,193 @@ def _generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: # Note that the percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string docstring_help = docstring_help.replace("%", "%%") - help_parts.append(docstring_help) + arg = dataclasses.replace( + arg, + help=docstring_help, + ) + else: + arg = dataclasses.replace( + arg, + help="", + ) - if arg.default is not None and hasattr(arg.default, "name"): - # Special case for enums. - help_parts.append(f"(default: {arg.default.name})") - elif not arg.required: - # General case. - help_parts.append("(default: %(default)s)") + if arg.default is None: + return dataclasses.replace( + arg, + metavar="{True,False}", + ) + elif arg.default is False: + return dataclasses.replace( + arg, + action="store_true", + type=None, + ) + elif arg.default is True: + return dataclasses.replace( + arg, + dest=arg.name, + name="no_" + arg.name, + action="store_false", + type=None, + ) + else: + assert False, "Invalid default" + + def nargs_from_sequences_lists_and_sets( + arg: ArgumentDefinition, + ) -> ArgumentDefinition: + """Transform for handling Sequence[T] and list types.""" + if get_origin(arg.type) in ( + collections.abc.Sequence, # different from typing.Sequence! + list, # different from typing.List! + set, # different from typing.Set! + ): + (typ,) = get_args(arg.type) + container_type = get_origin(arg.type) + if container_type is collections.abc.Sequence: + container_type = list - return dataclasses.replace(arg, help=" ".join(help_parts)) - else: - return arg + return dataclasses.replace( + arg, + type=typ, + # `*` is >=0 values, `+` is >=1 values + # We're going to require at least 1 value; if a user wants to accept no + # input, they can use Optional[Tuple[...]] + nargs="+", + field_action=lambda str_list: container_type( # type: ignore + instance_from_string(typ, x) for x in str_list + ), + ) + else: + return arg + def nargs_from_tuples(arg: ArgumentDefinition) -> ArgumentDefinition: + """Transform for handling Tuple[T, T, ...] types.""" -def _use_type_as_metavar(arg: ArgumentDefinition) -> ArgumentDefinition: - """Communicate the argument type using the metavar.""" - if hasattr(arg.type, "__name__") and arg.choices is None and arg.metavar is None: - return dataclasses.replace( - arg, metavar=arg.type.__name__.upper() # type: ignore - ) # type: ignore - else: - return arg + if arg.nargs is None and get_origin(arg.type) is tuple: + types = get_args(arg.type) + typeset = set(types) + typeset_no_ellipsis = typeset - {Ellipsis} + + if typeset_no_ellipsis != typeset: + # Ellipsis: variable argument counts + assert ( + len(typeset_no_ellipsis) == 1 + ), "If ellipsis is used, tuples must contain only one type." + (typ,) = typeset_no_ellipsis + + return dataclasses.replace( + arg, + # `*` is >=0 values, `+` is >=1 values. + # We're going to require at least 1 value; if a user wants to accept no + # input, they can use Optional[Tuple[...]]. + nargs="+", + type=typ, + field_action=lambda str_list: tuple( + instance_from_string(typ, x) for x in str_list + ), + ) + else: + # Tuples with more than one type + assert arg.metavar is None + + return dataclasses.replace( + arg, + nargs=len(types), + type=str, # Types will be converted in the dataclass reconstruction step. + metavar=tuple( + t.__name__.upper() if hasattr(t, "__name__") else "X" + for t in types + ), + # Field action: convert lists of strings to tuples of the correct types. + field_action=lambda str_list: tuple( + instance_from_string(typ, x) for typ, x in zip(types, str_list) + ), + ) + + else: + return arg + + def choices_from_literals(arg: ArgumentDefinition) -> ArgumentDefinition: + """For literal types, set choices.""" + if get_origin(arg.type) is Literal: + choices = get_args(arg.type) + assert ( + len(set(map(type, choices))) == 1 + ), "All choices in literal must have the same type!" + return dataclasses.replace( + arg, + type=type(next(iter(choices))), + choices=set(map(str, choices)), + ) + else: + return arg + def enums_as_strings(arg: ArgumentDefinition) -> ArgumentDefinition: + """For enums, use string representations.""" + if isinstance(arg.type, type) and issubclass(arg.type, enum.Enum): + if arg.choices is None: + choices = set(x.name for x in arg.type) + else: + choices = set(x.name for x in arg.choices) -_argument_transforms: List[Callable[[ArgumentDefinition], ArgumentDefinition]] = [ - _unwrap_final, - _unwrap_annotated, - _handle_optionals, - _populate_defaults, - _bool_flags, - _nargs_from_sequences_lists_and_sets, - _nargs_from_tuples, - _choices_from_literals, - _enums_as_strings, - _generate_helptext, - _use_type_as_metavar, -] + return dataclasses.replace( + arg, + choices=choices, + type=str, + default=None if arg.default is None else arg.default.name, + field_action=lambda enum_name: arg.type[enum_name], # type: ignore + ) + else: + return arg + + def generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: + """Generate helptext from docstring and argument name.""" + if arg.help is None: + help_parts = [] + docstring_help = _docstrings.get_field_docstring( + arg.parent_class, arg.field.name + ) + if docstring_help is not None: + # Note that the percent symbol needs some extra handling in argparse. + # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string + docstring_help = docstring_help.replace("%", "%%") + help_parts.append(docstring_help) + + if arg.default is not None and hasattr(arg.default, "name"): + # Special case for enums. + help_parts.append(f"(default: {arg.default.name})") + elif not arg.required: + # General case. + help_parts.append("(default: %(default)s)") + + return dataclasses.replace(arg, help=" ".join(help_parts)) + else: + return arg + + def use_type_as_metavar(arg: ArgumentDefinition) -> ArgumentDefinition: + """Communicate the argument type using the metavar.""" + if ( + hasattr(arg.type, "__name__") + and arg.choices is None + and arg.metavar is None + ): + return dataclasses.replace( + arg, metavar=arg.type.__name__.upper() # type: ignore + ) # type: ignore + else: + return arg + + return [ + unwrap_final, + unwrap_annotated, + handle_optionals, + populate_defaults, + bool_flags, + nargs_from_sequences_lists_and_sets, + nargs_from_tuples, + choices_from_literals, + enums_as_strings, + generate_helptext, + use_type_as_metavar, + ] diff --git a/dcargs/_construction.py b/dcargs/_construction.py index 2788b7f7a..d34771255 100644 --- a/dcargs/_construction.py +++ b/dcargs/_construction.py @@ -21,29 +21,6 @@ T = TypeVar("T") -def instance_from_string(typ: Type[T], arg: str) -> T: - """Given a type and and a string from the command-line, reconstruct an object. Not - intended to deal with generic types or containers; these are handled in the - "argument transformations". - - This is intended to replace all calls to `type(string)`, which can cause unexpected - behavior. As an example, note that the following argparse code will always print - `True`, because `bool("True") == bool("False") == bool("0") == True`. - ``` - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--flag", type=bool) - - print(parser.parse_args().flag) - ``` - """ - if typ is bool: - return _strings.bool_from_string(arg) # type: ignore - else: - return typ(arg) # type: ignore - - # Each argument is assigned an action, which determines how it's populated from the CLI # string. # diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index db5cb08b7..6440f4d52 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -13,7 +13,6 @@ class ParserDefinition: """Each parser contains a list of arguments and optionally a subparser.""" - # Track a list of (argument def, action) pairs. args: List[_arguments.ArgumentDefinition] nested_dataclass_field_names: List[str] subparsers: Optional["SubparsersDefinition"] diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index b3d714417..5a9eef49a 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -1,6 +1,6 @@ import dataclasses import enum -from typing import Generic, List, Type, TypeVar, Union +from typing import Generic, List, Tuple, Type, TypeVar, Union import pytest @@ -16,6 +16,26 @@ def _check_serialization_identity(cls: Type[T], instance: T) -> None: ScalarType = TypeVar("ScalarType") +def test_tuple_generic_variable(): + @dataclasses.dataclass + class TupleGenericVariable(Generic[ScalarType]): + xyz: Tuple[ScalarType, ...] + + assert dcargs.parse( + TupleGenericVariable[int], args=["--xyz", "1", "2", "3"] + ) == TupleGenericVariable((1, 2, 3)) + + +def test_tuple_generic_fixed(): + @dataclasses.dataclass + class TupleGenericFixed(Generic[ScalarType]): + xyz: Tuple[ScalarType, ScalarType, ScalarType] + + assert dcargs.parse( + TupleGenericFixed[int], args=["--xyz", "1", "2", "3"] + ) == TupleGenericFixed((1, 2, 3)) + + class CoordinateFrame(enum.Enum): WORLD = enum.auto() CAMERA = enum.auto() From 7692ab3cd142f659d27ba97ee3fe89fc868628f2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 19 Jan 2022 02:10:10 -0800 Subject: [PATCH 022/491] Nits, version bump --- dcargs/_arguments.py | 18 ++++++++++++------ dcargs/_construction.py | 37 +++++++------------------------------ dcargs/_parse.py | 25 +++++++++---------------- dcargs/_parsers.py | 34 +++++++++++++++++----------------- dcargs/_strings.py | 2 +- setup.py | 2 +- 6 files changed, 47 insertions(+), 71 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index d9b2cdc69..1659e6372 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -18,14 +18,13 @@ from typing_extensions import Final, Literal, _AnnotatedAlias, get_args, get_origin -from . import _construction, _docstrings, _strings +from . import _docstrings, _strings - -def _no_op_action(x): - return x +T = TypeVar("T") -T = TypeVar("T") +def _no_op_action(x: T) -> T: + return x @dataclasses.dataclass(frozen=True) @@ -42,7 +41,14 @@ class ArgumentDefinition: # Action that is called on parsed arguments. This handles conversions from strings # to our desired types. - field_action: _construction.FieldAction + # + # There are 2 options: + field_action: Union[ + # Most standard fields: these are converted from strings from the CLI. + Callable[[str], Any], + # Sequence fields! This should be used whenever argparse's `nargs` field is set. + Callable[[List[str]], Any], + ] # Fields that will be populated initially. # Important: from here on out, all fields correspond 1:1 to inputs to argparse's diff --git a/dcargs/_construction.py b/dcargs/_construction.py index d34771255..5f5d96d95 100644 --- a/dcargs/_construction.py +++ b/dcargs/_construction.py @@ -1,15 +1,4 @@ -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - List, - Set, - Tuple, - Type, - TypeVar, - Union, -) +from typing import TYPE_CHECKING, Any, Dict, Set, Tuple, Type, TypeVar from typing_extensions import get_args @@ -21,20 +10,9 @@ T = TypeVar("T") -# Each argument is assigned an action, which determines how it's populated from the CLI -# string. -# -# There are 2 options: -FieldAction = Union[ - # Most standard fields: these are converted from strings from the CLI. - Callable[[str], Any], - # Sequence fields! This should be used whenever argparse's `nargs` field is set. - Callable[[List[str]], Any], -] - - class FieldActionValueError(Exception): - """Exception raised when field actions fail; this is caused by""" + """Exception raised when field actions fail; this typically means that values from + the CLI are invalid.""" DataclassType = TypeVar("DataclassType") @@ -42,7 +20,7 @@ class FieldActionValueError(Exception): def construct_dataclass( cls: Type[DataclassType], - parser_definition: "_parsers.ParserDefinition", + parser_definition: "_parsers.ParserSpecification", value_from_arg: Dict[str, Any], field_name_prefix: str = "", ) -> Tuple[DataclassType, Set[str]]: @@ -84,13 +62,12 @@ def get_value_from_arg(arg: str) -> Any: ) if prefixed_field_name in arg_from_prefixed_field_name: - # Callable actions. Used for tuples, lists, sets, etc. + # Standard arguments. arg = arg_from_prefixed_field_name[prefixed_field_name] - action = arg.field_action value = get_value_from_arg(prefixed_field_name) if value is not None: try: - value = action(value) + value = arg.field_action(value) except ValueError as e: raise FieldActionValueError( f"Parsing error for {arg.get_flag()}: {e.args[0]}" @@ -132,7 +109,7 @@ def get_value_from_arg(arg: str) -> Any: assert chosen_cls is not None value, consumed_keywords_child = construct_dataclass( chosen_cls, - parser_definition.subparsers.parsers[subparser_name], + parser_definition.subparsers.parser_from_name[subparser_name], value_from_arg, ) consumed_keywords |= consumed_keywords_child diff --git a/dcargs/_parse.py b/dcargs/_parse.py index 84ff91b6a..959f96cf7 100644 --- a/dcargs/_parse.py +++ b/dcargs/_parse.py @@ -14,40 +14,33 @@ def parse( ) -> DataclassType: """Populate a dataclass via CLI args.""" - if description is None: - description = "" - - parser_definition = _parsers.ParserDefinition.from_dataclass( + # Map a dataclass to the relevant CLI arguments + subparsers. + parser_definition = _parsers.ParserSpecification.from_dataclass( cls, parent_dataclasses=set(), # Used for recursive calls. parent_type_from_typevar=None, # Used for recursive calls. default_instance=None, # Overrides for default values. This could also be exposed. ) + # Parse using argparse! parser = argparse.ArgumentParser( - description=_strings.dedent(description), + description="" if description is None else _strings.dedent(description), formatter_class=argparse.RawTextHelpFormatter, ) parser_definition.apply(parser) - - namespace = parser.parse_args(args) - - value_from_arg = vars(namespace) + value_from_arg = vars(parser.parse_args(args=args)) try: + # Attempt to construct a dataclass from whatever was passed in. out, consumed_keywords = _construction.construct_dataclass( - cls, - parser_definition, - value_from_arg, + cls, parser_definition, value_from_arg ) except _construction.FieldActionValueError as e: + # Emulate argparse's error behavior when invalid arguments are passed in. parser.print_usage() print() print(e.args[0]) raise SystemExit() - assert ( - consumed_keywords == value_from_arg.keys() - ), "Not all arguments were consumed!" - + assert consumed_keywords == value_from_arg.keys() return out diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 6440f4d52..7fc349dc6 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -10,12 +10,12 @@ @dataclasses.dataclass -class ParserDefinition: - """Each parser contains a list of arguments and optionally a subparser.""" +class ParserSpecification: + """Each parser contains a list of arguments and optionally some subparsers.""" args: List[_arguments.ArgumentDefinition] nested_dataclass_field_names: List[str] - subparsers: Optional["SubparsersDefinition"] + subparsers: Optional["SubparsersSpecification"] def apply(self, parser: argparse.ArgumentParser) -> None: """Create defined arguments and subparsers.""" @@ -33,13 +33,13 @@ def apply(self, parser: argparse.ArgumentParser) -> None: # Add subparsers. if self.subparsers is not None: - subparsers = parser.add_subparsers( + argparse_subparsers = parser.add_subparsers( dest=_strings.SUBPARSER_DEST_FMT.format(name=self.subparsers.name), description=self.subparsers.description, required=self.subparsers.required, ) - for name, subparser_def in self.subparsers.parsers.items(): - subparser = subparsers.add_parser(name) + for name, subparser_def in self.subparsers.parser_from_name.items(): + subparser = argparse_subparsers.add_parser(name) subparser_def.apply(subparser) @staticmethod @@ -48,7 +48,7 @@ def from_dataclass( parent_dataclasses: Optional[Set[Type]], parent_type_from_typevar: Optional[Dict[TypeVar, Type]], default_instance: Optional[T], - ) -> "ParserDefinition": + ) -> "ParserSpecification": """Create a parser definition from a dataclass.""" if parent_dataclasses is None: @@ -116,7 +116,7 @@ def from_dataclass( ) args.append(arg) - return ParserDefinition( + return ParserSpecification( args=args, nested_dataclass_field_names=nested_dataclass_field_names, subparsers=subparsers, @@ -124,12 +124,12 @@ def from_dataclass( @dataclasses.dataclass -class SubparsersDefinition: - """Structure for containing subparsers. Each subparser is a parser with a name.""" +class SubparsersSpecification: + """Structure for defining subparsers. Each subparser is a parser with a name.""" name: str description: Optional[str] - parsers: Dict[str, ParserDefinition] + parser_from_name: Dict[str, ParserSpecification] required: bool @@ -146,7 +146,7 @@ class _NestedDataclassHandler: def handle_unions_over_dataclasses( self, - ) -> Optional["SubparsersDefinition"]: + ) -> Optional["SubparsersSpecification"]: """Handle unions over dataclasses, which are converted to subparsers.. Returns `None` if not applicable.""" @@ -171,21 +171,21 @@ def handle_unions_over_dataclasses( self.field.default == dataclasses.MISSING ), "Default dataclass value not yet supported for subparser definitions" - parsers: Dict[str, ParserDefinition] = {} + parser_from_name: Dict[str, ParserSpecification] = {} for option in options_no_none: subparser_name = _strings.subparser_name_from_type(option) - parsers[subparser_name] = ParserDefinition.from_dataclass( + parser_from_name[subparser_name] = ParserSpecification.from_dataclass( option, self.parent_dataclasses, parent_type_from_typevar=self.type_from_typevar, default_instance=None, ) - subparsers = SubparsersDefinition( + subparsers = SubparsersSpecification( name=self.field.name, description=_docstrings.get_field_docstring(self.cls, self.field.name), - parsers=parsers, + parser_from_name=parser_from_name, required=(options == options_no_none), # Not required if no options. ) @@ -211,7 +211,7 @@ def handle_nested_dataclasses( elif self.field.default is not dataclasses.MISSING: default = self.field.default - child_definition = ParserDefinition.from_dataclass( + child_definition = ParserSpecification.from_dataclass( field_type, self.parent_dataclasses, parent_type_from_typevar=self.type_from_typevar, diff --git a/dcargs/_strings.py b/dcargs/_strings.py index cd1799c7e..7df37e249 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -47,4 +47,4 @@ def bool_from_string(text: str) -> bool: elif text in ("false", "0"): return False else: - raise ValueError(f"Boolean value expected, but got {text}.") + raise ValueError(f"Boolean (True/False or 1/0) expected, but got {text}.") diff --git a/setup.py b/setup.py index e1e9ef99c..47b12eb91 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.12", + version="0.0.13", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", From 37480d1c424cc784073c8a4c9172f76967089f16 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 21 Jan 2022 01:31:19 -0800 Subject: [PATCH 023/491] Fix metavars for generics, more refactor + tests --- dcargs/_arguments.py | 162 ++++++++++++----------- setup.py | 2 +- tests/test_docstrings.py | 47 ++++++- tests/test_generics_and_serialization.py | 10 +- 4 files changed, 136 insertions(+), 85 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 1659e6372..bbe41771f 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -23,8 +23,28 @@ T = TypeVar("T") -def _no_op_action(x: T) -> T: - return x +def instance_from_string(typ: Type, arg: str) -> T: + """Given a type and and a string from the command-line, reconstruct an object. Not + intended to deal with containers; these are handled in the argument + transformations. + + This is intended to replace all calls to `type(string)`, which can cause unexpected + behavior. As an example, note that the following argparse code will always print + `True`, because `bool("True") == bool("False") == bool("0") == True`. + ``` + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--flag", type=bool) + + print(parser.parse_args().flag) + ``` + """ + assert len(get_args(typ)) == 0, f"Type {typ} cannot be instantiated." + if typ is bool: + return _strings.bool_from_string(arg) # type: ignore + else: + return typ(arg) # type: ignore @dataclasses.dataclass(frozen=True) @@ -42,12 +62,16 @@ class ArgumentDefinition: # Action that is called on parsed arguments. This handles conversions from strings # to our desired types. # - # There are 2 options: + # There are 3 options: field_action: Union[ # Most standard fields: these are converted from strings from the CLI. Callable[[str], Any], # Sequence fields! This should be used whenever argparse's `nargs` field is set. Callable[[List[str]], Any], + # Special case: the only time that argparse doesn't give us a string is when the + # argument action is set to `store_true` or `store_false`. In this case, we get + # a bool directly, and the field action can be a no-op. + Callable[[bool], bool], ] # Fields that will be populated initially. @@ -72,17 +96,18 @@ def add_argument( """Add a defined argument to a parser.""" kwargs = {k: v for k, v in vars(self).items() if v is not None} + # Apply prefix for nested dataclasses. + if "dest" in kwargs: + kwargs["dest"] = self.prefix + kwargs["dest"] + # Important: as far as argparse is concerned, all inputs are strings. + # # Conversions from strings to our desired types happen in the "field action"; # this is a bit more flexible, and lets us handle more complex types like enums # and multi-type tuples. if "type" in kwargs: kwargs["type"] = str - # Don't pass field action into argparse. - if "dest" in kwargs: - kwargs["dest"] = self.prefix + kwargs["dest"] - kwargs.pop("field") kwargs.pop("parent_class") kwargs.pop("prefix") @@ -93,6 +118,7 @@ def add_argument( parser.add_argument(self.get_flag(), **kwargs) def get_flag(self) -> str: + """Get --flag representation, with a prefix applied for nested dataclasses.""" return "--" + (self.prefix + self.name).replace("_", "-") @staticmethod @@ -107,12 +133,17 @@ def make_from_field( assert field.init, "Field must be in class constructor" + # The default field action: this converts a string from argparse to the desired + # type of the argument. + def default_field_action(x: str) -> Any: + return instance_from_string(cast(Type, arg.type), x) + # Create initial argument. arg = ArgumentDefinition( prefix="", field=field, parent_class=parent_class, - field_action=_no_op_action, + field_action=default_field_action, name=field.name, type=field.type, default=default_override, @@ -120,34 +151,7 @@ def make_from_field( # Propagate argument through transforms until stable. prev_arg = arg - - def instance_from_string(typ: Type[T], arg: str) -> T: - """Given a type and and a string from the command-line, reconstruct an object. Not - intended to deal with containers; these are handled in the argument - transformations. - - This is intended to replace all calls to `type(string)`, which can cause unexpected - behavior. As an example, note that the following argparse code will always print - `True`, because `bool("True") == bool("False") == bool("0") == True`. - ``` - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--flag", type=bool) - - print(parser.parse_args().flag) - ``` - """ - if typ in type_from_typevar: - # The Type vs TypeVar annotations could be cleaned up... - typ = type_from_typevar[typ] # type: ignore - - if typ is bool: - return _strings.bool_from_string(arg) # type: ignore - else: - return typ(arg) # type: ignore - - argument_transforms = _get_argument_transforms(instance_from_string) + argument_transforms = _get_argument_transforms(type_from_typevar) while True: for transform in argument_transforms: # type: ignore # Apply transform. @@ -158,27 +162,28 @@ def instance_from_string(typ: Type[T], arg: str) -> T: break prev_arg = arg - if arg.field_action is _no_op_action and arg.type is not None: - cast_type = cast(Type, arg.type) - arg = dataclasses.replace( - arg, - field_action=lambda x: instance_from_string( - cast_type, - x, - ), - ) - elif arg.field_action is _no_op_action: - assert arg.action in ("store_true", "store_false") - return arg def _get_argument_transforms( - instance_from_string: Callable[[Type[T], str], T] + type_from_typevar: Dict[TypeVar, Type] ) -> List[Callable[[ArgumentDefinition], ArgumentDefinition]]: """Get a list of argument transformations.""" - def unwrap_final(arg: ArgumentDefinition) -> ArgumentDefinition: + def resolve_typevars(typ: Union[Type, TypeVar]) -> Type: + return type_from_typevar.get(cast(TypeVar, typ), cast(Type, typ)) + + # All transforms should start with `transform_`. + + def transform_resolve_arg_typevars(arg: ArgumentDefinition) -> ArgumentDefinition: + if arg.type is not None: + return dataclasses.replace( + arg, + type=resolve_typevars(arg.type), + ) + return arg + + def transform_unwrap_final(arg: ArgumentDefinition) -> ArgumentDefinition: """Treat Final[T] as just T.""" if get_origin(arg.type) is Final: (typ,) = get_args(arg.type) @@ -189,7 +194,7 @@ def unwrap_final(arg: ArgumentDefinition) -> ArgumentDefinition: else: return arg - def unwrap_annotated(arg: ArgumentDefinition) -> ArgumentDefinition: + def transform_unwrap_annotated(arg: ArgumentDefinition) -> ArgumentDefinition: """Treat Annotated[T, annotation] as just T.""" if hasattr(arg.type, "__class__") and arg.type.__class__ == _AnnotatedAlias: typ = get_origin(arg.type) @@ -200,7 +205,7 @@ def unwrap_annotated(arg: ArgumentDefinition) -> ArgumentDefinition: else: return arg - def handle_optionals(arg: ArgumentDefinition) -> ArgumentDefinition: + def transform_handle_optionals(arg: ArgumentDefinition) -> ArgumentDefinition: """Transform for handling Optional[T] types. Sets default to None and marks arg as not required.""" if get_origin(arg.type) is Union: @@ -218,7 +223,7 @@ def handle_optionals(arg: ArgumentDefinition) -> ArgumentDefinition: else: return arg - def populate_defaults(arg: ArgumentDefinition) -> ArgumentDefinition: + def transform_populate_defaults(arg: ArgumentDefinition) -> ArgumentDefinition: """Populate default values.""" if arg.default is not None: # Skip if another handler has already populated the default. @@ -238,7 +243,7 @@ def populate_defaults(arg: ArgumentDefinition) -> ArgumentDefinition: return dataclasses.replace(arg, default=default, required=required) - def bool_flags(arg: ArgumentDefinition) -> ArgumentDefinition: + def transform_bool_flags(arg: ArgumentDefinition) -> ArgumentDefinition: """For booleans, we use a `store_true` action.""" if arg.type != bool: return arg @@ -272,6 +277,7 @@ def bool_flags(arg: ArgumentDefinition) -> ArgumentDefinition: arg, action="store_true", type=None, + field_action=lambda x: x, # argparse will directly give us a bool! ) elif arg.default is True: return dataclasses.replace( @@ -280,11 +286,12 @@ def bool_flags(arg: ArgumentDefinition) -> ArgumentDefinition: name="no_" + arg.name, action="store_false", type=None, + field_action=lambda x: x, # argparse will directly give us a bool! ) else: assert False, "Invalid default" - def nargs_from_sequences_lists_and_sets( + def transform_nargs_from_sequences_lists_and_sets( arg: ArgumentDefinition, ) -> ArgumentDefinition: """Transform for handling Sequence[T] and list types.""" @@ -293,7 +300,8 @@ def nargs_from_sequences_lists_and_sets( list, # different from typing.List! set, # different from typing.Set! ): - (typ,) = get_args(arg.type) + assert arg.nargs is None, "Sequence types cannot be nested." + (typ,) = map(resolve_typevars, get_args(arg.type)) container_type = get_origin(arg.type) if container_type is collections.abc.Sequence: container_type = list @@ -312,16 +320,17 @@ def nargs_from_sequences_lists_and_sets( else: return arg - def nargs_from_tuples(arg: ArgumentDefinition) -> ArgumentDefinition: + def transform_nargs_from_tuples(arg: ArgumentDefinition) -> ArgumentDefinition: """Transform for handling Tuple[T, T, ...] types.""" if arg.nargs is None and get_origin(arg.type) is tuple: - types = get_args(arg.type) - typeset = set(types) - typeset_no_ellipsis = typeset - {Ellipsis} + assert arg.nargs is None, "Sequence types cannot be nested." + types = tuple(map(resolve_typevars, get_args(arg.type))) + typeset = set(types) # Note that sets are unordered. + typeset_no_ellipsis = typeset - {Ellipsis} # type: ignore if typeset_no_ellipsis != typeset: - # Ellipsis: variable argument counts + # Ellipsis: variable argument counts. assert ( len(typeset_no_ellipsis) == 1 ), "If ellipsis is used, tuples must contain only one type." @@ -339,13 +348,13 @@ def nargs_from_tuples(arg: ArgumentDefinition) -> ArgumentDefinition: ), ) else: - # Tuples with more than one type + # Tuples with more than one type. assert arg.metavar is None return dataclasses.replace( arg, nargs=len(types), - type=str, # Types will be converted in the dataclass reconstruction step. + type=str, # Types are converted in the field action. metavar=tuple( t.__name__.upper() if hasattr(t, "__name__") else "X" for t in types @@ -359,7 +368,7 @@ def nargs_from_tuples(arg: ArgumentDefinition) -> ArgumentDefinition: else: return arg - def choices_from_literals(arg: ArgumentDefinition) -> ArgumentDefinition: + def transform_choices_from_literals(arg: ArgumentDefinition) -> ArgumentDefinition: """For literal types, set choices.""" if get_origin(arg.type) is Literal: choices = get_args(arg.type) @@ -374,7 +383,7 @@ def choices_from_literals(arg: ArgumentDefinition) -> ArgumentDefinition: else: return arg - def enums_as_strings(arg: ArgumentDefinition) -> ArgumentDefinition: + def transform_enums_as_strings(arg: ArgumentDefinition) -> ArgumentDefinition: """For enums, use string representations.""" if isinstance(arg.type, type) and issubclass(arg.type, enum.Enum): if arg.choices is None: @@ -392,7 +401,7 @@ def enums_as_strings(arg: ArgumentDefinition) -> ArgumentDefinition: else: return arg - def generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: + def transform_generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: """Generate helptext from docstring and argument name.""" if arg.help is None: help_parts = [] @@ -416,11 +425,16 @@ def generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: else: return arg - def use_type_as_metavar(arg: ArgumentDefinition) -> ArgumentDefinition: + def transform_use_type_as_metavar(arg: ArgumentDefinition) -> ArgumentDefinition: """Communicate the argument type using the metavar.""" if ( hasattr(arg.type, "__name__") + # Don't generate metavar if target is still wrapping something, eg + # Optional[int] will have 1 arg. + and len(get_args(arg.type)) == 0 + # If choices is set, they'll be used by default. and arg.choices is None + # Don't generate metavar if one already exists. and arg.metavar is None ): return dataclasses.replace( @@ -429,16 +443,4 @@ def use_type_as_metavar(arg: ArgumentDefinition) -> ArgumentDefinition: else: return arg - return [ - unwrap_final, - unwrap_annotated, - handle_optionals, - populate_defaults, - bool_flags, - nargs_from_sequences_lists_and_sets, - nargs_from_tuples, - choices_from_literals, - enums_as_strings, - generate_helptext, - use_type_as_metavar, - ] + return [v for k, v in locals().items() if k.startswith("transform_")] diff --git a/setup.py b/setup.py index 47b12eb91..ad2e27612 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.13", + version="0.0.14", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index eea16fafd..c1d901cef 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -1,7 +1,7 @@ import contextlib import dataclasses import io -from typing import Optional, Tuple +from typing import Generic, List, Optional, Tuple, TypeVar import pytest @@ -159,3 +159,48 @@ class TupleHelptext: dcargs.parse(TupleHelptext, args=["--help"]) helptext = f.getvalue() assert "--x INT STR FLOAT\n" in helptext + + +def test_generic_helptext(): + T = TypeVar("T") + + @dataclasses.dataclass + class GenericTupleHelptext(Generic[T]): + x: T + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(GenericTupleHelptext[int], args=["--help"]) + helptext = f.getvalue() + assert "--x INT\n" in helptext + + +def test_generic_tuple_helptext(): + T = TypeVar("T") + + @dataclasses.dataclass + class GenericTupleHelptext(Generic[T]): + x: Tuple[T, T, T] + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(GenericTupleHelptext[int], args=["--help"]) + helptext = f.getvalue() + assert "--x INT INT INT\n" in helptext + + +def test_generic_list_helptext(): + T = TypeVar("T") + + @dataclasses.dataclass + class GenericTupleHelptext(Generic[T]): + x: List[T] + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(GenericTupleHelptext[int], args=["--help"]) + helptext = f.getvalue() + assert "--x INT [INT ...]\n" in helptext diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 5a9eef49a..f87923930 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -204,7 +204,7 @@ def test_generic_subparsers_in_container(): @dataclasses.dataclass class Command(Generic[T]): - a: List[int] + a: List[T] T1 = TypeVar("T1") T2 = TypeVar("T2") @@ -216,7 +216,9 @@ class Subparser(Generic[T1, T2]): parsed_instance = dcargs.parse( Subparser[Command[int], Command[float]], args="command-int --a 5 3".split(" ") ) - assert parsed_instance == Subparser(Command([5, 3])) + assert parsed_instance == Subparser(Command([5, 3])) and isinstance( + parsed_instance.command.a[0], int + ) _check_serialization_identity( Subparser[Command[int], Command[float]], parsed_instance ) @@ -224,7 +226,9 @@ class Subparser(Generic[T1, T2]): parsed_instance = dcargs.parse( Subparser[Command[int], Command[float]], args="command-float --a 7 2".split(" ") ) - assert parsed_instance == Subparser(Command([7.0, 2.0])) + assert parsed_instance == Subparser(Command([7.0, 2.0])) and isinstance( + parsed_instance.command.a[0], float + ) _check_serialization_identity( Subparser[Command[int], Command[float]], parsed_instance ) From 36ab615fbebea8068e595d8849f04efd07d8ad93 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 23 Jan 2022 21:36:50 -0800 Subject: [PATCH 024/491] Subparsers with defaults, README changes --- README.md | 243 +++++++++++++++++++--------------------- dcargs/_construction.py | 10 +- dcargs/_parse.py | 23 +++- dcargs/_parsers.py | 61 ++++++---- setup.py | 2 +- tests/test_dcargs.py | 58 +++++++++- 6 files changed, 235 insertions(+), 162 deletions(-) diff --git a/README.md b/README.md index 45cc41d56..cc48279ef 100644 --- a/README.md +++ b/README.md @@ -6,39 +6,62 @@ -* [Simple example](#simple-example) +* [Overview](#overview) +* [Core interface](#core-interface) +* [Motivation](#motivation) +* [Serialization](#serialization) * [Feature list](#feature-list) * [Comparisons to alternative tools](#comparisons-to-alternative-tools) -* [Nested example](#nested-example) -**dcargs** is a library for building dataclass-based argument parsers and -configuration objects. +### Overview -The vision: we use (potentially nested or generic) dataclasses to define -configuration objects that can be (a) populated via a CLI interface without -additional effort and (b) robustly and human-readably serialized. The result is -a statically typed replacement for not only `argparse`, but libraries likes -[YACS](https://github.com/rbgirshick/yacs) and -[ml_collections](https://github.com/google/ml_collections). +**`dcargs`** is a library for defining argument parsers and configuration +objects using standard Python dataclasses. -We expose a one-function argument parsing API: +Installation is simple: -- dcargs.parse(cls: Type[T], \*, description: - Optional[str]) -> T takes a dataclass type and instantiates it via an - argparse-style CLI interface. +``` +pip install dcargs +``` -And two functions for dataclass serialization: +### Core interface -- dcargs.from_yaml(cls: Type[T], stream: Union[str, - IO[str], bytes, IO[bytes]]) -> T and - dcargs.to_yaml(instance: T) -> str convert - between YAML-style strings and dataclass instances. In contrast to naively - dumping or loading (via pickle, PyYAML, etc), explicit type references enable - robustness against code reorganization and refactor. +Our core interface is composed of a single function, which instantiates a +dataclass from an automatically generated CLI interface: + +
+
+ + dcargs.parse(cls: Type[T], *, description: + Optional[str], args: Optional[Sequence[str]], default_instance: Optional[T]) -> T + + + +
Generate a CLI containing fields for a dataclass, and use it to create an
+instance of the class. Gracefully handles nested dataclasses, container types,
+generics, optional and default arguments, enums, and more.
 
-### Simple example
+Args:
+    cls: Dataclass type to instantiate.
+
+Keyword Args:
+    description: Description text for the parser, displayed when the --help flag is
+        passed in. Mirrors argument from `argparse.ArgumentParser()`.
+    args: If set, parse arguments from a sequence of strings instead of the
+        commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`.
+    default_instance: An instance of `T` to use for default values. Helpful for overriding fields
+        in an existing instance; if not specified, the field defaults are used instead.
+
+Returns:
+    Instantiated dataclass.
+ + +
+
+ +**Example usage** ```python import dataclasses @@ -56,13 +79,12 @@ class Args: if __name__ == "__main__": args = dcargs.parse(Args) print(args) - print() - print(dcargs.to_yaml(args)) ``` -Running `python simple.py --help` would print: +Which we can run to get: ``` +$ python simple.py --help usage: simple.py [-h] --field1 STR --field2 INT [--flag] required arguments: @@ -74,22 +96,67 @@ optional arguments: --flag A boolean flag. ``` -And, from `python simple.py --field1 string --field2 4`: - ``` +$ python simple.py --field1 string --field2 4 Args(field1='string', field2=4, flag=False) - -!dataclass:Args -field1: string -field2: 4 -flag: false ``` +Note that we support significantly more complex structures and annotations, +including nested dataclasses, container types, generics, optional and default +arguments, enums, and more. Examples of additional features can be found in the +[examples](./examples/) and [unit tests](./tests/); a +[feature list](#feature-list) is also included below. + +### Motivation + +Compared to other options, using dataclasses for configuration is: + +- **Low-effort.** Type annotations, docstrings, and default values for dataclass + fields can be used to automatically generate argument parsers. +- **Non-invasive.** Dataclasses themselves are part of the standard Python + library; defining them requires no external dependencies and they can be + easily instantiated without `dcargs` (for example, within quick experiments in + Jupyter notebooks). +- **Modular.** Most approaches to configuration objects require a centralized + definition of all configurable fields. Hierarchically nesting dataclasses, + however, makes it easy to distribute definitions, defaults, and documentation + of configurable fields across modules or source files. A model configuration + dataclass, for example, can be co-located in its entirety with the model + implementation and dropped into any experiment configuration dataclass with an + import --- this eliminates the redundancy you typically see with the argparse + equivalent and makes the entire module easy to port across codebases. +- **Strongly typed.** Unlike dynamic configuration namespaces produced by + libraries like `argparse`, `YACS`, `abseil`, or `ml_collections`, dataclasses + are robustly supported by static type checking tools (mypy, pyright, etc), as + well as IDEs and language servers. This means code can be checked + automatically for errors and typos, and IDE-assisted autocomplete, rename, + refactor, and jump operations work out-of-the-box. + +### Serialization + +As a secondary feature, we also introduce two functions for human-readable +dataclass serialization: + +- dcargs.from_yaml(cls: Type[T], stream: Union[str, + IO[str], bytes, IO[bytes]]) -> T and + dcargs.to_yaml(instance: T) -> str convert + between YAML-style strings and dataclass instances. + +The functions attempt to strike a balance between flexibility and robustness --- +in contrast to naively dumping or loading dataclass instances (via pickle, +PyYAML, etc), explicit type references enable custom tags that are robust +against code reorganization and refactor, while a PyYAML backend enables +serialization of arbitrary Python objects. + +Particularly for cases where serialized dataclasses need to exit the Python +ecosystem, [dacite](https://github.com/konradhalas/dacite) is also a good option +(at the cost of a little bit of flexibility). + ### Feature list The parse function supports a wide range of dataclass definitions, while -automatically generating helptext from comments/docstrings. Some of the basic -features are shown in the [nesting example below](#nested-example). +automatically generating helptext from comments/docstrings. A selection of +features are shown in the [examples](./examples/). Our unit tests cover many more complex type annotations, including classes containing: @@ -125,22 +192,22 @@ containing: ### Comparisons to alternative tools -There are several alternative libraries to the parsing functionality of -`dcargs`; here's a rough summary of some of them: +There are several alternatives for the parsing functionality of `dcargs`; here's +a rough summary of some of them: -| | dataclasses | attrs | Nesting | Subparsers | Containers | Choices from literals | Docstrings as helptext | Generics | -| ----------------------------------------------------------------------------------------------- | ----------- | ----- | ------- | ---------- | ---------- | -------------------------------------------------------- | ---------------------- | -------- | -| **dcargs** | ✓ | | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| **[datargs](https://github.com/roee30/datargs)** | ✓ | ✓ | | ✓ | ✓ | ✓ | | | -| **[typed-argument-parser](https://github.com/swansonk14/typed-argument-parser)** | | | | ✓ | ✓ | ✓ | ✓ | | -| **[simple-parsing](https://github.com/lebrice/SimpleParsing)** | ✓ | | ✓ | ✓ | ✓ | [soon](https://github.com/lebrice/SimpleParsing/pull/86) | ✓ | | -| **[argparse-dataclass](https://pypi.org/project/argparse-dataclass/)** | ✓ | | | | | | | | -| **[argparse-dataclasses](https://pypi.org/project/argparse-dataclasses/)** | ✓ | | | | | | | | -| **[dataclass-cli](https://github.com/malte-soe/dataclass-cli)** | ✓ | | | | | | | | -| **[clout](https://github.com/python-clout/clout)** | | ✓ | ✓ | | | | | | -| **[hf_argparser](https://huggingface.co/transformers/_modules/transformers/hf_argparser.html)** | ✓ | | | | ✓ | | | | +| | dataclasses | attrs | Choices from literals | Generics | Docstrings as helptext | Nesting | Subparsers | Containers | +| ------------------------------------------------------------------------------------------------------------ | ----------- | ----- | -------------------------------------------------------- | -------- | ---------------------- | ------- | ---------- | ---------- | +| **dcargs** | ✓ | | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| **[datargs](https://github.com/roee30/datargs)** | ✓ | ✓ | ✓ | | | | ✓ | ✓ | +| **[typed-argument-parser](https://github.com/swansonk14/typed-argument-parser)** | | | ✓ | | ✓ | | ✓ | ✓ | +| **[simple-parsing](https://github.com/lebrice/SimpleParsing)** | ✓ | | [soon](https://github.com/lebrice/SimpleParsing/pull/86) | | ✓ | ✓ | ✓ | ✓ | +| **[argparse-dataclass](https://pypi.org/project/argparse-dataclass/)** | ✓ | | | | | | | | +| **[argparse-dataclasses](https://pypi.org/project/argparse-dataclasses/)** | ✓ | | | | | | | | +| **[dataclass-cli](https://github.com/malte-soe/dataclass-cli)** | ✓ | | | | | | | | +| **[clout](https://github.com/python-clout/clout)** | | ✓ | | | | ✓ | | | +| **[hf_argparser](https://github.com/huggingface/transformers/blob/master/src/transformers/hf_argparser.py)** | ✓ | | | | | | | ✓ | -Some other distinguishing factors that `dcargs` has put effort into: +Some other distinguishing factors that we've put effort into: - Robust handling of forward references - Support for nested containers and generics @@ -152,83 +219,3 @@ Some other distinguishing factors that `dcargs` has put effort into: dataclass definitions. (in contrast, some of the libaries above rely heavily on dataclass field metadata, or on the more extreme end inheritance+decorators to make parsing-specific dataclasses) - -### Nested example - -This code: - -```python -"""An argument parsing example. - -Note that there are multiple possible ways to document dataclass attributes, all -of which are supported by the automatic helptext generator. -""" - -import dataclasses -import enum - -import dcargs - - -class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - -@dataclasses.dataclass -class OptimizerConfig: - # Variant of SGD to use. - type: OptimizerType - - # Learning rate to use. - learning_rate: float = 3e-4 - - # Coefficient for L2 regularization. - weight_decay: float = 1e-2 - - -@dataclasses.dataclass -class ExperimentConfig: - experiment_name: str # Experiment name to use. - - optimizer: OptimizerConfig - - seed: int = 0 - """Random seed. This is helpful for making sure that our experiments are - all reproducible!""" - - -if __name__ == "__main__": - config = dcargs.parse(ExperimentConfig, description=__doc__) - print(config) -``` - -Generates the following argument parser: - -``` -$ python example.py --help -usage: example.py [-h] --experiment-name STR --optimizer.type {ADAM,SGD} [--optimizer.learning-rate FLOAT] - [--optimizer.weight-decay FLOAT] [--seed INT] - -An argument parsing example. - -Note that there are multiple possible ways to document dataclass attributes, all -of which are supported by the automatic helptext generator. - -optional arguments: - -h, --help show this help message and exit - --optimizer.learning-rate FLOAT - Learning rate to use. (default: 0.0003) - --optimizer.weight-decay FLOAT - Coefficient for L2 regularization. (default: 0.01) - --seed INT Random seed. This is helpful for making sure that our experiments are - all reproducible! (default: 0) - -required arguments: - --experiment-name STR - Experiment name to use. - --optimizer.type {ADAM,SGD} - Variant of SGD to use. -``` - -Examples of additional features can be found in our [unit tests](./tests/). diff --git a/dcargs/_construction.py b/dcargs/_construction.py index 5f5d96d95..6f6278b89 100644 --- a/dcargs/_construction.py +++ b/dcargs/_construction.py @@ -93,9 +93,13 @@ def get_value_from_arg(arg: str) -> Any: subparser_name = get_value_from_arg(subparser_dest) if subparser_name is None: # No subparser selected -- this should only happen when we do either - # Optional[Union[A, B, ...]] or Union[A, B, None]. - assert type(None) in get_args(field_type) - value = None + # Optional[Union[A, B, ...]] or Union[A, B, None], or have a + # default/default_factory set. + assert ( + type(None) in get_args(field_type) + or parser_definition.subparsers.default_instance is not None + ) + value = parser_definition.subparsers.default_instance else: options = map( lambda x: x if x not in type_from_typevar else type_from_typevar[x], diff --git a/dcargs/_parse.py b/dcargs/_parse.py index 959f96cf7..5491d28fc 100644 --- a/dcargs/_parse.py +++ b/dcargs/_parse.py @@ -11,15 +11,32 @@ def parse( *, description: Optional[str] = None, args: Optional[Sequence[str]] = None, + default_instance: Optional[DataclassType] = None, ) -> DataclassType: - """Populate a dataclass via CLI args.""" - + """Generate a CLI containing fields for a dataclass, and use it to create an + instance of the class. Gracefully handles nested dataclasses, container types, + generics, optional and default arguments, enums, and more. + + Args: + cls: Dataclass type to instantiate. + + Keyword Args: + description: Description text for the parser, displayed when the --help flag is + passed in. Mirrors argument from `argparse.ArgumentParser()`. + args: If set, parse arguments from a sequence of strings instead of the + commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. + default_instance: An instance of `T` to use for default values. Helpful for overriding fields + in an existing instance; if not specified, the field defaults are used instead. + + Returns: + Instantiated dataclass. + """ # Map a dataclass to the relevant CLI arguments + subparsers. parser_definition = _parsers.ParserSpecification.from_dataclass( cls, parent_dataclasses=set(), # Used for recursive calls. parent_type_from_typevar=None, # Used for recursive calls. - default_instance=None, # Overrides for default values. This could also be exposed. + default_instance=default_instance, # Overrides for default values. ) # Parse using argparse! diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 7fc349dc6..bd70f4598 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -1,6 +1,6 @@ import argparse import dataclasses -from typing import Any, Dict, List, Optional, Set, Tuple, Type, TypeVar, Union +from typing import Any, Dict, Generic, List, Optional, Set, Tuple, Type, TypeVar, Union from typing_extensions import get_args, get_origin @@ -33,10 +33,18 @@ def apply(self, parser: argparse.ArgumentParser) -> None: # Add subparsers. if self.subparsers is not None: + title = "subcommands" + metavar = "{" + ",".join(self.subparsers.parser_from_name.keys()) + "}" + if not self.subparsers.required: + title = "optional " + title + metavar = f"[{metavar}]" + argparse_subparsers = parser.add_subparsers( dest=_strings.SUBPARSER_DEST_FMT.format(name=self.subparsers.name), description=self.subparsers.description, required=self.subparsers.required, + title=title, + metavar=metavar, ) for name, subparser_def in self.subparsers.parser_from_name.items(): subparser = argparse_subparsers.add_parser(name) @@ -77,7 +85,7 @@ def from_dataclass( if not field.init: continue - field_parser = _NestedDataclassHandler( + nested_handler = _NestedDataclassHandler( cls, field, type_from_typevar, @@ -86,7 +94,7 @@ def from_dataclass( ) # Try to create subparsers from this field. - subparsers_out = field_parser.handle_unions_over_dataclasses() + subparsers_out = nested_handler.handle_unions_over_dataclasses() if subparsers_out is not None: assert ( subparsers is None @@ -96,9 +104,8 @@ def from_dataclass( continue # Try to interpret field as a nested dataclass. - nested_out = field_parser.handle_nested_dataclasses() + nested_out = nested_handler.handle_nested_dataclasses() if nested_out is not None: - child_args, child_nested_field_names = nested_out args.extend(child_args) nested_dataclass_field_names.extend(child_nested_field_names) @@ -131,18 +138,19 @@ class SubparsersSpecification: description: Optional[str] parser_from_name: Dict[str, ParserSpecification] required: bool + default_instance: Optional[Any] @dataclasses.dataclass -class _NestedDataclassHandler: - """Helper functions for handling nested dataclasses, which are converted to either - subparsers or prefixed fields.""" +class _NestedDataclassHandler(Generic[T]): + """Helper for handling nested dataclasses, which are converted to either subparsers + or prefixed fields.""" - cls: Type + cls: Type[T] field: dataclasses.Field type_from_typevar: Dict[TypeVar, Type] parent_dataclasses: Set[Type] - default_instance: Any + default_instance: T def handle_unions_over_dataclasses( self, @@ -155,21 +163,18 @@ def handle_unions_over_dataclasses( return None # We don't use sets here to retain order of subcommands. - options = map( - lambda x: x - if x not in self.type_from_typevar - else self.type_from_typevar[x], - get_args(self.field.type), - ) + options = [ + self.type_from_typevar.get(typ, typ) for typ in get_args(self.field.type) + ] options_no_none = [o for o in options if o != type(None)] # noqa if len(options_no_none) < 2 or not all( map(_resolver.is_dataclass, options_no_none) ): return None - assert ( - self.field.default == dataclasses.MISSING - ), "Default dataclass value not yet supported for subparser definitions" + # assert ( + # self.field.default == dataclasses.MISSING + # ), "Default dataclass value not yet supported for subparser definitions" parser_from_name: Dict[str, ParserSpecification] = {} for option in options_no_none: @@ -182,15 +187,25 @@ def handle_unions_over_dataclasses( default_instance=None, ) - subparsers = SubparsersSpecification( + default_instance = None + if self.field.default is not dataclasses.MISSING: + default_instance = self.field.default + elif self.field.default_factory is not dataclasses.MISSING: + default_instance = self.field.default_factory() + + return SubparsersSpecification( name=self.field.name, + # If we wanted, we could add information about the default instance + # automatically, as is done for normal fields. But for now we just rely on + # the user to include it in the docstring. description=_docstrings.get_field_docstring(self.cls, self.field.name), parser_from_name=parser_from_name, - required=(options == options_no_none), # Not required if no options. + # Required if: type hint is not Optional[], or a default instance is + # provided. + required=(options == options_no_none) and default_instance is None, + default_instance=default_instance, ) - return subparsers - def handle_nested_dataclasses( self, ) -> Optional[Tuple[List[_arguments.ArgumentDefinition], List[str]]]: diff --git a/setup.py b/setup.py index ad2e27612..a7bb2bca7 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.14", + version="0.0.15", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index fca079216..c6a5b094d 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -108,6 +108,14 @@ class A: assert dcargs.parse(A, args=[]) == A() +def test_default_factory(): + @dataclasses.dataclass + class A: + x: int = dataclasses.field(default_factory=lambda: 5) + + assert dcargs.parse(A, args=[]) == A() + + def test_optional(): @dataclasses.dataclass class A: @@ -267,9 +275,45 @@ class Subparser: ) == Subparser(x=1, bc=SMTPServer(z=3)) with pytest.raises(SystemExit): - dcargs.parse(Subparser, args=["--x", "1", "b", "--z", "3"]) + # Missing subcommand. + dcargs.parse(Subparser, args=["--x", "1"]) + with pytest.raises(SystemExit): + # Wrong field. + dcargs.parse(Subparser, args=["--x", "1", "http-server", "--z", "3"]) + with pytest.raises(SystemExit): + # Wrong field. + dcargs.parse(Subparser, args=["--x", "1", "smtp-server", "--y", "3"]) + + +def test_subparser_with_default(): + @dataclasses.dataclass + class DefaultHTTPServer: + y: int + + @dataclasses.dataclass + class DefaultSMTPServer: + z: int + + @dataclasses.dataclass + class DefaultSubparser: + x: int + bc: Union[DefaultHTTPServer, DefaultSMTPServer] = DefaultHTTPServer(5) + + assert ( + dcargs.parse( + DefaultSubparser, args=["--x", "1", "default-http-server", "--y", "5"] + ) + == dcargs.parse(DefaultSubparser, args=["--x", "1"]) + == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=5)) + ) + assert dcargs.parse( + DefaultSubparser, args=["--x", "1", "default-smtp-server", "--z", "3"] + ) == DefaultSubparser(x=1, bc=DefaultSMTPServer(z=3)) + + with pytest.raises(SystemExit): + dcargs.parse(DefaultSubparser, args=["--x", "1", "b", "--z", "3"]) with pytest.raises(SystemExit): - dcargs.parse(Subparser, args=["--x", "1", "c", "--y", "3"]) + dcargs.parse(DefaultSubparser, args=["--x", "1", "c", "--y", "3"]) def test_optional_subparser(): @@ -297,9 +341,15 @@ class OptionalSubparser: ) with pytest.raises(SystemExit): - dcargs.parse(OptionalSubparser, args=["--x", "1", "B", "--z", "3"]) + # Wrong field. + dcargs.parse( + OptionalSubparser, args=["--x", "1", "optional-http-server", "--z", "3"] + ) with pytest.raises(SystemExit): - dcargs.parse(OptionalSubparser, args=["--x", "1", "C", "--y", "3"]) + # Wrong field. + dcargs.parse( + OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--y", "3"] + ) def test_parse_empty_description(): From 8a29b36c699da0c2cd94a05db749470415cd4225 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 23 Jan 2022 21:41:51 -0800 Subject: [PATCH 025/491] Improve literal metavar ordering --- README.md | 16 ++++++++++------ dcargs/_arguments.py | 3 ++- tests/test_docstrings.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index cc48279ef..0bb64d5f0 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,11 @@ Returns: **Example usage** +If you're familiar with dataclasses, writing a script with `dcargs.parse()` is +simple! + ```python +# examples/simple.py import dataclasses import dcargs @@ -81,7 +85,7 @@ if __name__ == "__main__": print(args) ``` -Which we can run to get: +We can run this to get: ``` $ python simple.py --help @@ -101,7 +105,7 @@ $ python simple.py --field1 string --field2 4 Args(field1='string', field2=4, flag=False) ``` -Note that we support significantly more complex structures and annotations, +Note that we also support significantly more complex structures and annotations, including nested dataclasses, container types, generics, optional and default arguments, enums, and more. Examples of additional features can be found in the [examples](./examples/) and [unit tests](./tests/); a @@ -111,9 +115,9 @@ arguments, enums, and more. Examples of additional features can be found in the Compared to other options, using dataclasses for configuration is: -- **Low-effort.** Type annotations, docstrings, and default values for dataclass +- **Low effort.** Type annotations, docstrings, and default values for dataclass fields can be used to automatically generate argument parsers. -- **Non-invasive.** Dataclasses themselves are part of the standard Python +- **Noninvasive.** Dataclasses themselves are part of the standard Python library; defining them requires no external dependencies and they can be easily instantiated without `dcargs` (for example, within quick experiments in Jupyter notebooks). @@ -123,7 +127,7 @@ Compared to other options, using dataclasses for configuration is: of configurable fields across modules or source files. A model configuration dataclass, for example, can be co-located in its entirety with the model implementation and dropped into any experiment configuration dataclass with an - import --- this eliminates the redundancy you typically see with the argparse + import — this eliminates the redundancy you typically see with the argparse equivalent and makes the entire module easy to port across codebases. - **Strongly typed.** Unlike dynamic configuration namespaces produced by libraries like `argparse`, `YACS`, `abseil`, or `ml_collections`, dataclasses @@ -142,7 +146,7 @@ dataclass serialization: dcargs.to_yaml(instance: T) -> str convert between YAML-style strings and dataclass instances. -The functions attempt to strike a balance between flexibility and robustness --- +The functions attempt to strike a balance between flexibility and robustness — in contrast to naively dumping or loading dataclass instances (via pickle, PyYAML, etc), explicit type references enable custom tags that are robust against code reorganization and refactor, while a PyYAML backend enables diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index bbe41771f..047047d32 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -378,7 +378,8 @@ def transform_choices_from_literals(arg: ArgumentDefinition) -> ArgumentDefiniti return dataclasses.replace( arg, type=type(next(iter(choices))), - choices=set(map(str, choices)), + # We use a list and not a set to preserve ordering. + choices=list(map(str, choices)), ) else: return arg diff --git a/tests/test_docstrings.py b/tests/test_docstrings.py index c1d901cef..f4ace342e 100644 --- a/tests/test_docstrings.py +++ b/tests/test_docstrings.py @@ -4,6 +4,7 @@ from typing import Generic, List, Optional, Tuple, TypeVar import pytest +from typing_extensions import Literal import dcargs @@ -204,3 +205,31 @@ class GenericTupleHelptext(Generic[T]): dcargs.parse(GenericTupleHelptext[int], args=["--help"]) helptext = f.getvalue() assert "--x INT [INT ...]\n" in helptext + + +def test_literal_helptext(): + @dataclasses.dataclass + class LiteralHelptext: + x: Literal[1, 2, 3] + """A number.""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(LiteralHelptext, args=["--help"]) + helptext = f.getvalue() + assert "--x {1,2,3} A number.\n" in helptext + + +def test_optional_literal_helptext(): + @dataclasses.dataclass + class OptionalLiteralHelptext: + x: Optional[Literal[1, 2, 3]] + """A number.""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(OptionalLiteralHelptext, args=["--help"]) + helptext = f.getvalue() + assert "--x {1,2,3} A number. (default: None)\n" in helptext From 4962293c9728db3c72b86e40270d91177202113f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 11 Feb 2022 11:47:20 -0800 Subject: [PATCH 026/491] Helptext + code style tweaks --- dcargs/_arguments.py | 22 +++++++++++++++++----- dcargs/_docstrings.py | 6 +++--- dcargs/_parsers.py | 6 +++--- tests/test_dcargs.py | 29 +++++++++++++---------------- 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 047047d32..1fb0d5152 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -372,12 +372,20 @@ def transform_choices_from_literals(arg: ArgumentDefinition) -> ArgumentDefiniti """For literal types, set choices.""" if get_origin(arg.type) is Literal: choices = get_args(arg.type) - assert ( - len(set(map(type, choices))) == 1 + typ = type(next(iter(choices))) + + assert typ not in ( + list, + tuple, + set, + ), "Containers not supported in literals." + assert all( + map(lambda c: type(c) == typ, choices) ), "All choices in literal must have the same type!" + return dataclasses.replace( arg, - type=type(next(iter(choices))), + type=typ, # We use a list and not a set to preserve ordering. choices=list(map(str, choices)), ) @@ -388,9 +396,13 @@ def transform_enums_as_strings(arg: ArgumentDefinition) -> ArgumentDefinition: """For enums, use string representations.""" if isinstance(arg.type, type) and issubclass(arg.type, enum.Enum): if arg.choices is None: - choices = set(x.name for x in arg.type) + # We use a list and not a set to preserve ordering. + choices = list(x.name for x in arg.type) else: - choices = set(x.name for x in arg.choices) + # `arg.choices` is set; this occurs when we have enums in a literal + # type. + choices = list(x.name for x in arg.choices) + assert len(choices) == len(set(choices)) return dataclasses.replace( arg, diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 7757991a3..ef4a9491c 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -10,21 +10,21 @@ from . import _strings -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class _Token: token_type: int content: str line_number: int -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class _FieldData: index: int line_number: int prev_field_line_number: int -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class _ClassTokenization: tokens: List[_Token] tokens_from_line: Dict[int, List[_Token]] diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index bd70f4598..b8e142ea4 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -9,7 +9,7 @@ T = TypeVar("T") -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class ParserSpecification: """Each parser contains a list of arguments and optionally some subparsers.""" @@ -130,7 +130,7 @@ def from_dataclass( ) -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class SubparsersSpecification: """Structure for defining subparsers. Each subparser is a parser with a name.""" @@ -141,7 +141,7 @@ class SubparsersSpecification: default_instance: Optional[Any] -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class _NestedDataclassHandler(Generic[T]): """Helper for handling nested dataclasses, which are converted to either subparsers or prefixed fields.""" diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index c6a5b094d..65047919f 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -17,22 +17,19 @@ class ManyTypes: f: float p: pathlib.Path - assert ( - dcargs.parse( - ManyTypes, - args=[ - "--i", - "5", - "--s", - "5", - "--f", - "5", - "--p", - "~", - ], - ) - == ManyTypes(i=5, s="5", f=5.0, p=pathlib.Path("~")) - ) + assert dcargs.parse( + ManyTypes, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--p", + "~", + ], + ) == ManyTypes(i=5, s="5", f=5.0, p=pathlib.Path("~")) def test_required(): From 3843736aacd69db64808b501a02333d56c348573 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 13 Feb 2022 02:38:59 -0800 Subject: [PATCH 027/491] Fix literals containing enums, example --- dcargs/_arguments.py | 41 +++++++++++++++-------------------------- dcargs/_strings.py | 7 +++---- examples/literals.py | 30 ++++++++++++++++++++++++++++++ setup.py | 2 +- tests/test_dcargs.py | 28 ++++++++++++++++++++++++---- 5 files changed, 73 insertions(+), 35 deletions(-) create mode 100644 examples/literals.py diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 1fb0d5152..5390252bc 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -107,6 +107,8 @@ def add_argument( # and multi-type tuples. if "type" in kwargs: kwargs["type"] = str + if "choices" in kwargs: + kwargs["choices"] = list(map(str, kwargs["choices"])) kwargs.pop("field") kwargs.pop("parent_class") @@ -243,36 +245,20 @@ def transform_populate_defaults(arg: ArgumentDefinition) -> ArgumentDefinition: return dataclasses.replace(arg, default=default, required=required) - def transform_bool_flags(arg: ArgumentDefinition) -> ArgumentDefinition: - """For booleans, we use a `store_true` action.""" - if arg.type != bool: + def transform_booleans(arg: ArgumentDefinition) -> ArgumentDefinition: + """Set choices or actions for booleans.""" + if arg.type != bool or arg.choices is not None: return arg - # Populate helptext for boolean flags => don't show default value, which can be - # confusing. - docstring_help = _docstrings.get_field_docstring( - arg.parent_class, arg.field.name - ) - if docstring_help is not None: - # Note that the percent symbol needs some extra handling in argparse. - # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string - docstring_help = docstring_help.replace("%", "%%") - arg = dataclasses.replace( - arg, - help=docstring_help, - ) - else: - arg = dataclasses.replace( - arg, - help="", - ) - if arg.default is None: + # If no default is passed in, the user must explicitly choose between `True` + # and `False`. return dataclasses.replace( arg, - metavar="{True,False}", + choices=(True, False), ) elif arg.default is False: + # Default `False` => --flag passed in flips to `True`. return dataclasses.replace( arg, action="store_true", @@ -280,6 +266,7 @@ def transform_bool_flags(arg: ArgumentDefinition) -> ArgumentDefinition: field_action=lambda x: x, # argparse will directly give us a bool! ) elif arg.default is True: + # Default `True` => --no-flag passed in flips to `False`. return dataclasses.replace( arg, dest=arg.name, @@ -386,8 +373,7 @@ def transform_choices_from_literals(arg: ArgumentDefinition) -> ArgumentDefiniti return dataclasses.replace( arg, type=typ, - # We use a list and not a set to preserve ordering. - choices=list(map(str, choices)), + choices=choices, ) else: return arg @@ -427,7 +413,10 @@ def transform_generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: docstring_help = docstring_help.replace("%", "%%") help_parts.append(docstring_help) - if arg.default is not None and hasattr(arg.default, "name"): + if arg.action is not None: + # Don't show defaults for boolean flags. + assert arg.action in ("store_true", "store_false") + elif arg.default is not None and hasattr(arg.default, "name"): # Special case for enums. help_parts.append(f"(default: {arg.default.name})") elif not arg.required: diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 7df37e249..e7223e145 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -41,10 +41,9 @@ def subparser_name_from_type(cls: Type) -> str: def bool_from_string(text: str) -> bool: - text = text.lower() - if text in ("true", "1"): + if text == "True": return True - elif text in ("false", "0"): + elif text == "False": return False else: - raise ValueError(f"Boolean (True/False or 1/0) expected, but got {text}.") + raise ValueError(f"Boolean (True/False) expected, but got {text}.") diff --git a/examples/literals.py b/examples/literals.py new file mode 100644 index 000000000..a34cf4d6e --- /dev/null +++ b/examples/literals.py @@ -0,0 +1,30 @@ +import dataclasses +import enum +from typing import Literal + +import dcargs + + +class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + +@dataclasses.dataclass +class Args: + enum: Color + restricted_enum: Literal[Color.RED, Color.GREEN] + integer: Literal[0, 1, 2, 3] + string: Literal["red", "green"] + + restricted_enum_with_default: Literal[Color.RED, Color.GREEN] = Color.GREEN + integer_with_default: Literal[0, 1, 2, 3] = 3 + string_with_Default: Literal["red", "green"] = "red" + + +if __name__ == "__main__": + args = dcargs.parse(Args) + print(args) + print() + print(dcargs.to_yaml(args)) diff --git a/setup.py b/setup.py index a7bb2bca7..caf536e25 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.15", + version="0.0.16", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 65047919f..74c200a33 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -51,12 +51,16 @@ class A: with pytest.raises(SystemExit): dcargs.parse(A, args=[]) - assert dcargs.parse(A, args=["--x", "1"]) == A(True) - assert dcargs.parse(A, args=["--x", "true"]) == A(True) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x", "1"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x", "true"]) assert dcargs.parse(A, args=["--x", "True"]) == A(True) - assert dcargs.parse(A, args=["--x", "0"]) == A(False) - assert dcargs.parse(A, args=["--x", "false"]) == A(False) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x", "0"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x", "false"]) assert dcargs.parse(A, args=["--x", "False"]) == A(False) @@ -151,6 +155,22 @@ class A: assert dcargs.parse(A, args=["--x", "3"]) +def test_literal_enum(): + class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + @dataclasses.dataclass + class A: + x: Literal[Color.RED, Color.GREEN] + + assert dcargs.parse(A, args=["--x", "RED"]) == A(x=Color.RED) + assert dcargs.parse(A, args=["--x", "GREEN"]) == A(x=Color.GREEN) + with pytest.raises(SystemExit): + assert dcargs.parse(A, args=["--x", "BLUE"]) + + def test_optional_literal(): @dataclasses.dataclass class A: From 06ce8a89adf7095ce1f68b7ee2770e27a427f176 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 20 Feb 2022 18:12:42 -0800 Subject: [PATCH 028/491] Create LICENSE --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..83cd2bb52 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Brent Yi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From ca9af2bd344c897be2ebf765583a7f07a84299b8 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 25 Mar 2022 11:50:41 -0700 Subject: [PATCH 029/491] Add tests for collections with defaults --- tests/test_collections.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_collections.py b/tests/test_collections.py index e6a3e192d..bbcee0ffe 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -18,6 +18,17 @@ class A: dcargs.parse(A, args=[]) +def test_tuples_with_default(): + @dataclasses.dataclass + class A: + x: Tuple[int, int, int] = (0, 1, 2) + + assert dcargs.parse(A, args=[]) == A(x=(0, 1, 2)) + assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + + def test_tuples_fixed_multitype(): @dataclasses.dataclass class A: @@ -105,6 +116,15 @@ class A: dcargs.parse(A, args=[]) +def test_lists_with_default(): + @dataclasses.dataclass + class A: + x: List[int] = dataclasses.field(default_factory=[0, 1, 2].copy) + + assert dcargs.parse(A, args=[]) == A(x=[0, 1, 2]) + assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + + def test_lists_bool(): @dataclasses.dataclass class A: @@ -131,6 +151,17 @@ class A: dcargs.parse(A, args=[]) +def test_sets_with_default(): + @dataclasses.dataclass + class A: + x: Set[int] = dataclasses.field(default_factory={0, 1, 2}.copy) + + assert dcargs.parse(A, args=[]) == A(x={0, 1, 2}) + assert dcargs.parse(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + + def test_optional_sequences(): @dataclasses.dataclass class A: From 2fc51d9d053dd1f0f7a35012f31b389a12061c30 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 25 Mar 2022 12:23:31 -0700 Subject: [PATCH 030/491] Fix faulty enum check during helptext generation --- dcargs/_arguments.py | 2 +- setup.py | 2 +- .../{test_docstrings.py => test_helptext.py} | 22 +++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) rename tests/{test_docstrings.py => test_helptext.py} (90%) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 5390252bc..a01577b70 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -416,7 +416,7 @@ def transform_generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: if arg.action is not None: # Don't show defaults for boolean flags. assert arg.action in ("store_true", "store_false") - elif arg.default is not None and hasattr(arg.default, "name"): + elif arg.default is not None and isinstance(arg.default, enum.Enum): # Special case for enums. help_parts.append(f"(default: {arg.default.name})") elif not arg.required: diff --git a/setup.py b/setup.py index caf536e25..c6770693f 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.16", + version="0.0.17", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_docstrings.py b/tests/test_helptext.py similarity index 90% rename from tests/test_docstrings.py rename to tests/test_helptext.py index f4ace342e..b94e5c23c 100644 --- a/tests/test_docstrings.py +++ b/tests/test_helptext.py @@ -1,6 +1,8 @@ import contextlib import dataclasses +import enum import io +import pathlib from typing import Generic, List, Optional, Tuple, TypeVar import pytest @@ -30,6 +32,26 @@ class Helptext: assert "--z INT Documentation 3 (default: 3)\n" in helptext +def test_helptext_defaults(): + class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + @dataclasses.dataclass + class HelptextWithVariousDefaults: + x: pathlib.Path = pathlib.Path("/some/path/to/a/file") + y: Color = Color.RED + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(HelptextWithVariousDefaults, args=["--help"]) + helptext = f.getvalue() + assert "--x PATH (default: /some/path/to/a/file)\n" in helptext + assert "--y {RED,GREEN,BLUE} (default: RED)\n" in helptext + + def test_multiline_helptext(): @dataclasses.dataclass class HelptextMultiline: From 52e8d7179b1f63d0c5efc777e4daab3ec59c1fd7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 31 Mar 2022 11:07:56 -0700 Subject: [PATCH 031/491] Address README comments from @kevin-thankyou-lin --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0bb64d5f0..7428f3f69 100644 --- a/README.md +++ b/README.md @@ -188,8 +188,10 @@ containing: - Nested combinations of the above: `Optional[Literal[T]]`, `Final[Optional[Sequence[T]]]`, etc - Nested dataclasses - - Simple nesting (see `OptimizerConfig` example below) - - Unions over nested dataclasses (subparsers) + - Simple nesting (see `OptimizerConfig` in + [./examples/example.py](./examples/example.py)) + - Unions over nested dataclasses (subparsers, see + [./examples/subparsers.py](./examples/subparsers.py)) - Optional unions over nested dataclasses (optional subparsers) - Generic dataclasses (including nested generics, see [./examples/generics.py](./examples/generics.py)) @@ -223,3 +225,6 @@ Some other distinguishing factors that we've put effort into: dataclass definitions. (in contrast, some of the libaries above rely heavily on dataclass field metadata, or on the more extreme end inheritance+decorators to make parsing-specific dataclasses) +- POSIX compatibility. For example, field names with underscores + (`argument_name`) are parsed with hyphens (`--argument-name`). + ([why](https://stackoverflow.com/questions/1253679/should-command-line-options-in-posix-style-operating-systems-be-underscore-style)) From 598d32c084533b9fe4094285c39559037d854db9 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 5 Apr 2022 14:35:40 -0700 Subject: [PATCH 032/491] (Needs test) Support descriptions from dataclass docstrings --- README.md | 3 ++- dcargs/_docstrings.py | 21 ++++++++++++++++++++- dcargs/_parse.py | 10 +++++++--- dcargs/_parsers.py | 11 ++++++----- examples/example.py | 6 +++++- examples/subparsers.py | 10 ++++++++++ 6 files changed, 50 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 7428f3f69..94cbc97f3 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,8 @@ Args: Keyword Args: description: Description text for the parser, displayed when the --help flag is - passed in. Mirrors argument from `argparse.ArgumentParser()`. + passed in. If not specified, the dataclass docstring is used. Mirrors argument + from `argparse.ArgumentParser()`. args: If set, parse arguments from a sequence of strings instead of the commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. default_instance: An instance of `T` to use for default values. Helpful for overriding fields diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index ef4a9491c..b2d004f7a 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -7,7 +7,7 @@ from typing_extensions import get_origin -from . import _strings +from . import _resolver, _strings @dataclasses.dataclass(frozen=True) @@ -175,3 +175,22 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: return "\n".join(comments[::-1]) return None + + +def get_dataclass_docstring(cls: Type) -> str: + """Get dataclass docstring, but only if it is hand-specified. + + Note that the `dataclasses.dataclass` will automatically populate __doc__ based on + the fields of the class if a docstring is not specified; this helper will ignore + these docstrings.""" + cls, _unused = _resolver.resolve_generic_classes(cls) + + # Docstring should never be `None` with a dataclass. + assert cls.__doc__ is not None + + # Ignore any default docstrings, as generated by `dataclasses.dataclass`. + default_doc = cls.__name__ + str(inspect.signature(cls)).replace(" -> None", "") + if cls.__doc__ == default_doc: + return "" + + return cls.__doc__ diff --git a/dcargs/_parse.py b/dcargs/_parse.py index 5491d28fc..bb864e2c0 100644 --- a/dcargs/_parse.py +++ b/dcargs/_parse.py @@ -1,7 +1,7 @@ import argparse from typing import Optional, Sequence, Type, TypeVar -from . import _construction, _parsers, _strings +from . import _construction, _docstrings, _parsers, _strings DataclassType = TypeVar("DataclassType") @@ -22,7 +22,8 @@ def parse( Keyword Args: description: Description text for the parser, displayed when the --help flag is - passed in. Mirrors argument from `argparse.ArgumentParser()`. + passed in. If not specified, the dataclass docstring is used. Mirrors argument + from `argparse.ArgumentParser()`. args: If set, parse arguments from a sequence of strings instead of the commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. default_instance: An instance of `T` to use for default values. Helpful for overriding fields @@ -39,9 +40,12 @@ def parse( default_instance=default_instance, # Overrides for default values. ) + if description is None: + description = _docstrings.get_dataclass_docstring(cls) + # Parse using argparse! parser = argparse.ArgumentParser( - description="" if description is None else _strings.dedent(description), + description=_strings.dedent(description), formatter_class=argparse.RawTextHelpFormatter, ) parser_definition.apply(parser) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index b8e142ea4..656b2a490 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -13,6 +13,7 @@ class ParserSpecification: """Each parser contains a list of arguments and optionally some subparsers.""" + cls: Type args: List[_arguments.ArgumentDefinition] nested_dataclass_field_names: List[str] subparsers: Optional["SubparsersSpecification"] @@ -47,7 +48,10 @@ def apply(self, parser: argparse.ArgumentParser) -> None: metavar=metavar, ) for name, subparser_def in self.subparsers.parser_from_name.items(): - subparser = argparse_subparsers.add_parser(name) + subparser = argparse_subparsers.add_parser( + name, + description=_docstrings.get_dataclass_docstring(subparser_def.cls), + ) subparser_def.apply(subparser) @staticmethod @@ -124,6 +128,7 @@ def from_dataclass( args.append(arg) return ParserSpecification( + cls=cls, args=args, nested_dataclass_field_names=nested_dataclass_field_names, subparsers=subparsers, @@ -172,10 +177,6 @@ def handle_unions_over_dataclasses( ): return None - # assert ( - # self.field.default == dataclasses.MISSING - # ), "Default dataclass value not yet supported for subparser definitions" - parser_from_name: Dict[str, ParserSpecification] = {} for option in options_no_none: subparser_name = _strings.subparser_name_from_type(option) diff --git a/examples/example.py b/examples/example.py index 11b355a48..5812be497 100644 --- a/examples/example.py +++ b/examples/example.py @@ -29,6 +29,10 @@ class OptimizerConfig: @dataclasses.dataclass class ExperimentConfig: + """A nested experiment configuration. Note that the argument parser description is + pulled from this docstring by default, but can also be overrided with + `dcargs.parse`'s `description=` flag.""" + experiment_name: str # Experiment name to use. optimizer: OptimizerConfig @@ -39,6 +43,6 @@ class ExperimentConfig: if __name__ == "__main__": - config = dcargs.parse(ExperimentConfig, description=__doc__) + config = dcargs.parse(ExperimentConfig) print(config) print(dcargs.to_yaml(config)) diff --git a/examples/subparsers.py b/examples/subparsers.py index aa4f1d2db..ec4c87039 100644 --- a/examples/subparsers.py +++ b/examples/subparsers.py @@ -8,16 +8,26 @@ @dataclasses.dataclass class Args: + """Example of a version control-style subcommand interface.""" + + # Subcommand to use; we support "checkout" and "commit". + # + # If desired, we also support default values for subparsers, eg + # command: Union[Checkout, Commit] = Checkout("main") command: Union[Checkout, Commit] @dataclasses.dataclass class Checkout: + """Checkout a branch.""" + branch: str @dataclasses.dataclass class Commit: + """Commit changes.""" + message: str all: bool = False From 547029814e696667ea33f65e28f606577830931a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 9 Apr 2022 12:20:51 -0700 Subject: [PATCH 033/491] Helptext tests, comment contiguity constraint --- dcargs/_docstrings.py | 6 ++++++ tests/test_helptext.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index b2d004f7a..8a3e82c66 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -160,14 +160,20 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: # Check for comment on the line before the field. comment_index = field_data.index comments: List[str] = [] + current_line_number = field_data.line_number while True: comment_index -= 1 comment_token = tokenization.tokens[comment_index] if ( + # Looking for comments! comment_token.token_type == tokenize.COMMENT + # Comments should come after the previous field. and comment_token.line_number > field_data.prev_field_line_number + # And be contiguous. + and comment_token.line_number == current_line_number - 1 ): assert comment_token.content.startswith("#") + current_line_number -= 1 comments.append(comment_token.content[1:].strip()) else: break diff --git a/tests/test_helptext.py b/tests/test_helptext.py index b94e5c23c..2eb8c8100 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -14,6 +14,8 @@ def test_helptext(): @dataclasses.dataclass class Helptext: + """This docstring should be printed as a description.""" + x: int # Documentation 1 # Documentation 2 @@ -27,6 +29,7 @@ class Helptext: with contextlib.redirect_stdout(f): dcargs.parse(Helptext, args=["--help"]) helptext = f.getvalue() + assert Helptext.__doc__ in helptext assert "--x INT Documentation 1\n" in helptext assert "--y INT Documentation 2\n" in helptext assert "--z INT Documentation 3 (default: 3)\n" in helptext @@ -57,6 +60,8 @@ def test_multiline_helptext(): class HelptextMultiline: x: int # Documentation 1 + # This comment should be ignored! + # Documentation 2 # Next line of documentation 2 y: int From a04f0881ed4022840bfd72cb0705896a2e8844df Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 9 Apr 2022 13:09:37 -0700 Subject: [PATCH 034/491] Mutable default warning, default_factory fix --- dcargs/_docstrings.py | 8 ++++---- dcargs/_parsers.py | 27 +++++++++++++++++++++++---- dcargs/_serialization.py | 14 ++++++++------ examples/example.py | 4 ++-- examples/generics.py | 6 +++--- examples/generics_alt.py | 6 +++--- examples/literals.py | 2 +- examples/subparsers.py | 6 +++--- setup.py | 2 +- tests/test_dcargs.py | 4 +++- tests/test_helptext.py | 4 ++-- tests/test_nested.py | 26 +++++++++++++++++++++----- 12 files changed, 74 insertions(+), 35 deletions(-) diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 8a3e82c66..c33b81eaa 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -102,10 +102,10 @@ def get_class_tokenization_with_field( found_field = True break - assert ( - found_field - ), "Docstring parsing error -- this usually means that there are multiple \ - dataclasses in the same file with the same name but different scopes." + assert found_field, ( + "Docstring parsing error -- this usually means that there are multiple" + " dataclasses in the same file with the same name but different scopes." + ) return tokenization diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 656b2a490..27b776a06 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -1,5 +1,6 @@ import argparse import dataclasses +import warnings from typing import Any, Dict, Generic, List, Optional, Set, Tuple, Type, TypeVar, Union from typing_extensions import get_args, get_origin @@ -191,6 +192,7 @@ def handle_unions_over_dataclasses( default_instance = None if self.field.default is not dataclasses.MISSING: default_instance = self.field.default + self._ensure_dataclass_used_as_default_is_frozen(default_instance) elif self.field.default_factory is not dataclasses.MISSING: default_instance = self.field.default_factory() @@ -221,17 +223,20 @@ def handle_nested_dataclasses( return None # Add arguments for nested dataclasses. - default = None + default_instance = None if self.default_instance is not None: - default = getattr(self.default_instance, self.field.name) + default_instance = getattr(self.default_instance, self.field.name) elif self.field.default is not dataclasses.MISSING: - default = self.field.default + default_instance = self.field.default + self._ensure_dataclass_used_as_default_is_frozen(default_instance) + elif self.field.default_factory is not dataclasses.MISSING: + default_instance = self.field.default_factory() child_definition = ParserSpecification.from_dataclass( field_type, self.parent_dataclasses, parent_type_from_typevar=self.type_from_typevar, - default_instance=default, + default_instance=default_instance, ) child_args = child_definition.args @@ -249,3 +254,17 @@ def handle_nested_dataclasses( ] return child_args, nested_dataclass_field_names + + def _ensure_dataclass_used_as_default_is_frozen( + self, default_instance: Any + ) -> None: + """Ensure that a dataclass type used directly as a default value is marked as + frozen.""" + assert dataclasses.is_dataclass(default_instance) + cls = type(default_instance) + if not cls.__dataclass_params__.frozen: + warnings.warn( + f"Mutable type {cls} is used as a default value for {self.field.name}. " + "This is dangerous! Consider using" + f" `dataclasses.field(default_factory=...)` or marking {cls} as frozen." + ) diff --git a/dcargs/_serialization.py b/dcargs/_serialization.py index 0df45d6e3..20b5d3137 100644 --- a/dcargs/_serialization.py +++ b/dcargs/_serialization.py @@ -84,9 +84,10 @@ class DataclassLoader(yaml.Loader): contained_types = list(_get_contained_special_types_from_type(cls)) contained_type_names = list(map(lambda cls: cls.__name__, contained_types)) - assert len(set(contained_type_names)) == len( - contained_type_names - ), f"Contained dataclass type names must all be unique, but got {contained_type_names}" + assert len(set(contained_type_names)) == len(contained_type_names), ( + "Contained dataclass type names must all be unique, but got" + f" {contained_type_names}" + ) loader: yaml.Loader node: yaml.Node @@ -122,9 +123,10 @@ class DataclassDumper(yaml.Dumper): contained_type_names = list(map(lambda cls: cls.__name__, contained_types)) # Note: this is currently a stricter than necessary assert. - assert len(set(contained_type_names)) == len( - contained_type_names - ), f"Contained dataclass/enum names must all be unique, but got {contained_type_names}" + assert len(set(contained_type_names)) == len(contained_type_names), ( + "Contained dataclass/enum names must all be unique, but got" + f" {contained_type_names}" + ) dumper: yaml.Dumper data: Any diff --git a/examples/example.py b/examples/example.py index 5812be497..f18a22c8a 100644 --- a/examples/example.py +++ b/examples/example.py @@ -15,7 +15,7 @@ class OptimizerType(enum.Enum): SGD = enum.auto() -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class OptimizerConfig: # Variant of SGD to use. type: OptimizerType @@ -27,7 +27,7 @@ class OptimizerConfig: weight_decay: float = 1e-2 -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class ExperimentConfig: """A nested experiment configuration. Note that the argument parser description is pulled from this docstring by default, but can also be overrided with diff --git a/examples/generics.py b/examples/generics.py index ecb22ce08..819ca359c 100644 --- a/examples/generics.py +++ b/examples/generics.py @@ -6,7 +6,7 @@ ScalarType = TypeVar("ScalarType") -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class Point3(Generic[ScalarType]): x: ScalarType y: ScalarType @@ -14,14 +14,14 @@ class Point3(Generic[ScalarType]): frame_id: str -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class Triangle(Generic[ScalarType]): a: Point3[ScalarType] b: Point3[ScalarType] c: Point3[ScalarType] -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class Args: point_continuous: Point3[float] point_discrete: Point3[int] diff --git a/examples/generics_alt.py b/examples/generics_alt.py index bfe0a6c57..77270a057 100644 --- a/examples/generics_alt.py +++ b/examples/generics_alt.py @@ -7,7 +7,7 @@ ShapeType = TypeVar("ShapeType") -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class Point3(Generic[ScalarType]): x: ScalarType y: ScalarType @@ -15,14 +15,14 @@ class Point3(Generic[ScalarType]): frame_id: str -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class Triangle: a: Point3[float] b: Point3[float] c: Point3[float] -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class Args(Generic[ShapeType]): point_continuous: Point3[float] point_discrete: Point3[int] diff --git a/examples/literals.py b/examples/literals.py index a34cf4d6e..0676e8b56 100644 --- a/examples/literals.py +++ b/examples/literals.py @@ -11,7 +11,7 @@ class Color(enum.Enum): BLUE = enum.auto() -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class Args: enum: Color restricted_enum: Literal[Color.RED, Color.GREEN] diff --git a/examples/subparsers.py b/examples/subparsers.py index ec4c87039..0e95f5d27 100644 --- a/examples/subparsers.py +++ b/examples/subparsers.py @@ -6,7 +6,7 @@ import dcargs -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class Args: """Example of a version control-style subcommand interface.""" @@ -17,14 +17,14 @@ class Args: command: Union[Checkout, Commit] -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class Checkout: """Checkout a branch.""" branch: str -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class Commit: """Commit changes.""" diff --git a/setup.py b/setup.py index c6770693f..11c96159f 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.17", + version="0.0.18", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 74c200a33..684324808 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -314,7 +314,9 @@ class DefaultSMTPServer: @dataclasses.dataclass class DefaultSubparser: x: int - bc: Union[DefaultHTTPServer, DefaultSMTPServer] = DefaultHTTPServer(5) + bc: Union[DefaultHTTPServer, DefaultSMTPServer] = dataclasses.field( + default_factory=lambda: DefaultHTTPServer(5) + ) assert ( dcargs.parse( diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 2eb8c8100..bef6935de 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -81,8 +81,8 @@ class HelptextMultiline: in helptext ) assert ( - " --z INT Documentation 3\n Next line of documentation 3 (default: 3)\n" - in helptext + " --z INT Documentation 3\n Next line of documentation 3" + " (default: 3)\n" in helptext ) diff --git a/tests/test_nested.py b/tests/test_nested.py index afbf105ba..3975ad662 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -39,11 +39,11 @@ class Nested: def test_default_nested(): - @dataclasses.dataclass + @dataclasses.dataclass(frozen=True) class B: y: int = 3 - @dataclasses.dataclass + @dataclasses.dataclass(frozen=True) class Nested: x: int b: B = B(y=5) @@ -55,15 +55,15 @@ class Nested: def test_double_default_nested(): - @dataclasses.dataclass + @dataclasses.dataclass(frozen=True) class Child: y: int - @dataclasses.dataclass + @dataclasses.dataclass(frozen=True) class Parent: c: Child - @dataclasses.dataclass + @dataclasses.dataclass(frozen=True) class Grandparent: x: int b: Parent = Parent(Child(y=5)) @@ -76,6 +76,22 @@ class Grandparent: ) +def test_default_factory_nested(): + @dataclasses.dataclass + class B: + y: int = 3 + + @dataclasses.dataclass + class Nested: + x: int + b: B = dataclasses.field(default_factory=lambda: B(y=5)) + + assert dcargs.parse(Nested, args=["--x", "1", "--b.y", "3"]) == Nested( + x=1, b=B(y=3) + ) + assert dcargs.parse(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) + + # TODO: implement this! # def test_optional_nested(): # @dataclasses.dataclass From 3e2fa7122f5c367db4fb641eb443341655b97f2b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 10 Apr 2022 00:05:20 -0700 Subject: [PATCH 035/491] Consolidate logic for handling defaults --- dcargs/_arguments.py | 28 +++------- dcargs/_parsers.py | 83 +++++++++++++++------------- examples/subparsers.py | 3 +- tests/test_dcargs.py | 103 +---------------------------------- tests/test_helptext.py | 7 ++- tests/test_nested.py | 119 ++++++++++++++++++++++++++++++++++++++++- 6 files changed, 178 insertions(+), 165 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index a01577b70..bb478557a 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -128,7 +128,7 @@ def make_from_field( parent_class: Type, field: dataclasses.Field, type_from_typevar: Dict[TypeVar, Type], - default_override: Optional[Any], + default: Optional[Any], ) -> "ArgumentDefinition": """Create an argument definition from a field. Also returns a field action, which specifies special instructions for reconstruction.""" @@ -148,7 +148,7 @@ def default_field_action(x: str) -> Any: field_action=default_field_action, name=field.name, type=field.type, - default=default_override, + default=default, ) # Propagate argument through transforms until stable. @@ -225,25 +225,13 @@ def transform_handle_optionals(arg: ArgumentDefinition) -> ArgumentDefinition: else: return arg - def transform_populate_defaults(arg: ArgumentDefinition) -> ArgumentDefinition: - """Populate default values.""" - if arg.default is not None: - # Skip if another handler has already populated the default. - return arg - - default = None - required = True - if arg.field.default is not dataclasses.MISSING: - default = arg.field.default - required = False - elif arg.field.default_factory is not dataclasses.MISSING: # type: ignore - default = arg.field.default_factory() # type: ignore - required = False - - if arg.required is not None: - required = arg.required + def transform_required(arg: ArgumentDefinition) -> ArgumentDefinition: + """Set `required=True` if a default value is set.""" - return dataclasses.replace(arg, default=default, required=required) + # Don't set if default is set, or if required flag is already set. + if arg.default is not None or arg.required is not None: + return arg + return dataclasses.replace(arg, required=True) def transform_booleans(arg: ArgumentDefinition) -> ArgumentDefinition: """Set choices or actions for booleans.""" diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 27b776a06..e1cf88a54 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -10,6 +10,44 @@ T = TypeVar("T") +def _ensure_dataclass_instance_used_as_default_is_frozen( + field: dataclasses.Field, default_instance: Any +) -> None: + """Ensure that a dataclass type used directly as a default value is marked as + frozen.""" + assert dataclasses.is_dataclass(default_instance) + cls = type(default_instance) + if not cls.__dataclass_params__.frozen: + warnings.warn( + f"Mutable type {cls} is used as a default value for `{field.name}`. This is" + " dangerous! Consider using `dataclasses.field(default_factory=...)` or" + f" marking {cls} as frozen." + ) + + +def _get_field_default( + field: dataclasses.Field, parent_default_instance: Any +) -> Optional[Any]: + """Helper for getting the default instance for a field.""" + field_default_instance = None + if field.default is not dataclasses.MISSING: + # Populate default from usual default value, or + # `dataclasses.field(default=...)`. + field_default_instance = field.default + if dataclasses.is_dataclass(field_default_instance): + _ensure_dataclass_instance_used_as_default_is_frozen( + field, field_default_instance + ) + elif field.default_factory is not dataclasses.MISSING: + # Populate default from `dataclasses.field(default_factory=...)`. + field_default_instance = field.default_factory() + + if parent_default_instance is not None: + # Populate default from explicit `default_instance` in `dcargs.parse()`. + field_default_instance = getattr(parent_default_instance, field.name) + return field_default_instance + + @dataclasses.dataclass(frozen=True) class ParserSpecification: """Each parser contains a list of arguments and optionally some subparsers.""" @@ -90,12 +128,13 @@ def from_dataclass( if not field.init: continue + field_default_instance = _get_field_default(field, default_instance) nested_handler = _NestedDataclassHandler( cls, field, type_from_typevar, parent_dataclasses, - default_instance, + field_default_instance, ) # Try to create subparsers from this field. @@ -122,9 +161,7 @@ def from_dataclass( cls, field, type_from_typevar, - default_override=getattr(default_instance, field.name) - if default_instance is not None - else None, + default=field_default_instance, ) args.append(arg) @@ -156,7 +193,7 @@ class _NestedDataclassHandler(Generic[T]): field: dataclasses.Field type_from_typevar: Dict[TypeVar, Type] parent_dataclasses: Set[Type] - default_instance: T + default_instance: Optional[T] def handle_unions_over_dataclasses( self, @@ -189,13 +226,6 @@ def handle_unions_over_dataclasses( default_instance=None, ) - default_instance = None - if self.field.default is not dataclasses.MISSING: - default_instance = self.field.default - self._ensure_dataclass_used_as_default_is_frozen(default_instance) - elif self.field.default_factory is not dataclasses.MISSING: - default_instance = self.field.default_factory() - return SubparsersSpecification( name=self.field.name, # If we wanted, we could add information about the default instance @@ -205,8 +235,8 @@ def handle_unions_over_dataclasses( parser_from_name=parser_from_name, # Required if: type hint is not Optional[], or a default instance is # provided. - required=(options == options_no_none) and default_instance is None, - default_instance=default_instance, + required=(options == options_no_none) and self.default_instance is None, + default_instance=self.default_instance, ) def handle_nested_dataclasses( @@ -223,20 +253,11 @@ def handle_nested_dataclasses( return None # Add arguments for nested dataclasses. - default_instance = None - if self.default_instance is not None: - default_instance = getattr(self.default_instance, self.field.name) - elif self.field.default is not dataclasses.MISSING: - default_instance = self.field.default - self._ensure_dataclass_used_as_default_is_frozen(default_instance) - elif self.field.default_factory is not dataclasses.MISSING: - default_instance = self.field.default_factory() - child_definition = ParserSpecification.from_dataclass( field_type, self.parent_dataclasses, parent_type_from_typevar=self.type_from_typevar, - default_instance=default_instance, + default_instance=self.default_instance, ) child_args = child_definition.args @@ -254,17 +275,3 @@ def handle_nested_dataclasses( ] return child_args, nested_dataclass_field_names - - def _ensure_dataclass_used_as_default_is_frozen( - self, default_instance: Any - ) -> None: - """Ensure that a dataclass type used directly as a default value is marked as - frozen.""" - assert dataclasses.is_dataclass(default_instance) - cls = type(default_instance) - if not cls.__dataclass_params__.frozen: - warnings.warn( - f"Mutable type {cls} is used as a default value for {self.field.name}. " - "This is dangerous! Consider using" - f" `dataclasses.field(default_factory=...)` or marking {cls} as frozen." - ) diff --git a/examples/subparsers.py b/examples/subparsers.py index 0e95f5d27..f2d8dc6f3 100644 --- a/examples/subparsers.py +++ b/examples/subparsers.py @@ -33,5 +33,6 @@ class Commit: if __name__ == "__main__": - args = dcargs.parse(Args) + # args = dcargs.parse(Args) + args = dcargs.parse(Args, default_instance=Args(command=Checkout(branch="main"))) print(args) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 684324808..723f6693f 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -1,7 +1,7 @@ import dataclasses import enum import pathlib -from typing import ClassVar, Optional, Union +from typing import ClassVar, Optional import pytest from typing_extensions import Annotated, Final, Literal # Backward compatibility. @@ -270,107 +270,6 @@ class A: # ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) -def test_subparser(): - @dataclasses.dataclass - class HTTPServer: - y: int - - @dataclasses.dataclass - class SMTPServer: - z: int - - @dataclasses.dataclass - class Subparser: - x: int - bc: Union[HTTPServer, SMTPServer] - - assert dcargs.parse( - Subparser, args=["--x", "1", "http-server", "--y", "3"] - ) == Subparser(x=1, bc=HTTPServer(y=3)) - assert dcargs.parse( - Subparser, args=["--x", "1", "smtp-server", "--z", "3"] - ) == Subparser(x=1, bc=SMTPServer(z=3)) - - with pytest.raises(SystemExit): - # Missing subcommand. - dcargs.parse(Subparser, args=["--x", "1"]) - with pytest.raises(SystemExit): - # Wrong field. - dcargs.parse(Subparser, args=["--x", "1", "http-server", "--z", "3"]) - with pytest.raises(SystemExit): - # Wrong field. - dcargs.parse(Subparser, args=["--x", "1", "smtp-server", "--y", "3"]) - - -def test_subparser_with_default(): - @dataclasses.dataclass - class DefaultHTTPServer: - y: int - - @dataclasses.dataclass - class DefaultSMTPServer: - z: int - - @dataclasses.dataclass - class DefaultSubparser: - x: int - bc: Union[DefaultHTTPServer, DefaultSMTPServer] = dataclasses.field( - default_factory=lambda: DefaultHTTPServer(5) - ) - - assert ( - dcargs.parse( - DefaultSubparser, args=["--x", "1", "default-http-server", "--y", "5"] - ) - == dcargs.parse(DefaultSubparser, args=["--x", "1"]) - == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=5)) - ) - assert dcargs.parse( - DefaultSubparser, args=["--x", "1", "default-smtp-server", "--z", "3"] - ) == DefaultSubparser(x=1, bc=DefaultSMTPServer(z=3)) - - with pytest.raises(SystemExit): - dcargs.parse(DefaultSubparser, args=["--x", "1", "b", "--z", "3"]) - with pytest.raises(SystemExit): - dcargs.parse(DefaultSubparser, args=["--x", "1", "c", "--y", "3"]) - - -def test_optional_subparser(): - @dataclasses.dataclass - class OptionalHTTPServer: - y: int - - @dataclasses.dataclass - class OptionalSMTPServer: - z: int - - @dataclasses.dataclass - class OptionalSubparser: - x: int - bc: Optional[Union[OptionalHTTPServer, OptionalSMTPServer]] - - assert dcargs.parse( - OptionalSubparser, args=["--x", "1", "optional-http-server", "--y", "3"] - ) == OptionalSubparser(x=1, bc=OptionalHTTPServer(y=3)) - assert dcargs.parse( - OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--z", "3"] - ) == OptionalSubparser(x=1, bc=OptionalSMTPServer(z=3)) - assert dcargs.parse(OptionalSubparser, args=["--x", "1"]) == OptionalSubparser( - x=1, bc=None - ) - - with pytest.raises(SystemExit): - # Wrong field. - dcargs.parse( - OptionalSubparser, args=["--x", "1", "optional-http-server", "--z", "3"] - ) - with pytest.raises(SystemExit): - # Wrong field. - dcargs.parse( - OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--y", "3"] - ) - - def test_parse_empty_description(): """If the file has no dosctring, it should be treated as an empty string.""" diff --git a/tests/test_helptext.py b/tests/test_helptext.py index bef6935de..154c1dfff 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -30,7 +30,7 @@ class Helptext: dcargs.parse(Helptext, args=["--help"]) helptext = f.getvalue() assert Helptext.__doc__ in helptext - assert "--x INT Documentation 1\n" in helptext + assert "required arguments:\n --x INT Documentation 1\n" in helptext assert "--y INT Documentation 2\n" in helptext assert "--z INT Documentation 3 (default: 3)\n" in helptext @@ -51,7 +51,10 @@ class HelptextWithVariousDefaults: with contextlib.redirect_stdout(f): dcargs.parse(HelptextWithVariousDefaults, args=["--help"]) helptext = f.getvalue() - assert "--x PATH (default: /some/path/to/a/file)\n" in helptext + assert ( + "show this help message and exit\n --x PATH (default:" + " /some/path/to/a/file)\n" in helptext + ) assert "--y {RED,GREEN,BLUE} (default: RED)\n" in helptext diff --git a/tests/test_nested.py b/tests/test_nested.py index 3975ad662..e6e48dd42 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -1,4 +1,5 @@ import dataclasses +from typing import Optional, Union import pytest @@ -32,8 +33,10 @@ class Nested: x: int b: B - assert dcargs.parse(Nested, args=["--x", "1", "--b.y", "3"]) == Nested( - x=1, b=B(y=3) + assert ( + Nested(x=1, b=B(y=3)) + == dcargs.parse(Nested, args=["--x", "1", "--b.y", "3"]) + == dcargs.parse(Nested, args=[], default_instance=Nested(x=1, b=B(y=3))) ) assert dcargs.parse(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=3)) @@ -115,3 +118,115 @@ class Nested: # assert dcargs.parse( # OptionalNested, args=["--x", "1", "--b.y", "2", "--b.z", "3"] # ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) + + +def test_subparser(): + @dataclasses.dataclass + class HTTPServer: + y: int + + @dataclasses.dataclass + class SMTPServer: + z: int + + @dataclasses.dataclass + class Subparser: + x: int + bc: Union[HTTPServer, SMTPServer] + + assert dcargs.parse( + Subparser, args=["--x", "1", "http-server", "--y", "3"] + ) == Subparser(x=1, bc=HTTPServer(y=3)) + assert dcargs.parse( + Subparser, args=["--x", "1", "smtp-server", "--z", "3"] + ) == Subparser(x=1, bc=SMTPServer(z=3)) + + with pytest.raises(SystemExit): + # Missing subcommand. + dcargs.parse(Subparser, args=["--x", "1"]) + with pytest.raises(SystemExit): + # Wrong field. + dcargs.parse(Subparser, args=["--x", "1", "http-server", "--z", "3"]) + with pytest.raises(SystemExit): + # Wrong field. + dcargs.parse(Subparser, args=["--x", "1", "smtp-server", "--y", "3"]) + + +def test_subparser_with_default(): + @dataclasses.dataclass + class DefaultHTTPServer: + y: int + + @dataclasses.dataclass + class DefaultSMTPServer: + z: int + + @dataclasses.dataclass + class DefaultSubparser: + x: int + bc: Union[DefaultHTTPServer, DefaultSMTPServer] = dataclasses.field( + default_factory=lambda: DefaultHTTPServer(5) + ) + + assert ( + dcargs.parse( + DefaultSubparser, args=["--x", "1", "default-http-server", "--y", "5"] + ) + == dcargs.parse(DefaultSubparser, args=["--x", "1"]) + == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=5)) + ) + assert dcargs.parse( + DefaultSubparser, args=["--x", "1", "default-smtp-server", "--z", "3"] + ) == DefaultSubparser(x=1, bc=DefaultSMTPServer(z=3)) + assert ( + dcargs.parse( + DefaultSubparser, args=["--x", "1", "default-http-server", "--y", "8"] + ) + == dcargs.parse( + DefaultSubparser, + args=[], + default_instance=DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)), + ) + == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)) + ) + + with pytest.raises(SystemExit): + dcargs.parse(DefaultSubparser, args=["--x", "1", "b", "--z", "3"]) + with pytest.raises(SystemExit): + dcargs.parse(DefaultSubparser, args=["--x", "1", "c", "--y", "3"]) + + +def test_optional_subparser(): + @dataclasses.dataclass + class OptionalHTTPServer: + y: int + + @dataclasses.dataclass + class OptionalSMTPServer: + z: int + + @dataclasses.dataclass + class OptionalSubparser: + x: int + bc: Optional[Union[OptionalHTTPServer, OptionalSMTPServer]] + + assert dcargs.parse( + OptionalSubparser, args=["--x", "1", "optional-http-server", "--y", "3"] + ) == OptionalSubparser(x=1, bc=OptionalHTTPServer(y=3)) + assert dcargs.parse( + OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--z", "3"] + ) == OptionalSubparser(x=1, bc=OptionalSMTPServer(z=3)) + assert dcargs.parse(OptionalSubparser, args=["--x", "1"]) == OptionalSubparser( + x=1, bc=None + ) + + with pytest.raises(SystemExit): + # Wrong field. + dcargs.parse( + OptionalSubparser, args=["--x", "1", "optional-http-server", "--z", "3"] + ) + with pytest.raises(SystemExit): + # Wrong field. + dcargs.parse( + OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--y", "3"] + ) From 6d81978e4e06a6aeff91f0ca51f0d86ab7542e11 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 21 Apr 2022 10:00:59 -0700 Subject: [PATCH 036/491] More robust redesign for argument instantiation --- dcargs/_arguments.py | 442 +++++++++----------------------------- dcargs/_construction.py | 7 +- dcargs/_docstrings.py | 2 + dcargs/_instantiators.py | 245 +++++++++++++++++++++ dcargs/_parsers.py | 38 ++-- dcargs/_resolver.py | 2 + dcargs/_serialization.py | 2 + dcargs/_strings.py | 47 +++- examples/subparsers.py | 3 +- setup.py | 2 +- tests/test_collections.py | 52 ++++- tests/test_dcargs.py | 4 +- 12 files changed, 481 insertions(+), 365 deletions(-) create mode 100644 dcargs/_instantiators.py diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index bb478557a..c51ffc697 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -1,50 +1,9 @@ import argparse -import collections.abc import dataclasses import enum -from typing import ( - Any, - Callable, - Dict, - List, - Optional, - Set, - Tuple, - Type, - TypeVar, - Union, - cast, -) - -from typing_extensions import Final, Literal, _AnnotatedAlias, get_args, get_origin - -from . import _docstrings, _strings - -T = TypeVar("T") - - -def instance_from_string(typ: Type, arg: str) -> T: - """Given a type and and a string from the command-line, reconstruct an object. Not - intended to deal with containers; these are handled in the argument - transformations. - - This is intended to replace all calls to `type(string)`, which can cause unexpected - behavior. As an example, note that the following argparse code will always print - `True`, because `bool("True") == bool("False") == bool("0") == True`. - ``` - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--flag", type=bool) - - print(parser.parse_args().flag) - ``` - """ - assert len(get_args(typ)) == 0, f"Type {typ} cannot be instantiated." - if typ is bool: - return _strings.bool_from_string(arg) # type: ignore - else: - return typ(arg) # type: ignore +from typing import Any, Dict, Optional, Set, Tuple, Type, TypeVar, Union + +from . import _docstrings, _instantiators @dataclasses.dataclass(frozen=True) @@ -61,18 +20,7 @@ class ArgumentDefinition: # Action that is called on parsed arguments. This handles conversions from strings # to our desired types. - # - # There are 3 options: - field_action: Union[ - # Most standard fields: these are converted from strings from the CLI. - Callable[[str], Any], - # Sequence fields! This should be used whenever argparse's `nargs` field is set. - Callable[[List[str]], Any], - # Special case: the only time that argparse doesn't give us a string is when the - # argument action is set to `store_true` or `store_false`. In this case, we get - # a bool directly, and the field action can be a no-op. - Callable[[bool], bool], - ] + instantiator: Optional[_instantiators.Instantiator] # Fields that will be populated initially. # Important: from here on out, all fields correspond 1:1 to inputs to argparse's @@ -113,7 +61,7 @@ def add_argument( kwargs.pop("field") kwargs.pop("parent_class") kwargs.pop("prefix") - kwargs.pop("field_action") + kwargs.pop("instantiator") kwargs.pop("name") # Note that the name must be passed in as a position argument. @@ -135,302 +83,126 @@ def make_from_field( assert field.init, "Field must be in class constructor" - # The default field action: this converts a string from argparse to the desired - # type of the argument. - def default_field_action(x: str) -> Any: - return instance_from_string(cast(Type, arg.type), x) - - # Create initial argument. arg = ArgumentDefinition( prefix="", field=field, parent_class=parent_class, - field_action=default_field_action, + instantiator=None, name=field.name, type=field.type, default=default, ) - - # Propagate argument through transforms until stable. - prev_arg = arg - argument_transforms = _get_argument_transforms(type_from_typevar) - while True: - for transform in argument_transforms: # type: ignore - # Apply transform. - arg = transform(arg) - - # Stability check. - if arg == prev_arg: - break - prev_arg = arg - + arg = _transform_required_if_default_set(arg) + arg = _transform_handle_boolean_flags(arg) + arg = _transform_recursive_instantiator_from_type(arg, type_from_typevar) + arg = _transform_generate_helptext(arg) + arg = _transform_convert_defaults_to_strings(arg) return arg -def _get_argument_transforms( - type_from_typevar: Dict[TypeVar, Type] -) -> List[Callable[[ArgumentDefinition], ArgumentDefinition]]: - """Get a list of argument transformations.""" +def _transform_required_if_default_set(arg: ArgumentDefinition) -> ArgumentDefinition: + """Set `required=True` if a default value is set.""" - def resolve_typevars(typ: Union[Type, TypeVar]) -> Type: - return type_from_typevar.get(cast(TypeVar, typ), cast(Type, typ)) + # Don't set if default is set, or if required flag is already set. + if arg.default is not None: + return dataclasses.replace(arg, required=False) + else: + return dataclasses.replace(arg, required=True) - # All transforms should start with `transform_`. - def transform_resolve_arg_typevars(arg: ArgumentDefinition) -> ArgumentDefinition: - if arg.type is not None: - return dataclasses.replace( - arg, - type=resolve_typevars(arg.type), - ) +def _transform_handle_boolean_flags(arg: ArgumentDefinition) -> ArgumentDefinition: + """""" + if arg.type is not bool: return arg - def transform_unwrap_final(arg: ArgumentDefinition) -> ArgumentDefinition: - """Treat Final[T] as just T.""" - if get_origin(arg.type) is Final: - (typ,) = get_args(arg.type) - return dataclasses.replace( - arg, - type=typ, - ) - else: - return arg - - def transform_unwrap_annotated(arg: ArgumentDefinition) -> ArgumentDefinition: - """Treat Annotated[T, annotation] as just T.""" - if hasattr(arg.type, "__class__") and arg.type.__class__ == _AnnotatedAlias: - typ = get_origin(arg.type) - return dataclasses.replace( - arg, - type=typ, - ) - else: - return arg - - def transform_handle_optionals(arg: ArgumentDefinition) -> ArgumentDefinition: - """Transform for handling Optional[T] types. Sets default to None and marks arg as - not required.""" - if get_origin(arg.type) is Union: - options = set(get_args(arg.type)) - assert ( - len(options) == 2 and type(None) in options - ), "Union must be either over dataclasses (for subparsers) or Optional" - (typ,) = options - {type(None)} - required = False - return dataclasses.replace( - arg, - type=typ, - required=required, - ) - else: - return arg - - def transform_required(arg: ArgumentDefinition) -> ArgumentDefinition: - """Set `required=True` if a default value is set.""" + if arg.default is None: + # If no default is passed in, we treat bools as a normal parameter. + return arg + elif arg.default is False: + # Default `False` => --flag passed in flips to `True`. + return dataclasses.replace( + arg, + action="store_true", + type=None, + instantiator=lambda x: x, # argparse will directly give us a bool! + ) + elif arg.default is True: + # Default `True` => --no-flag passed in flips to `False`. + return dataclasses.replace( + arg, + dest=arg.name, + name="no_" + arg.name, + action="store_false", + type=None, + instantiator=lambda x: x, # argparse will directly give us a bool! + ) + else: + assert False, "Invalid default" - # Don't set if default is set, or if required flag is already set. - if arg.default is not None or arg.required is not None: - return arg - return dataclasses.replace(arg, required=True) - def transform_booleans(arg: ArgumentDefinition) -> ArgumentDefinition: - """Set choices or actions for booleans.""" - if arg.type != bool or arg.choices is not None: - return arg - - if arg.default is None: - # If no default is passed in, the user must explicitly choose between `True` - # and `False`. - return dataclasses.replace( - arg, - choices=(True, False), - ) - elif arg.default is False: - # Default `False` => --flag passed in flips to `True`. - return dataclasses.replace( - arg, - action="store_true", - type=None, - field_action=lambda x: x, # argparse will directly give us a bool! - ) - elif arg.default is True: - # Default `True` => --no-flag passed in flips to `False`. - return dataclasses.replace( - arg, - dest=arg.name, - name="no_" + arg.name, - action="store_false", - type=None, - field_action=lambda x: x, # argparse will directly give us a bool! - ) - else: - assert False, "Invalid default" - - def transform_nargs_from_sequences_lists_and_sets( - arg: ArgumentDefinition, - ) -> ArgumentDefinition: - """Transform for handling Sequence[T] and list types.""" - if get_origin(arg.type) in ( - collections.abc.Sequence, # different from typing.Sequence! - list, # different from typing.List! - set, # different from typing.Set! - ): - assert arg.nargs is None, "Sequence types cannot be nested." - (typ,) = map(resolve_typevars, get_args(arg.type)) - container_type = get_origin(arg.type) - if container_type is collections.abc.Sequence: - container_type = list - - return dataclasses.replace( - arg, - type=typ, - # `*` is >=0 values, `+` is >=1 values - # We're going to require at least 1 value; if a user wants to accept no - # input, they can use Optional[Tuple[...]] - nargs="+", - field_action=lambda str_list: container_type( # type: ignore - instance_from_string(typ, x) for x in str_list - ), - ) - else: - return arg - - def transform_nargs_from_tuples(arg: ArgumentDefinition) -> ArgumentDefinition: - """Transform for handling Tuple[T, T, ...] types.""" - - if arg.nargs is None and get_origin(arg.type) is tuple: - assert arg.nargs is None, "Sequence types cannot be nested." - types = tuple(map(resolve_typevars, get_args(arg.type))) - typeset = set(types) # Note that sets are unordered. - typeset_no_ellipsis = typeset - {Ellipsis} # type: ignore - - if typeset_no_ellipsis != typeset: - # Ellipsis: variable argument counts. - assert ( - len(typeset_no_ellipsis) == 1 - ), "If ellipsis is used, tuples must contain only one type." - (typ,) = typeset_no_ellipsis - - return dataclasses.replace( - arg, - # `*` is >=0 values, `+` is >=1 values. - # We're going to require at least 1 value; if a user wants to accept no - # input, they can use Optional[Tuple[...]]. - nargs="+", - type=typ, - field_action=lambda str_list: tuple( - instance_from_string(typ, x) for x in str_list - ), - ) - else: - # Tuples with more than one type. - assert arg.metavar is None - - return dataclasses.replace( - arg, - nargs=len(types), - type=str, # Types are converted in the field action. - metavar=tuple( - t.__name__.upper() if hasattr(t, "__name__") else "X" - for t in types - ), - # Field action: convert lists of strings to tuples of the correct types. - field_action=lambda str_list: tuple( - instance_from_string(typ, x) for typ, x in zip(types, str_list) - ), - ) +def _transform_recursive_instantiator_from_type( + arg: ArgumentDefinition, + type_from_typevar: Dict[TypeVar, Type], +) -> ArgumentDefinition: + """The bulkiest bit: recursively analyze the type annotation and use it to determine how""" + if arg.instantiator is not None: + return arg + instantiator, metadata = _instantiators.instantiator_from_type( + arg.type, # type: ignore + type_from_typevar, + ) + return dataclasses.replace( + arg, + instantiator=instantiator, + choices=metadata.choices, + nargs=metadata.nargs, + required=(not metadata.is_optional) and arg.required, + # Ignore metavar if choices is set. + metavar=metadata.metavar if metadata.choices is None else None, + ) + + +def _transform_generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: + """Generate helptext from docstring and argument name.""" + help_parts = [] + docstring_help = _docstrings.get_field_docstring(arg.parent_class, arg.field.name) + if docstring_help is not None: + # Note that the percent symbol needs some extra handling in argparse. + # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string + docstring_help = docstring_help.replace("%", "%%") + help_parts.append(docstring_help) + + if arg.action is not None: + # Don't show defaults for boolean flags. + assert arg.action in ("store_true", "store_false") + elif arg.default is not None and isinstance(arg.default, enum.Enum): + # Special case for enums. + help_parts.append(f"(default: {arg.default.name})") + elif not arg.required: + # General case. We intentionally don't use the % template, because the default + # will be casted to a string and that can make unnecessary quotation marks + # appear in the helptext. + help_parts.append(f"(default: {arg.default})") + + return dataclasses.replace(arg, help=" ".join(help_parts)) + + +def _transform_convert_defaults_to_strings( + arg: ArgumentDefinition, +) -> ArgumentDefinition: + """Sets all default values to strings, as required as input to our instantiator + functions. Special-cased for enums.""" + + def as_str(x: Any) -> str: + if isinstance(x, enum.Enum): + return x.name else: - return arg - - def transform_choices_from_literals(arg: ArgumentDefinition) -> ArgumentDefinition: - """For literal types, set choices.""" - if get_origin(arg.type) is Literal: - choices = get_args(arg.type) - typ = type(next(iter(choices))) - - assert typ not in ( - list, - tuple, - set, - ), "Containers not supported in literals." - assert all( - map(lambda c: type(c) == typ, choices) - ), "All choices in literal must have the same type!" - - return dataclasses.replace( - arg, - type=typ, - choices=choices, - ) - else: - return arg - - def transform_enums_as_strings(arg: ArgumentDefinition) -> ArgumentDefinition: - """For enums, use string representations.""" - if isinstance(arg.type, type) and issubclass(arg.type, enum.Enum): - if arg.choices is None: - # We use a list and not a set to preserve ordering. - choices = list(x.name for x in arg.type) - else: - # `arg.choices` is set; this occurs when we have enums in a literal - # type. - choices = list(x.name for x in arg.choices) - assert len(choices) == len(set(choices)) - - return dataclasses.replace( - arg, - choices=choices, - type=str, - default=None if arg.default is None else arg.default.name, - field_action=lambda enum_name: arg.type[enum_name], # type: ignore - ) - else: - return arg - - def transform_generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: - """Generate helptext from docstring and argument name.""" - if arg.help is None: - help_parts = [] - docstring_help = _docstrings.get_field_docstring( - arg.parent_class, arg.field.name - ) - if docstring_help is not None: - # Note that the percent symbol needs some extra handling in argparse. - # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string - docstring_help = docstring_help.replace("%", "%%") - help_parts.append(docstring_help) - - if arg.action is not None: - # Don't show defaults for boolean flags. - assert arg.action in ("store_true", "store_false") - elif arg.default is not None and isinstance(arg.default, enum.Enum): - # Special case for enums. - help_parts.append(f"(default: {arg.default.name})") - elif not arg.required: - # General case. - help_parts.append("(default: %(default)s)") - - return dataclasses.replace(arg, help=" ".join(help_parts)) - else: - return arg - - def transform_use_type_as_metavar(arg: ArgumentDefinition) -> ArgumentDefinition: - """Communicate the argument type using the metavar.""" - if ( - hasattr(arg.type, "__name__") - # Don't generate metavar if target is still wrapping something, eg - # Optional[int] will have 1 arg. - and len(get_args(arg.type)) == 0 - # If choices is set, they'll be used by default. - and arg.choices is None - # Don't generate metavar if one already exists. - and arg.metavar is None - ): - return dataclasses.replace( - arg, metavar=arg.type.__name__.upper() # type: ignore - ) # type: ignore - else: - return arg + return str(x) - return [v for k, v in locals().items() if k.startswith("transform_")] + if arg.default is None or arg.action is not None: + return arg + elif arg.nargs is not None: + return dataclasses.replace(arg, default=tuple(map(as_str, arg.default))) + else: + return dataclasses.replace(arg, default=as_str(arg.default)) diff --git a/dcargs/_construction.py b/dcargs/_construction.py index 6f6278b89..4b01a6da4 100644 --- a/dcargs/_construction.py +++ b/dcargs/_construction.py @@ -1,3 +1,5 @@ +"""Core functionality for instantiating dataclasses from argparse namespaces.""" + from typing import TYPE_CHECKING, Any, Dict, Set, Tuple, Type, TypeVar from typing_extensions import get_args @@ -54,7 +56,7 @@ def get_value_from_arg(arg: str) -> Any: value: Any prefixed_field_name = field_name_prefix + field.name - # Resolve field type + # Resolve field type. field_type = ( type_from_typevar[field.type] # type: ignore if field.type in type_from_typevar @@ -67,7 +69,8 @@ def get_value_from_arg(arg: str) -> Any: value = get_value_from_arg(prefixed_field_name) if value is not None: try: - value = arg.field_action(value) + assert arg.instantiator is not None + value = arg.instantiator(value) except ValueError as e: raise FieldActionValueError( f"Parsing error for {arg.get_flag()}: {e.args[0]}" diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index c33b81eaa..8606cd5f6 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -1,3 +1,5 @@ +"""Helpers for parsing dataclass docstrings. Used for helptext generation.""" + import dataclasses import functools import inspect diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py new file mode 100644 index 000000000..532676cc9 --- /dev/null +++ b/dcargs/_instantiators.py @@ -0,0 +1,245 @@ +"""Helper for using type annotations to recursively generate instantiator functions, +which map strings (or, in some cases, sequences of strings) to the annotated type. + +Some examples of type annotations and the desired instantiators: +``` + int + + lambda string: int(str) + + List[int] + + lambda strings: list( + [int(x) for x in strings] + ) + + List[Color], where Color is an enum + + lambda strings: list( + [Color[x] for x in strings] + ) + + Tuple[int, float] + + lambda strings: tuple( + [ + typ(x) + for typ, x in zip( + (int, float), + strings, + ) + ] + ) +``` +""" + +import collections +import dataclasses +import enum +from typing import Any, Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union + +from typing_extensions import Final, Literal, _AnnotatedAlias, get_args, get_origin + +from . import _strings + +Instantiator = Union[ + # Most standard fields: these are converted from strings from the CLI. + Callable[[str], Any], + # Sequence fields! This should be used whenever argparse's `nargs` field is set. + Callable[[List[str]], Any], + # Special case: the only time that argparse doesn't give us a string is when the + # argument action is set to `store_true` or `store_false`. In this case, we get + # a bool directly, and the field action can be a no-op. + Callable[[bool], bool], +] + + +@dataclasses.dataclass +class InstantiatorMetadata: + nargs: Optional[Union[str, int]] + metavar: Union[str, Tuple[str, ...]] + choices: Optional[Tuple[Any, ...]] + is_optional: bool + + +class UnsupportedTypeAnnotationError(Exception): + """Exception raised when field actions fail; this typically means that values from + the CLI are invalid.""" + + +def instantiator_from_type( + typ: Type, type_from_typevar: Dict[TypeVar, Type] +) -> Tuple[Instantiator, InstantiatorMetadata]: + """Recursive helper for parsing type annotations. + + Returns two things: + - An instantiator function, for instantiating the type from a string or list of + strings. The latter applies when argparse's `nargs` parameter is set. + - A metadata structure, which specifies parameters for argparse. + """ + + # Resolve typevars. + if typ in type_from_typevar: + return instantiator_from_type( + type_from_typevar[typ], # type: ignore + type_from_typevar, + ) + + # Address container types. If a matching container is found, this will recursively + # call instantiator_from_type(). + container_out = _instantiator_from_container_type(typ, type_from_typevar) + if container_out is not None: + return container_out + + # Construct instantiators for raw types. + auto_choices: Optional[Tuple[str, ...]] = None + if typ is bool: + auto_choices = ("True", "False") + elif issubclass(typ, enum.Enum): + auto_choices = tuple(x.name for x in typ) + return lambda arg: _strings.instance_from_string(typ, arg), InstantiatorMetadata( + nargs=None, + metavar=typ.__name__.upper(), + choices=auto_choices, + is_optional=False, + ) + + +def _instantiator_from_container_type( + typ: Type, type_from_typevar: Dict[TypeVar, Type] +) -> Optional[Tuple[Instantiator, InstantiatorMetadata]]: + """Attempt to create an instantiator from a container type. Returns `None` is no + container type is found.""" + + type_origin = get_origin(typ) + if type_origin is None: + return None + + # Unwrap Final types. + if type_origin is Final: + (contained_type,) = get_args(typ) + return instantiator_from_type(contained_type, type_from_typevar) + + # Unwrap Annotated types. + if hasattr(typ, "__class__") and typ.__class__ == _AnnotatedAlias: + return instantiator_from_type(type_origin, type_from_typevar) + + # List, tuples, and sequences. + if type_origin in ( + collections.abc.Sequence, # different from typing.Sequence! + list, # different from typing.List! + set, # different from typing.Set! + ): + (contained_type,) = get_args(typ) + container_type = type_origin + if container_type is collections.abc.Sequence: + container_type = list + + make, inner_meta = _instantiator_from_type_inner( + contained_type, type_from_typevar + ) + return lambda strings: container_type( + [make(x) for x in strings] + ), InstantiatorMetadata( + nargs="+", + metavar=inner_meta.metavar, + choices=inner_meta.choices, + is_optional=False, + ) + + # Tuples. + if type_origin is tuple: + types = get_args(typ) + typeset = set(types) # Note that sets are unordered. + typeset_no_ellipsis = typeset - {Ellipsis} # type: ignore + + if typeset_no_ellipsis != typeset: + # Ellipsis: variable argument counts. + if len(typeset_no_ellipsis) > 1: + raise UnsupportedTypeAnnotationError( + "When an ellipsis is used, tuples must contain only one type." + ) + (contained_type,) = typeset_no_ellipsis + + make, inner_meta = _instantiator_from_type_inner( + contained_type, type_from_typevar + ) + return lambda strings: tuple( + [make(x) for x in strings] + ), InstantiatorMetadata( + nargs="+", + metavar=inner_meta.metavar, + choices=inner_meta.choices, + is_optional=False, + ) + + else: + instantiators, metas = zip( + *map( + lambda t: _instantiator_from_type_inner(t, type_from_typevar), + types, + ) + ) + if len(set(m.choices for m in metas)) > 1: + raise UnsupportedTypeAnnotationError( + "Due to constraints in argparse, all choices in fixed-length tuples" + " must match. This restricts mixing enums & literals with other" + " types." + ) + return lambda strings: tuple( + make(x) for make, x in zip(instantiators, strings) + ), InstantiatorMetadata( + nargs=len(types), + metavar=tuple(m.metavar for m in metas), + choices=metas[0].choices, + is_optional=False, + ) + + # Optionals. + if type_origin is Union: + options = set(get_args(typ)) + assert ( + len(options) == 2 and type(None) in options + ), "Union must be either over dataclasses (for subparsers) or Optional" + (typ,) = options - {type(None)} + instantiator, metadata = _instantiator_from_type_inner( + typ, type_from_typevar, allow_sequences=True + ) + return instantiator, dataclasses.replace(metadata, is_optional=True) + + # Literals. + if type_origin is Literal: + choices = get_args(typ) + contained_type = type(next(iter(choices))) + assert all( + map(lambda c: type(c) == contained_type, choices) + ), "All choices in literal must have the same type!" + if issubclass(contained_type, enum.Enum): + choices = tuple(map(lambda x: x.name, choices)) + instantiator, metadata = _instantiator_from_type_inner( + contained_type, type_from_typevar + ) + assert ( + # Choices provided by the contained type + metadata.choices is None + or len(set(choices) - set(metadata.choices)) == 0 + ) + return instantiator, dataclasses.replace(metadata, choices=choices) + + return None + + +def _instantiator_from_type_inner( + typ: Type, + type_from_typevar: Dict[TypeVar, Type], + allow_sequences: bool = False, + allow_optional: bool = False, +) -> Tuple[Instantiator, InstantiatorMetadata]: + """Thin wrapper over instantiator_from_type, with some extra asserts for catching + errors.""" + out = instantiator_from_type(typ, type_from_typevar) + if not allow_sequences and out[1].nargs is not None: + raise UnsupportedTypeAnnotationError("Nested sequence types are not supported!") + if not allow_optional and out[1].is_optional: + raise UnsupportedTypeAnnotationError("Nested optional types are not supported!") + return out diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index e1cf88a54..af3e4382a 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -1,3 +1,5 @@ +"""Abstractions for creating argparse parsers from a dataclass definition.""" + import argparse import dataclasses import warnings @@ -5,7 +7,7 @@ from typing_extensions import get_args, get_origin -from . import _arguments, _docstrings, _resolver, _strings +from . import _arguments, _docstrings, _instantiators, _resolver, _strings T = TypeVar("T") @@ -114,9 +116,10 @@ def from_dataclass( if typ in parent_type_from_typevar: type_from_typevar[typevar] = parent_type_from_typevar[typ] # type: ignore - assert ( - cls not in parent_dataclasses - ), f"Found a cyclic dataclass dependency with type {cls}" + if cls in parent_dataclasses: + raise _instantiators.UnsupportedTypeAnnotationError( + f"Found a cyclic dataclass dependency with type {cls}." + ) parent_dataclasses = parent_dataclasses | {cls} args = [] @@ -140,9 +143,10 @@ def from_dataclass( # Try to create subparsers from this field. subparsers_out = nested_handler.handle_unions_over_dataclasses() if subparsers_out is not None: - assert ( - subparsers is None - ), "Only one subparser (union over dataclasses) is allowed per class" + if subparsers is not None: + raise _instantiators.UnsupportedTypeAnnotationError( + "Only one subparser (union over dataclasses) is allowed per class." + ) subparsers = subparsers_out continue @@ -157,12 +161,20 @@ def from_dataclass( continue # Handle simple fields! - arg = _arguments.ArgumentDefinition.make_from_field( - cls, - field, - type_from_typevar, - default=field_default_instance, - ) + try: + arg = _arguments.ArgumentDefinition.make_from_field( + cls, + field, + type_from_typevar, + default=field_default_instance, + ) + except _instantiators.UnsupportedTypeAnnotationError as e: + # Catch unsupported annotation errors, and make the error message more + # informative. + raise _instantiators.UnsupportedTypeAnnotationError( + f"Error when parsing {cls.__name__}.{field.name} of type" + f" {field.type}: {e.args[0]}" + ) args.append(arg) return ParserSpecification( diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 5adab5af9..2d8d990d9 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -1,3 +1,5 @@ +"""Utilities for resolving generic types and forward references.""" + import copy import dataclasses import functools diff --git a/dcargs/_serialization.py b/dcargs/_serialization.py index 20b5d3137..924430ba0 100644 --- a/dcargs/_serialization.py +++ b/dcargs/_serialization.py @@ -1,3 +1,5 @@ +"""Type-safe, human-readable serialization helpers.""" + import dataclasses import datetime import enum diff --git a/dcargs/_strings.py b/dcargs/_strings.py index e7223e145..7ee848795 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -1,7 +1,12 @@ +"""Utilities for working with strings.""" + +import enum import functools import re import textwrap -from typing import Type +from typing import Type, TypeVar + +from typing_extensions import get_args from . import _resolver @@ -40,10 +45,38 @@ def subparser_name_from_type(cls: Type) -> str: ) -def bool_from_string(text: str) -> bool: - if text == "True": - return True - elif text == "False": - return False +T = TypeVar("T") + + +def instance_from_string(typ: Type[T], arg: str) -> T: + """Given a type and and a string from the command-line, reconstruct an object. Not + intended to deal with containers. + + This is intended to replace all calls to `type(string)`, which can cause unexpected + behavior. As an example, note that the following argparse code will always print + `True`, because `bool("True") == bool("False") == bool("0") == True`. + ``` + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--flag", type=bool) + + print(parser.parse_args().flag) + ``` + """ + assert len(get_args(typ)) == 0, f"Type {typ} cannot be instantiated." + if typ is bool: + if arg == "True": + return True # type: ignore + elif arg == "False": + return False # type: ignore + else: + raise ValueError(f"Boolean (True/False) expected, but got {arg}.") + elif issubclass(typ, enum.Enum): + try: + return typ[arg] # type: ignore + except KeyError as e: + # Raise enum key errors as value errors. + raise ValueError(*e.args) else: - raise ValueError(f"Boolean (True/False) expected, but got {text}.") + return typ(arg) # type: ignore diff --git a/examples/subparsers.py b/examples/subparsers.py index f2d8dc6f3..0e95f5d27 100644 --- a/examples/subparsers.py +++ b/examples/subparsers.py @@ -33,6 +33,5 @@ class Commit: if __name__ == "__main__": - # args = dcargs.parse(Args) - args = dcargs.parse(Args, default_instance=Args(command=Checkout(branch="main"))) + args = dcargs.parse(Args) print(args) diff --git a/setup.py b/setup.py index 11c96159f..06f417855 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.18", + version="0.0.19", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_collections.py b/tests/test_collections.py index bbcee0ffe..55556af92 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -1,7 +1,9 @@ import dataclasses +import enum from typing import List, Optional, Sequence, Set, Tuple import pytest +from typing_extensions import Literal import dcargs @@ -116,13 +118,57 @@ class A: dcargs.parse(A, args=[]) -def test_lists_with_default(): +def test_list_with_literal(): @dataclasses.dataclass class A: - x: List[int] = dataclasses.field(default_factory=[0, 1, 2].copy) + x: List[Literal[1, 2, 3]] - assert dcargs.parse(A, args=[]) == A(x=[0, 1, 2]) assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x", "1", "2", "3", "4"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + +def test_list_with_enums(): + class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + @dataclasses.dataclass + class A: + x: List[Color] + + assert dcargs.parse(A, args=["--x", "RED", "RED", "BLUE"]) == A( + x=[Color.RED, Color.RED, Color.BLUE] + ) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x", "RED", "RED", "YELLOW"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + +def test_lists_with_default(): + class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + @dataclasses.dataclass + class A: + x: List[Color] = dataclasses.field( + default_factory=[Color.RED, Color.GREEN].copy + ) + + assert dcargs.parse(A, args=[]) == A(x=[Color.RED, Color.GREEN]) + assert dcargs.parse(A, args=["--x", "RED", "GREEN", "BLUE"]) == A( + x=[Color.RED, Color.GREEN, Color.BLUE] + ) def test_lists_bool(): diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 723f6693f..b24a8cd81 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -142,7 +142,7 @@ class EnumClassB: assert dcargs.parse(EnumClassA, args=["--color", "RED"]) == EnumClassA( color=Color.RED ) - assert dcargs.parse(EnumClassB) == EnumClassB() + assert dcargs.parse(EnumClassB, args=[]) == EnumClassB() def test_literal(): @@ -277,4 +277,4 @@ def test_parse_empty_description(): class A: x: int = 0 - assert dcargs.parse(A, description=None) == A(x=0) + assert dcargs.parse(A, description=None, args=[]) == A(x=0) From 38f8e88ac3477edc25e85d32cc93bc20c172ed8b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 21 Apr 2022 17:31:53 -0700 Subject: [PATCH 037/491] Some TLC for helptext generation --- dcargs/_arguments.py | 9 ++--- dcargs/_construction.py | 5 ++- dcargs/_parsers.py | 81 +++++++++++++++++++++++++++++++++-------- examples/example.py | 17 +++++---- setup.py | 4 +- tests/test_helptext.py | 2 +- 6 files changed, 86 insertions(+), 32 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index c51ffc697..d2229691d 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -30,7 +30,7 @@ class ArgumentDefinition: default: Optional[Any] # Fields that will be handled by argument transformations. - required: Optional[bool] = None + required: bool = False action: Optional[str] = None nargs: Optional[Union[int, str]] = None choices: Optional[Set[Any]] = None @@ -103,11 +103,10 @@ def make_from_field( def _transform_required_if_default_set(arg: ArgumentDefinition) -> ArgumentDefinition: """Set `required=True` if a default value is set.""" - # Don't set if default is set, or if required flag is already set. - if arg.default is not None: - return dataclasses.replace(arg, required=False) - else: + # Mark arg as required if a default is set. + if arg.default is None: return dataclasses.replace(arg, required=True) + return arg def _transform_handle_boolean_flags(arg: ArgumentDefinition) -> ArgumentDefinition: diff --git a/dcargs/_construction.py b/dcargs/_construction.py index 4b01a6da4..203e7d933 100644 --- a/dcargs/_construction.py +++ b/dcargs/_construction.py @@ -75,7 +75,10 @@ def get_value_from_arg(arg: str) -> Any: raise FieldActionValueError( f"Parsing error for {arg.get_flag()}: {e.args[0]}" ) - elif prefixed_field_name in parser_definition.nested_dataclass_field_names: + elif ( + prefixed_field_name + in parser_definition.helptext_from_nested_dataclass_field_name + ): # Nested dataclasses. value, consumed_keywords_child = construct_dataclass( field_type, diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index af3e4382a..321dc6a01 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -2,9 +2,11 @@ import argparse import dataclasses +import shutil import warnings from typing import Any, Dict, Generic, List, Optional, Set, Tuple, Type, TypeVar, Union +import termcolor from typing_extensions import get_args, get_origin from . import _arguments, _docstrings, _instantiators, _resolver, _strings @@ -56,22 +58,64 @@ class ParserSpecification: cls: Type args: List[_arguments.ArgumentDefinition] - nested_dataclass_field_names: List[str] + helptext_from_nested_dataclass_field_name: Dict[str, Optional[str]] subparsers: Optional["SubparsersSpecification"] def apply(self, parser: argparse.ArgumentParser) -> None: """Create defined arguments and subparsers.""" - # Put required group at start of group list. - required_group = parser.add_argument_group("required arguments") + def format_group_name(nested_field_name: str, required: bool) -> str: + if required: + prefix = termcolor.colored("required", attrs=["bold"]) + else: + prefix = termcolor.colored("optional", attrs=["bold", "dark"]) + suffix = " arguments" + if nested_field_name == "": + suffix = suffix[1:] + + return ( + prefix + + " " + + nested_field_name.replace("_", " ").replace(".", " • ") + + suffix + ) + + optional_group_from_prefix: Dict[str, argparse._ArgumentGroup] = { + "": parser._action_groups[1], + } + required_group_from_prefix: Dict[str, argparse._ArgumentGroup] = { + "": parser.add_argument_group(format_group_name("", required=True)), + } + + # Break some API boundaries to rename the optional group, and + parser._action_groups[1].title = format_group_name("", required=False) parser._action_groups = parser._action_groups[::-1] # Add each argument. for arg in self.args: if arg.required: - arg.add_argument(required_group) + target_groups, other_groups = ( + required_group_from_prefix, + optional_group_from_prefix, + ) else: - arg.add_argument(parser) + target_groups, other_groups = ( + optional_group_from_prefix, + required_group_from_prefix, + ) + + if arg.prefix not in target_groups: + nested_field_name = arg.prefix[:-1] + target_groups[arg.prefix] = parser.add_argument_group( + format_group_name(nested_field_name, required=arg.required), + # Add a description, but only to the first group for a field. + description=self.helptext_from_nested_dataclass_field_name[ + nested_field_name + ] + if arg.prefix not in other_groups + else None, + ) + arg.add_argument(target_groups[arg.prefix]) # Add subparsers. if self.subparsers is not None: @@ -123,7 +167,7 @@ def from_dataclass( parent_dataclasses = parent_dataclasses | {cls} args = [] - nested_dataclass_field_names = [] + helptext_from_nested_dataclass_field_name = {} subparsers = None for field in _resolver.resolved_fields(cls): # type: ignore @@ -145,7 +189,8 @@ def from_dataclass( if subparsers_out is not None: if subparsers is not None: raise _instantiators.UnsupportedTypeAnnotationError( - "Only one subparser (union over dataclasses) is allowed per class." + "Only one subparser (union over dataclasses) is allowed per" + " class." ) subparsers = subparsers_out @@ -156,8 +201,12 @@ def from_dataclass( if nested_out is not None: child_args, child_nested_field_names = nested_out args.extend(child_args) - nested_dataclass_field_names.extend(child_nested_field_names) - nested_dataclass_field_names.append(field.name) + helptext_from_nested_dataclass_field_name.update( + child_nested_field_names + ) + helptext_from_nested_dataclass_field_name[ + field.name + ] = _docstrings.get_field_docstring(cls, field.name) continue # Handle simple fields! @@ -180,7 +229,7 @@ def from_dataclass( return ParserSpecification( cls=cls, args=args, - nested_dataclass_field_names=nested_dataclass_field_names, + helptext_from_nested_dataclass_field_name=helptext_from_nested_dataclass_field_name, subparsers=subparsers, ) @@ -253,7 +302,7 @@ def handle_unions_over_dataclasses( def handle_nested_dataclasses( self, - ) -> Optional[Tuple[List[_arguments.ArgumentDefinition], List[str]]]: + ) -> Optional[Tuple[List[_arguments.ArgumentDefinition], Dict[str, Optional[str]]]]: """Handle nested dataclasses. Returns `None` if not applicable.""" # Resolve field type field_type = ( @@ -281,9 +330,9 @@ def handle_nested_dataclasses( + arg.prefix, ) - nested_dataclass_field_names = [ - self.field.name + _strings.NESTED_DATACLASS_DELIMETER + x - for x in child_definition.nested_dataclass_field_names - ] + helptext_from_nested_dataclass_field_name = { + self.field.name + _strings.NESTED_DATACLASS_DELIMETER + x: y + for x, y in child_definition.helptext_from_nested_dataclass_field_name.items() + } - return child_args, nested_dataclass_field_names + return child_args, helptext_from_nested_dataclass_field_name diff --git a/examples/example.py b/examples/example.py index f18a22c8a..e12091d87 100644 --- a/examples/example.py +++ b/examples/example.py @@ -1,7 +1,8 @@ """An argument parsing example. -Note that there are multiple possible ways to document dataclass attributes, all -of which are supported by the automatic helptext generator. +Note that multiple possible documentation styles are supported by the field helptext +generator; we could also use docstring-style triple quote comments, or #-style comments +on the same line. """ import dataclasses @@ -17,8 +18,8 @@ class OptimizerType(enum.Enum): @dataclasses.dataclass(frozen=True) class OptimizerConfig: - # Variant of SGD to use. - type: OptimizerType + # Gradient-based optimizer to use. + algorithm: OptimizerType = OptimizerType.ADAM # Learning rate to use. learning_rate: float = 3e-4 @@ -33,13 +34,15 @@ class ExperimentConfig: pulled from this docstring by default, but can also be overrided with `dcargs.parse`'s `description=` flag.""" - experiment_name: str # Experiment name to use. + # Experiment name to use. + experiment_name: str + # Various configurable options for our optimizer. optimizer: OptimizerConfig + # Random seed. This is helpful for making sure that our experiments are all + # reproducible! seed: int = 0 - """Random seed. This is helpful for making sure that our experiments are - all reproducible!""" if __name__ == "__main__": diff --git a/setup.py b/setup.py index 06f417855..1cc813e3f 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.19", + version="0.0.20", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", @@ -16,7 +16,7 @@ packages=find_packages(), package_data={"dcargs": ["py.typed"]}, python_requires=">=3.7", - install_requires=["typing_extensions>=4.0.0", "pyyaml"], + install_requires=["typing_extensions>=4.0.0", "pyyaml", "termcolor"], extras_require={ "testing": [ "pytest", diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 154c1dfff..9e80649ad 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -30,7 +30,7 @@ class Helptext: dcargs.parse(Helptext, args=["--help"]) helptext = f.getvalue() assert Helptext.__doc__ in helptext - assert "required arguments:\n --x INT Documentation 1\n" in helptext + assert ":\n --x INT Documentation 1\n" in helptext assert "--y INT Documentation 2\n" in helptext assert "--z INT Documentation 3 (default: 3)\n" in helptext From a3a37b0be7e50bf5b97c77585100b8f75ec15177 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 30 Apr 2022 17:25:56 -0700 Subject: [PATCH 038/491] Coverage + code hygiene --- .coveragerc | 27 +++++ .github/workflows/coverage.yml | 32 ++++++ README.md | 104 ++++++++++---------- dcargs/__init__.py | 2 + dcargs/_docstrings.py | 7 +- dcargs/_instantiators.py | 26 ++--- dcargs/_parsers.py | 8 +- dcargs/_strings.py | 13 +-- examples/docstrings.py | 21 ++++ examples/{example.py => nested.py} | 4 +- tests/test_collections.py | 12 +++ tests/test_dcargs.py | 30 ++++++ tests/test_errors.py | 119 +++++++++++++++++++++++ tests/test_generics_and_serialization.py | 28 ++++++ tests/test_helptext.py | 34 +++++++ 15 files changed, 382 insertions(+), 85 deletions(-) create mode 100644 .coveragerc create mode 100644 .github/workflows/coverage.yml create mode 100644 examples/docstrings.py rename examples/{example.py => nested.py} (92%) create mode 100644 tests/test_errors.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..74c0558ed --- /dev/null +++ b/.coveragerc @@ -0,0 +1,27 @@ +[report] +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't compute coverage for abstract methods, properties + @abstract + @abc\.abstract + + # or warnings + warnings + + # or empty function bodies + pass + \.\.\. + + # or typing imports + TYPE_CHECKING + + # or assert statements & errors + assert + + # or anything that's not implemented + NotImplementedError() + + # or fallback imports + except ImportError: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 000000000..679a55132 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,32 @@ +name: coverage + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[testing]" + - name: Generate coverage report + run: | + pytest --cov=dcargs --cov-report=xml + - name: Upload to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./coverage.xml + flags: unittests + name: codecov-umbrella + fail_ci_if_error: true diff --git a/README.md b/README.md index 94cbc97f3..7c0d54577 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,15 @@ ![build](https://github.com/brentyi/dcargs/workflows/build/badge.svg) ![mypy](https://github.com/brentyi/dcargs/workflows/mypy/badge.svg?branch=master) ![lint](https://github.com/brentyi/dcargs/workflows/lint/badge.svg) +[![codecov](https://codecov.io/gh/brentyi/dcargs/branch/master/graph/badge.svg)](https://codecov.io/gh/brentyi/dcargs) * [Overview](#overview) * [Core interface](#core-interface) -* [Motivation](#motivation) * [Serialization](#serialization) * [Feature list](#feature-list) -* [Comparisons to alternative tools](#comparisons-to-alternative-tools) +* [Alternative tools](#alternative-tools) @@ -20,12 +20,31 @@ **`dcargs`** is a library for defining argument parsers and configuration objects using standard Python dataclasses. -Installation is simple: - ``` pip install dcargs ``` +Compared to other options, using dataclasses for configuration is: + +- **Strongly typed.** Unlike dynamic configuration namespaces produced by + libraries like `argparse`, `YACS`, `abseil`, or `ml_collections`, this means + that IDE-assisted autocomplete, rename, refactor, go-to-definition operations + work out-of-the-box, as do static checking tools like `mypy` and `pyright`. +- **Modular.** Most approaches to configuration objects require a centralized + definition of all configurable fields. Hierarchically nesting dataclasses, + however, makes it easy to distribute definitions, defaults, and documentation + of configurable fields across modules or source files. A model configuration + dataclass, for example, can be co-located in its entirety with the model + implementation and dropped into any experiment configuration with an import — + this eliminates redundancy and makes the entire module easy to port across + codebases. +- **Noninvasive.** Dataclasses themselves are part of the standard Python + library; defining them requires no external dependencies and they can be + easily instantiated without `dcargs` (for example, within quick experiments in + Jupyter notebooks). +- **Low effort.** Type annotations, docstrings, and default values for dataclass + fields can be used to automatically generate argument parsers. + ### Core interface Our core interface is composed of a single function, which instantiates a @@ -112,31 +131,6 @@ arguments, enums, and more. Examples of additional features can be found in the [examples](./examples/) and [unit tests](./tests/); a [feature list](#feature-list) is also included below. -### Motivation - -Compared to other options, using dataclasses for configuration is: - -- **Low effort.** Type annotations, docstrings, and default values for dataclass - fields can be used to automatically generate argument parsers. -- **Noninvasive.** Dataclasses themselves are part of the standard Python - library; defining them requires no external dependencies and they can be - easily instantiated without `dcargs` (for example, within quick experiments in - Jupyter notebooks). -- **Modular.** Most approaches to configuration objects require a centralized - definition of all configurable fields. Hierarchically nesting dataclasses, - however, makes it easy to distribute definitions, defaults, and documentation - of configurable fields across modules or source files. A model configuration - dataclass, for example, can be co-located in its entirety with the model - implementation and dropped into any experiment configuration dataclass with an - import — this eliminates the redundancy you typically see with the argparse - equivalent and makes the entire module easy to port across codebases. -- **Strongly typed.** Unlike dynamic configuration namespaces produced by - libraries like `argparse`, `YACS`, `abseil`, or `ml_collections`, dataclasses - are robustly supported by static type checking tools (mypy, pyright, etc), as - well as IDEs and language servers. This means code can be checked - automatically for errors and typos, and IDE-assisted autocomplete, rename, - refactor, and jump operations work out-of-the-box. - ### Serialization As a secondary feature, we also introduce two functions for human-readable @@ -190,36 +184,17 @@ containing: `Final[Optional[Sequence[T]]]`, etc - Nested dataclasses - Simple nesting (see `OptimizerConfig` in - [./examples/example.py](./examples/example.py)) + [./examples/nested.py](./examples/nested.py)) - Unions over nested dataclasses (subparsers, see [./examples/subparsers.py](./examples/subparsers.py)) - Optional unions over nested dataclasses (optional subparsers) - Generic dataclasses (including nested generics, see [./examples/generics.py](./examples/generics.py)) -### Comparisons to alternative tools - -There are several alternatives for the parsing functionality of `dcargs`; here's -a rough summary of some of them: - -| | dataclasses | attrs | Choices from literals | Generics | Docstrings as helptext | Nesting | Subparsers | Containers | -| ------------------------------------------------------------------------------------------------------------ | ----------- | ----- | -------------------------------------------------------- | -------- | ---------------------- | ------- | ---------- | ---------- | -| **dcargs** | ✓ | | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| **[datargs](https://github.com/roee30/datargs)** | ✓ | ✓ | ✓ | | | | ✓ | ✓ | -| **[typed-argument-parser](https://github.com/swansonk14/typed-argument-parser)** | | | ✓ | | ✓ | | ✓ | ✓ | -| **[simple-parsing](https://github.com/lebrice/SimpleParsing)** | ✓ | | [soon](https://github.com/lebrice/SimpleParsing/pull/86) | | ✓ | ✓ | ✓ | ✓ | -| **[argparse-dataclass](https://pypi.org/project/argparse-dataclass/)** | ✓ | | | | | | | | -| **[argparse-dataclasses](https://pypi.org/project/argparse-dataclasses/)** | ✓ | | | | | | | | -| **[dataclass-cli](https://github.com/malte-soe/dataclass-cli)** | ✓ | | | | | | | | -| **[clout](https://github.com/python-clout/clout)** | | ✓ | | | | ✓ | | | -| **[hf_argparser](https://github.com/huggingface/transformers/blob/master/src/transformers/hf_argparser.py)** | ✓ | | | | | | | ✓ | - -Some other distinguishing factors that we've put effort into: +Some other design decisions that we've put effort into: -- Robust handling of forward references -- Support for nested containers and generics - Strong typing: we actively avoid relying on strings or dynamic namespace - objects (eg `argparse.Namespace`) + objects (eg `argparse.Namespace`). - Simplicity + strict abstractions: we're focused on a single function API, and don't leak any argparse implementation details to the user level. We also intentionally don't offer any way to add argument parsing-specific logic to @@ -228,4 +203,29 @@ Some other distinguishing factors that we've put effort into: to make parsing-specific dataclasses) - POSIX compatibility. For example, field names with underscores (`argument_name`) are parsed with hyphens (`--argument-name`). - ([why](https://stackoverflow.com/questions/1253679/should-command-line-options-in-posix-style-operating-systems-be-underscore-style)) + ([why?](https://stackoverflow.com/questions/1253679/should-command-line-options-in-posix-style-operating-systems-be-underscore-style)) + +### Alternative tools + +The core functionality of `dcargs` --- generating an argument parser from type +annotations --- can be found as a subset of the features offered by many other +libraries. A summary of some distinguishing features: + +| | Choices from literals | Generics | Docstrings as helptext | Nesting | Subparsers | Containers | +| ------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------- | -------- | ---------------------- | ------- | ---------- | ---------- | +| **dcargs** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| **[datargs](https://github.com/roee30/datargs)** | ✓ | | | | ✓ | ✓ | +| **[tap](https://github.com/swansonk14/typed-argument-parser)** | ✓ | | ✓ | | ✓ | ✓ | +| **[simple-parsing](https://github.com/lebrice/SimpleParsing)** | [soon](https://github.com/lebrice/SimpleParsing/pull/86) | | ✓ | ✓ | ✓ | ✓ | +| **[argparse-dataclass](https://pypi.org/project/argparse-dataclass/)** | | | | | | | +| **[argparse-dataclasses](https://pypi.org/project/argparse-dataclasses/)** | | | | | | | +| **[dataclass-cli](https://github.com/malte-soe/dataclass-cli)** | | | | | | | +| **[clout](https://pypi.org/project/clout/)** | | | | ✓ | | | +| **[hf_argparser](https://github.com/huggingface/transformers/blob/master/src/transformers/hf_argparser.py)** | | | | | | ✓ | +| **[pyrallis](https://github.com/eladrich/pyrallis/)** | | | ✓ | ✓ | | ✓ | + +Note that most of these other libraries offer other features that you might find +useful, such as `attrs` support (`datargs`, `clout`), registration for custom +types (`pyrallis`), different approaches for serialization and config files +(`tap`, `pyrallis`), simultaneous parsing of multiple dataclasses +(`simple-parsing`), etc. diff --git a/dcargs/__init__.py b/dcargs/__init__.py index 1883e95df..c99bae5bc 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,7 +1,9 @@ +from ._instantiators import UnsupportedTypeAnnotationError from ._parse import parse from ._serialization import from_yaml, to_yaml __all__ = [ + "UnsupportedTypeAnnotationError", "parse", "from_yaml", "to_yaml", diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 8606cd5f6..6ff4dd13a 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -82,10 +82,9 @@ def get_class_tokenization_with_field( found_field: bool = False classes_to_search = cls.mro() for search_cls in classes_to_search: - # Unwrap generics. - origin_cls = get_origin(search_cls) - if origin_cls is not None: - search_cls = origin_cls + # Inherited generics seem challenging for now. + # https://github.com/python/typing/issues/777 + assert get_origin(search_cls) is None # Skip parent classes that aren't dataclasses. if not dataclasses.is_dataclass(search_cls): diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index 532676cc9..8424f03f1 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -38,7 +38,7 @@ import enum from typing import Any, Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union -from typing_extensions import Final, Literal, _AnnotatedAlias, get_args, get_origin +from typing_extensions import Final, Literal, get_args, get_origin from . import _strings @@ -84,6 +84,14 @@ def instantiator_from_type( type_from_typevar[typ], # type: ignore type_from_typevar, ) + elif isinstance(typ, TypeVar): + # Found an unbound TypeVar. This could be because inheriting from generics is + # currently not implemented. It's unclear whether this is feasible, because + # generics are lost in the mro: https://github.com/python/typing/issues/777 + raise UnsupportedTypeAnnotationError( + f"Found unbound TypeVar {typ}. Note that inheritance from generic dataclass" + " types is currently not supported." + ) # Address container types. If a matching container is found, this will recursively # call instantiator_from_type(). @@ -120,10 +128,6 @@ def _instantiator_from_container_type( (contained_type,) = get_args(typ) return instantiator_from_type(contained_type, type_from_typevar) - # Unwrap Annotated types. - if hasattr(typ, "__class__") and typ.__class__ == _AnnotatedAlias: - return instantiator_from_type(type_origin, type_from_typevar) - # List, tuples, and sequences. if type_origin in ( collections.abc.Sequence, # different from typing.Sequence! @@ -154,11 +158,9 @@ def _instantiator_from_container_type( typeset_no_ellipsis = typeset - {Ellipsis} # type: ignore if typeset_no_ellipsis != typeset: - # Ellipsis: variable argument counts. - if len(typeset_no_ellipsis) > 1: - raise UnsupportedTypeAnnotationError( - "When an ellipsis is used, tuples must contain only one type." - ) + # Ellipsis: variable argument counts. When an ellipsis is used, tuples must + # contain only one type. + assert len(typeset_no_ellipsis) == 1 (contained_type,) = typeset_no_ellipsis make, inner_meta = _instantiator_from_type_inner( @@ -226,7 +228,9 @@ def _instantiator_from_container_type( ) return instantiator, dataclasses.replace(metadata, choices=choices) - return None + raise UnsupportedTypeAnnotationError( # pragma: no cover + f"Unsupported type {typ} with origin {type_origin}" + ) def _instantiator_from_type_inner( diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 321dc6a01..5a1643a03 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -2,7 +2,6 @@ import argparse import dataclasses -import shutil import warnings from typing import Any, Dict, Generic, List, Optional, Set, Tuple, Type, TypeVar, Union @@ -47,7 +46,7 @@ def _get_field_default( field_default_instance = field.default_factory() if parent_default_instance is not None: - # Populate default from explicit `default_instance` in `dcargs.parse()`. + # Populate default from some parent, eg `default_instance` in `dcargs.parse()`. field_default_instance = getattr(parent_default_instance, field.name) return field_default_instance @@ -142,15 +141,12 @@ def format_group_name(nested_field_name: str, required: bool) -> str: @staticmethod def from_dataclass( cls: Type[T], - parent_dataclasses: Optional[Set[Type]], + parent_dataclasses: Set[Type], parent_type_from_typevar: Optional[Dict[TypeVar, Type]], default_instance: Optional[T], ) -> "ParserSpecification": """Create a parser definition from a dataclass.""" - if parent_dataclasses is None: - parent_dataclasses = set() - assert _resolver.is_dataclass(cls) cls, type_from_typevar = _resolver.resolve_generic_classes(cls) diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 7ee848795..96357121c 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -66,17 +66,8 @@ def instance_from_string(typ: Type[T], arg: str) -> T: """ assert len(get_args(typ)) == 0, f"Type {typ} cannot be instantiated." if typ is bool: - if arg == "True": - return True # type: ignore - elif arg == "False": - return False # type: ignore - else: - raise ValueError(f"Boolean (True/False) expected, but got {arg}.") + return {"True": True, "False": False}[arg] # type: ignore elif issubclass(typ, enum.Enum): - try: - return typ[arg] # type: ignore - except KeyError as e: - # Raise enum key errors as value errors. - raise ValueError(*e.args) + return typ[arg] # type: ignore else: return typ(arg) # type: ignore diff --git a/examples/docstrings.py b/examples/docstrings.py new file mode 100644 index 000000000..319b6c351 --- /dev/null +++ b/examples/docstrings.py @@ -0,0 +1,21 @@ +import dataclasses + +import dcargs + + +@dcargs.parse +@dataclasses.dataclass +class Args: + # Color input. + red: int + green: int + blue: int + + # Boolean values. + # These are useful! + flag: bool + flag1: bool = False + flag2: bool = True + + +print(Args.red) diff --git a/examples/example.py b/examples/nested.py similarity index 92% rename from examples/example.py rename to examples/nested.py index e12091d87..12e7ce71b 100644 --- a/examples/example.py +++ b/examples/nested.py @@ -8,6 +8,8 @@ import dataclasses import enum +from typing_extensions import Annotated + import dcargs @@ -22,7 +24,7 @@ class OptimizerConfig: algorithm: OptimizerType = OptimizerType.ADAM # Learning rate to use. - learning_rate: float = 3e-4 + learning_rate: Annotated[float, "Learning rate"] = 3e-4 # Coefficient for L2 regularization. weight_decay: float = 1e-2 diff --git a/tests/test_collections.py b/tests/test_collections.py index 55556af92..b8614adfd 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -20,6 +20,18 @@ class A: dcargs.parse(A, args=[]) +def test_tuples_fixed_mixed(): + @dataclasses.dataclass + class A: + x: Tuple[int, str, int] + + assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=(1, "2", 3)) + with pytest.raises(SystemExit): + dcargs.parse(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.parse(A, args=[]) + + def test_tuples_with_default(): @dataclasses.dataclass class A: diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index b24a8cd81..15bc8600c 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -32,6 +32,36 @@ class ManyTypes: ) == ManyTypes(i=5, s="5", f=5.0, p=pathlib.Path("~")) +def test_init_false(): + @dataclasses.dataclass + class InitFalseDataclass: + i: int + s: str + f: float + p: pathlib.Path + ignored: str = dataclasses.field(default="hello", init=False) + + assert dcargs.parse( + InitFalseDataclass, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--p", + "~", + ], + ) == InitFalseDataclass(i=5, s="5", f=5.0, p=pathlib.Path("~")) + + with pytest.raises(SystemExit): + dcargs.parse( + InitFalseDataclass, + args=["--i", "5", "--s", "5", "--f", "5", "--p", "~", "--ignored", "blah"], + ) + + def test_required(): @dataclasses.dataclass class A: diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 000000000..3eff2ad77 --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,119 @@ +import dataclasses +from typing import Generic, Optional, Tuple, TypeVar, Union + +import pytest +from typing_extensions import Literal + +import dcargs + + +def test_choices_in_tuples(): + """Due to argparse limitations, all parameters of `choices` must match. In the + future, we might avoid this by implementing choice restrictions manually.""" + # OK + @dataclasses.dataclass + class A: + x: Tuple[bool, bool] + + assert dcargs.parse(A, args=["--x", "True", "False"]) == A((True, False)) + + # OK + @dataclasses.dataclass + class A: + x: Tuple[bool, Literal["True", "False"]] + + assert dcargs.parse(A, args=["--x", "True", "False"]) == A((True, "False")) + + # Not OK: same argument, different choices. + @dataclasses.dataclass + class A: + x: Tuple[bool, Literal["True", "False", "None"]] + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.parse(A, args=["--x", "True", "False"]) + + +def test_nested_sequence_types(): + """Unclear how to handle nested sequences, so we don't support them.""" + + @dataclasses.dataclass + class A: + x: Tuple[Tuple[int, ...], ...] + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.parse(A, args=["--x", "0", "1"]) + + +def test_nested_optional_types(): + """Unclear how to handle optionals nested in other types, so we don't support + them. + + In the future, we might support "None" as a special-case keyword. But this is a bit + weird because Optional[str] might interprete "None" as either a string or an actual + `None` value.""" + + @dataclasses.dataclass + class A: + x: Tuple[Optional[int], ...] + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.parse(A, args=["--x", "0", "1"]) + + +def test_multiple_subparsers(): + """argparse doesn't support multiple subparsers.""" + + @dataclasses.dataclass + class Subcommmand1: + pass + + @dataclasses.dataclass + class Subcommand2: + pass + + @dataclasses.dataclass + class MultipleSubparsers: + x: Union[Subcommmand1, Subcommand2] + y: Union[Subcommmand1, Subcommand2] + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.parse(MultipleSubparsers) + + +# Must be global. +@dataclasses.dataclass +class _CycleDataclass: + x: "_CycleDataclass" + + +def test_cycle(): + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.parse(_CycleDataclass) + + +def test_generic_inherited(): + """Inheriting from generics is currently not implemented. It's unclear whether this + is feasible, because generics are lost in the mro: + https://github.com/python/typing/issues/777""" + + class UnrelatedParentClass: + pass + + T = TypeVar("T") + + @dataclasses.dataclass + class ActualParentClass(Generic[T]): + x: T # Documentation 1 + + # Documentation 2 + y: T + + z: T = 3 + """Documentation 3""" + + @dataclasses.dataclass + class ChildClass(UnrelatedParentClass, ActualParentClass[int]): + pass + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.parse(ChildClass, args=["--x", 1, "--y", 2, "--z", 3]) diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index f87923930..fd63d4a8b 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -232,3 +232,31 @@ class Subparser(Generic[T1, T2]): _check_serialization_identity( Subparser[Command[int], Command[float]], parsed_instance ) + + +# Not implemented. It's unclear whether this is feasible, because generics are lost in +# the mro: https://github.com/python/typing/issues/777 +# +# def test_generic_inherited(): +# class UnrelatedParentClass: +# pass +# +# T = TypeVar("T") +# +# @dataclasses.dataclass +# class ActualParentClass(Generic[T]): +# x: T # Documentation 1 +# +# # Documentation 2 +# y: T +# +# z: T = 3 +# """Documentation 3""" +# +# @dataclasses.dataclass +# class ChildClass(UnrelatedParentClass, ActualParentClass[int]): +# pass +# +# assert dcargs.parse(ChildClass, args=["--x", 1, "--y", 2, "--z", 3]) == ChildClass( +# 1, 2, 3 +# ) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 9e80649ad..c000f3010 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -35,6 +35,40 @@ class Helptext: assert "--z INT Documentation 3 (default: 3)\n" in helptext +def test_helptext_inherited(): + class UnrelatedParentClass: + pass + + @dataclasses.dataclass + class ActualParentClass: + """This docstring should be printed as a description.""" + + x: int # Documentation 1 + + # Documentation 2 + y: int + + # fmt: off + + z: int = 3 + def some_method(self) -> None: # noqa + """Coverage stress test.""" + pass + # fmt: on + + @dataclasses.dataclass + class ChildClass(UnrelatedParentClass, ActualParentClass): + pass + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(ChildClass, args=["--help"]) + helptext = f.getvalue() + assert ":\n --x INT Documentation 1\n" in helptext + assert "--y INT Documentation 2\n" in helptext + + def test_helptext_defaults(): class Color(enum.Enum): RED = enum.auto() From c27a114055370e77f059db0d421bc72cb743717c Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 26 May 2022 19:28:40 -0700 Subject: [PATCH 039/491] Improve helptext generation - Comment parsing is now more robust; it also now supports grouping, where the same comment is associated with multiple fields. - Printed defaults, particularly for sequence types (lists, tuples, sets), are now more consistently formatted. --- README.md | 5 +- dcargs/_arguments.py | 20 +++- dcargs/_docstrings.py | 133 ++++++++++++++++----------- examples/{docstrings.py => flags.py} | 8 +- setup.py | 2 +- tests/test_helptext.py | 36 +++++++- 6 files changed, 136 insertions(+), 68 deletions(-) rename examples/{docstrings.py => flags.py} (65%) diff --git a/README.md b/README.md index 7c0d54577..b89d1a163 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,8 @@ Compared to other options, using dataclasses for configuration is: easily instantiated without `dcargs` (for example, within quick experiments in Jupyter notebooks). - **Low effort.** Type annotations, docstrings, and default values for dataclass - fields can be used to automatically generate argument parsers. + fields can be used to automatically generate argument parsers with informative + helptext. ### Core interface @@ -201,7 +202,7 @@ Some other design decisions that we've put effort into: dataclass definitions. (in contrast, some of the libaries above rely heavily on dataclass field metadata, or on the more extreme end inheritance+decorators to make parsing-specific dataclasses) -- POSIX compatibility. For example, field names with underscores +- Hyphens over underscores. For example, field names with underscores (`argument_name`) are parsed with hyphens (`--argument-name`). ([why?](https://stackoverflow.com/questions/1253679/should-command-line-options-in-posix-style-operating-systems-be-underscore-style)) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index d2229691d..350ecdec7 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -1,6 +1,7 @@ import argparse import dataclasses import enum +import shlex from typing import Any, Dict, Optional, Set, Tuple, Type, TypeVar, Union from . import _docstrings, _instantiators @@ -179,10 +180,21 @@ def _transform_generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: # Special case for enums. help_parts.append(f"(default: {arg.default.name})") elif not arg.required: - # General case. We intentionally don't use the % template, because the default - # will be casted to a string and that can make unnecessary quotation marks - # appear in the helptext. - help_parts.append(f"(default: {arg.default})") + # Include default value in helptext. We intentionally don't use the % template + # because the types of all arguments are set to strings, which will cause the + # default to be casted to a string and introduce extra quotation marks. + if arg.nargs is not None and hasattr(arg.default, "__iter__"): + # For tuple types, we might have arg.default as (0, 1, 2, 3). + # For list types, we might have arg.default as [0, 1, 2, 3]. + # For set types, we might have arg.default as {0, 1, 2, 3}. + # + # In all cases, we want to display (default: 0 1 2 3), for consistency with + # the format that argparse expects when we set nargs. + assert arg.default is not None # Just for type checker. + default_parts = map(shlex.quote, map(str, arg.default)) + help_parts.append(f"(default: {' '.join(default_parts)})") + else: + help_parts.append(f"(default: {shlex.quote(str(arg.default))})") return dataclasses.replace(arg, help=" ".join(help_parts)) diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 6ff4dd13a..fb857ee15 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -16,61 +16,82 @@ class _Token: token_type: int content: str - line_number: int + logical_line: int + actual_line: int @dataclasses.dataclass(frozen=True) class _FieldData: index: int - line_number: int - prev_field_line_number: int + logical_line: int + actual_line: int + prev_field_logical_line: int @dataclasses.dataclass(frozen=True) class _ClassTokenization: tokens: List[_Token] - tokens_from_line: Dict[int, List[_Token]] + tokens_from_logical_line: Dict[int, List[_Token]] + tokens_from_actual_line: Dict[int, List[_Token]] field_data_from_name: Dict[str, _FieldData] @staticmethod @functools.lru_cache(maxsize=8) - def make(cls) -> "_ClassTokenization": + def make(clz) -> "_ClassTokenization": """Parse the source code of a class, and cache some tokenization information.""" - readline = io.BytesIO(inspect.getsource(cls).encode("utf-8")).readline + readline = io.BytesIO(inspect.getsource(clz).encode("utf-8")).readline tokens: List[_Token] = [] - tokens_from_line: Dict[int, List[_Token]] = {1: []} + tokens_from_logical_line: Dict[int, List[_Token]] = {1: []} + tokens_from_actual_line: Dict[int, List[_Token]] = {1: []} field_data_from_name: Dict[str, _FieldData] = {} - line_number: int = 1 + logical_line: int = 1 + actual_line: int = 1 for toktype, tok, start, end, line in tokenize.tokenize(readline): - if toktype in (tokenize.NEWLINE, tokenize.NL): - line_number += 1 - tokens_from_line[line_number] = [] + # Note: we only track logical line numbers, which are delimited by + # `tokenize.NEWLINE`. `tokenize.NL` tokens appear when logical lines are + # broken into multiple lines of code; these are ignored. + if toktype == tokenize.NEWLINE: + logical_line += 1 + actual_line += 1 + tokens_from_logical_line[logical_line] = [] + tokens_from_actual_line[actual_line] = [] + elif toktype == tokenize.NL: + actual_line += 1 + tokens_from_actual_line[actual_line] = [] elif toktype is not tokenize.INDENT: - token = _Token(token_type=toktype, content=tok, line_number=line_number) + token = _Token( + token_type=toktype, + content=tok, + logical_line=logical_line, + actual_line=actual_line, + ) tokens.append(token) - tokens_from_line[line_number].append(token) + tokens_from_logical_line[logical_line].append(token) + tokens_from_actual_line[actual_line].append(token) - prev_field_line_number: int = 1 + prev_field_logical_line: int = 1 for i, token in enumerate(tokens[:-1]): if token.token_type == tokenize.NAME: - # Naive heuristic for field names + # Naive heuristic for field names. if ( tokens[i + 1].content == ":" - and tokens[i] == tokens_from_line[tokens[i].line_number][0] + and tokens[i] == tokens_from_actual_line[tokens[i].actual_line][0] and token.content not in field_data_from_name ): field_data_from_name[token.content] = _FieldData( index=i, - line_number=token.line_number, - prev_field_line_number=prev_field_line_number, + logical_line=token.logical_line, + actual_line=token.actual_line, + prev_field_logical_line=prev_field_logical_line, ) - prev_field_line_number = token.line_number + prev_field_logical_line = token.logical_line return _ClassTokenization( tokens=tokens, - tokens_from_line=tokens_from_line, + tokens_from_logical_line=tokens_from_logical_line, + tokens_from_actual_line=tokens_from_actual_line, field_data_from_name=field_data_from_name, ) @@ -120,13 +141,13 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: field_data = tokenization.field_data_from_name[field_name] - # Check for docstring-style comment. - line_number = field_data.line_number + 1 - while ( - line_number in tokenization.tokens_from_line - and len(tokenization.tokens_from_line[line_number]) > 0 + # Check for docstring-style comment. This should be on the next logical line. + logical_line = field_data.logical_line + 1 + if ( + logical_line in tokenization.tokens_from_logical_line + and len(tokenization.tokens_from_logical_line[logical_line]) >= 1 ): - first_token = tokenization.tokens_from_line[line_number][0] + first_token = tokenization.tokens_from_logical_line[logical_line][0] first_token_content = first_token.content.strip() # Found a docstring! @@ -137,49 +158,49 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: ): return _strings.dedent(first_token_content[3:-3]) - # Found the next field. - if ( - first_token.token_type == tokenize.NAME - and len(tokenization.tokens_from_line[line_number]) >= 2 - and tokenization.tokens_from_line[line_number][1].content == ":" - ): - break - - # Found a method. - if first_token.content == "def": - break - - line_number += 1 - # Check for comment on the same line as the field. - final_token_on_line = tokenization.tokens_from_line[field_data.line_number][-1] + final_token_on_line = tokenization.tokens_from_logical_line[ + field_data.logical_line + ][-1] if final_token_on_line.token_type == tokenize.COMMENT: comment: str = final_token_on_line.content assert comment.startswith("#") return comment[1:].strip() - # Check for comment on the line before the field. - comment_index = field_data.index + # Check for comments that come before the field. This is intentionally written to + # support comments covering multiple (grouped) fields, for example: + # + # # Optimizer hyperparameters. + # learning_rate: float + # beta1: float + # beta2: float + # + # In this case, 'Optimizer hyperparameters' will be treated as the docstring for all + # 3 fields. comments: List[str] = [] - current_line_number = field_data.line_number - while True: - comment_index -= 1 - comment_token = tokenization.tokens[comment_index] + current_actual_line = field_data.actual_line - 1 + while current_actual_line in tokenization.tokens_from_actual_line: + actual_line_tokens = tokenization.tokens_from_actual_line[current_actual_line] + current_actual_line -= 1 + + # We stop looking if we find an empty line. + if len(actual_line_tokens) == 0: + break + + # Record single comments! if ( - # Looking for comments! - comment_token.token_type == tokenize.COMMENT - # Comments should come after the previous field. - and comment_token.line_number > field_data.prev_field_line_number - # And be contiguous. - and comment_token.line_number == current_line_number - 1 + len(actual_line_tokens) == 1 + and actual_line_tokens[0].token_type == tokenize.COMMENT ): + (comment_token,) = actual_line_tokens assert comment_token.content.startswith("#") - current_line_number -= 1 comments.append(comment_token.content[1:].strip()) - else: + elif len(comments) > 0: + # Comments should be contiguous. break + if len(comments) > 0: - return "\n".join(comments[::-1]) + return "\n".join(reversed(comments)) return None diff --git a/examples/docstrings.py b/examples/flags.py similarity index 65% rename from examples/docstrings.py rename to examples/flags.py index 319b6c351..b0cfc7e58 100644 --- a/examples/docstrings.py +++ b/examples/flags.py @@ -3,7 +3,6 @@ import dcargs -@dcargs.parse @dataclasses.dataclass class Args: # Color input. @@ -12,10 +11,13 @@ class Args: blue: int # Boolean values. - # These are useful! flag: bool flag1: bool = False flag2: bool = True -print(Args.red) +if __name__ == "__main__": + args = dcargs.parse(Args) + print(args) + print() + print(dcargs.to_yaml(args)) diff --git a/setup.py b/setup.py index 1cc813e3f..f849f32d4 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.0.20", + version="0.0.21", description="Portable, reusable, strongly typed CLIs from dataclass definitions", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_helptext.py b/tests/test_helptext.py index c000f3010..92edb3f07 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -123,6 +123,25 @@ class HelptextMultiline: ) +def test_grouped_helptext(): + @dataclasses.dataclass + class HelptextGrouped: + x: int # Documentation 1 + + # Description of both y and z. + y: int + z: int = 3 + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(HelptextGrouped, args=["--help"]) + helptext = f.getvalue() + assert " --x INT Documentation 1\n" in helptext + assert " --y INT Description of both y and z.\n" in helptext + assert " --z INT Description of both y and z. (default: 3)\n" in helptext + + def test_none_default_value_helptext(): @dataclasses.dataclass class Config: @@ -178,7 +197,7 @@ class Child(Parent): dcargs.parse(Child, args=["--help"]) helptext = f.getvalue() assert ( - "--x STR Helptext. (default: This docstring may be tougher to parse!)\n" + "--x STR Helptext. (default: 'This docstring may be tougher to parse!')\n" in helptext ) @@ -208,7 +227,7 @@ class Child2(Parent2): dcargs.parse(Child2, args=["--help"]) helptext = f.getvalue() assert ( - "--x STR Helptext! (default: This docstring may be tougher to parse?)\n" + "--x STR Helptext! (default: 'This docstring may be tougher to parse?')\n" in helptext ) @@ -226,6 +245,19 @@ class TupleHelptext: assert "--x INT STR FLOAT\n" in helptext +def test_tuple_helptext_defaults(): + @dataclasses.dataclass + class TupleHelptextDefaults: + x: Tuple[int, str, str] = (5, "hello world", "hello") + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.parse(TupleHelptextDefaults, args=["--help"]) + helptext = f.getvalue() + assert "--x INT STR STR (default: 5 'hello world' hello)\n" in helptext + + def test_generic_helptext(): T = TypeVar("T") From 88e80002ea5daed1c6e55d6bcbff8e517d7ed753 Mon Sep 17 00:00:00 2001 From: kevin-thankyou-lin <33344633+kevin-thankyou-lin@users.noreply.github.com> Date: Mon, 30 May 2022 09:48:17 -0700 Subject: [PATCH 040/491] Suppress TypeError from docstring parser in Jupyter notebooks (#3) * Catch jupyter notebook TypeError exception * Fix formatting --- dcargs/_docstrings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index fb857ee15..f3ee65429 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -118,6 +118,10 @@ def get_class_tokenization_with_field( # there's no docstring. assert "could not find class definition" in e.args[0] return None + except TypeError as e: # pragma: no cover + # Notebooks cause “___ is a built-in class” TypeError. + assert "built-in class" in e.args[0] + return None # Grab field-specific tokenization data. if field_name in tokenization.field_data_from_name: From da287709895961d4ef6d6adb3992f2264db3810f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 17 Jun 2022 23:09:25 -0700 Subject: [PATCH 041/491] v0.1.0: expand library scope beyond dataclasses Instead of just dataclasses, we now support general typed callables: functions, standard classes, attrs, etc. --- .coverage | Bin 0 -> 53248 bytes README.md | 347 +++++++++----- dcargs/__init__.py | 24 +- dcargs/_arguments.py | 74 +-- dcargs/{_construction.py => _calling.py} | 67 ++- dcargs/_cli.py | 99 ++++ dcargs/_docstrings.py | 50 +- dcargs/_instantiators.py | 8 - dcargs/_parse.py | 67 --- dcargs/_parsers.py | 450 +++++++++++------- dcargs/_resolver.py | 13 +- dcargs/_serialization.py | 2 +- dcargs/_strings.py | 3 +- dcargs/py.typed | 0 examples/0_simple_function.py | 20 + examples/{simple.py => 1_simple_dataclass.py} | 5 +- examples/2_enums_and_collections.py | 64 +++ examples/{flags.py => 3_flags.py} | 4 +- examples/{literals.py => 4_literals.py} | 2 +- examples/5_simple_class.py | 23 + .../{nested.py => 6_nested_dataclasses.py} | 8 +- examples/7_subparsers.py | 29 ++ examples/{generics_alt.py => 8_generics.py} | 2 +- examples/generics.py | 43 -- examples/subparsers.py | 37 -- setup.py | 12 +- tests/conftest.py | 4 + tests/test_attrs.py | 77 +++ tests/test_collections.py | 110 ++--- tests/test_dcargs.py | 145 +++--- tests/test_dynamic_dataclasses.py | 4 +- tests/test_errors.py | 16 +- tests/test_forward_ref.py | 16 +- tests/test_generics_and_serialization.py | 70 +-- tests/test_helptext.py | 35 +- tests/test_nested.py | 70 ++- tests/test_positional_ignore_py37.py | 57 +++ 37 files changed, 1261 insertions(+), 796 deletions(-) create mode 100644 .coverage rename dcargs/{_construction.py => _calling.py} (70%) create mode 100644 dcargs/_cli.py delete mode 100644 dcargs/_parse.py delete mode 100644 dcargs/py.typed create mode 100644 examples/0_simple_function.py rename examples/{simple.py => 1_simple_dataclass.py} (74%) create mode 100644 examples/2_enums_and_collections.py rename examples/{flags.py => 3_flags.py} (77%) rename examples/{literals.py => 4_literals.py} (95%) create mode 100644 examples/5_simple_class.py rename examples/{nested.py => 6_nested_dataclasses.py} (86%) create mode 100644 examples/7_subparsers.py rename examples/{generics_alt.py => 8_generics.py} (93%) delete mode 100644 examples/generics.py delete mode 100644 examples/subparsers.py create mode 100644 tests/conftest.py create mode 100644 tests/test_attrs.py create mode 100644 tests/test_positional_ignore_py37.py diff --git a/.coverage b/.coverage new file mode 100644 index 0000000000000000000000000000000000000000..38c7029b9bba90d95f8bb110594d9023add2a7b8 GIT binary patch literal 53248 zcmeI4ZEPGz8OL{T&$qW{`=(7~ol+Bdp=r-IM-7q=JN$s#4YP1w{frfJ9L%MQsIAB`VZHD^lLtNJJ$FN)iYpv z?2GU0*ynz!5?TM%-OTRnGc(Wgn`dTjcK3Gs)`tyOS4S+nthwrdaEl;{!sk>~5Cnz3 zt@I5ynRcS#0mWk7{&KsD(6fDCE88iw#EuK>(bliBOw0FMv(0~KnNYsld{TZ)DbWdR zAOHd&00JQ35t!N9BB$1^6QBH=tL00&YikAF_QTc(whe9BF{JL;^7)5{R6kAaOsKSF zGU^u9whpKj-Bw48lCBzN(I{xHVUDWqm>x=39lgkBJW8WEj|=vA`T1VXC{iw0AEhEH zwo%sXDRsX-)$IucwYmDF>nBhlx-n{U3AHoG-lf|5h;HjUMg3e=(-^Pn z^NruF)^c`xW|&sD%8eioxT86ix_7N8kXP!H&UlG_c;&RJYk9{xYW!Ru1ub8{H7=(* zI>e*no}+L)xxC>z^E=7{rI)7LMcb{w)NsV+~Px;Iti z42B14dQ5Zp;Nzs#yk(Vi&Ge@P5`561stiZ*f0TVi zV8_@&_9*RO0|5{K0T2KI5C8!X009sH0T2KI5Ll5wOH^9V!Y3+vZ4SJ)5WJ2;FSiFb#SZ zrl@6gbHd41rUo`=^M*+~uI{+$ZZ)0B(DMTK73q<}QHLM%bSFE1CS&Ur%l0J4+;VA< zgmZi+y*EJ?McJDIdz1Z}{gs_r(Le}*00@8p2!H?xfB*=900@8p2!H?x+yn$#r1fHW z0U{cg+C+Z+L28!T#o%H?q*+?8gcln4{r^Pr%K|&ie$2kX_OM~kYO!w@$Y-hBw78T7VAfBYuShAp%=*1qRROW_=Bn&sGzfdxG&0^#7a`Zrf zo6C_TzyF^|4h!sU*2j*rpCk`6k?l?XC^?*bd&wq7A_#y02!H?xfB*=900@8p2!H?x z2trmWhz(Y!i`V~&ZfS2L`338LAu2CN29{y;)WO{d?^+oh8KVEsp(h6 z&pvx?{Mk$I&Q1P)Zt|)7-+k{;=Heyk_9k9a(}&+2`1uFlYB_cC^u;Ub#@-w^t-YSbxMOkxxyXgJ@pPdreEWQ8#AM7ve4Er7X4SS9Kf)cTT00@8p2!H?x zfB*=900@8p2!H?x+;9T?s)LwFc#)!bQ9SNNvg}2%m=`rQc~La#MUv!2kw}y-0eJWS zgAOHd&00JNY0w4ea zAOHd&00JOz6ACFqs@lEQ;H4`d$1c+kXyC=VnaM0mj8|Nno3K_Z+0 literal 0 HcmV?d00001 diff --git a/README.md b/README.md index b89d1a163..ba7e48a2e 100644 --- a/README.md +++ b/README.md @@ -8,87 +8,161 @@ * [Overview](#overview) -* [Core interface](#core-interface) +* [Examples](#examples) + * [Functions](#functions) + * [Dataclasses](#dataclasses) + * [Nested dataclasses](#nested-dataclasses) * [Serialization](#serialization) -* [Feature list](#feature-list) * [Alternative tools](#alternative-tools) -### Overview +## Overview -**`dcargs`** is a library for defining argument parsers and configuration -objects using standard Python dataclasses. +**`dcargs`** is a library for strongly-typed argument parsers and configuration +objects. -``` +```bash pip install dcargs ``` -Compared to other options, using dataclasses for configuration is: - -- **Strongly typed.** Unlike dynamic configuration namespaces produced by - libraries like `argparse`, `YACS`, `abseil`, or `ml_collections`, this means - that IDE-assisted autocomplete, rename, refactor, go-to-definition operations - work out-of-the-box, as do static checking tools like `mypy` and `pyright`. -- **Modular.** Most approaches to configuration objects require a centralized - definition of all configurable fields. Hierarchically nesting dataclasses, - however, makes it easy to distribute definitions, defaults, and documentation - of configurable fields across modules or source files. A model configuration - dataclass, for example, can be co-located in its entirety with the model - implementation and dropped into any experiment configuration with an import — - this eliminates redundancy and makes the entire module easy to port across - codebases. -- **Noninvasive.** Dataclasses themselves are part of the standard Python - library; defining them requires no external dependencies and they can be - easily instantiated without `dcargs` (for example, within quick experiments in - Jupyter notebooks). -- **Low effort.** Type annotations, docstrings, and default values for dataclass - fields can be used to automatically generate argument parsers with informative - helptext. - -### Core interface - -Our core interface is composed of a single function, which instantiates a -dataclass from an automatically generated CLI interface: +Our core interface generates CLI interfaces from type-annotated callables, which +may be functions, classes, or dataclasses. The goal is a tool that's lightweight +enough for simple interactive scripts, but flexible enough to replace heavier +frameworks typically used to build hierarchical configuration systems.
- dcargs.parse(cls: Type[T], *, description: + dcargs.cli(f: Callable[..., T], *, description: Optional[str], args: Optional[Sequence[str]], default_instance: Optional[T]) -> T -
Generate a CLI containing fields for a dataclass, and use it to create an
-instance of the class. Gracefully handles nested dataclasses, container types,
-generics, optional and default arguments, enums, and more.
+
Call `f(...)`, with arguments populated from an automatically generated CLI
+interface.
+
+`f` should have type-annotated inputs, and can be a function, class, or dataclass.
+Note that if `f` is a class, `dcargs.cli()` returns an instance.
+
+The parser is generated by populating helptext from docstrings and types from
+annotations; a broad range of core type annotations are supported...
+    - Types natively accepted by `argparse`: str, int, float, pathlib.Path, etc.
+    - Default values for optional parameters.
+    - Booleans, which are automatically converted to flags when provided a default
+      value.
+    - Enums (via `enum.Enum`).
+    - Various container types. Some examples:
+      - `typing.ClassVar`.
+      - `typing.Optional`.
+      - `typing.Literal`.
+      - `typing.Sequence`.
+      - `typing.List`.
+      - `typing.Tuple`, such as `typing.Tuple[T1, T2, T3]` or
+        `typing.Tuple[T, ...]`.
+      - `typing.Set`.
+      - `typing.Final` and `typing.Annotated`.
+      - Nested combinations of the above: `Optional[Literal[T]]`,
+        `Final[Optional[Sequence[T]]]`, etc.
+    - Nested dataclasses.
+      - Simple nesting.
+      - Unions over nested dataclasses (subparsers).
+      - Optional unions over nested dataclasses (optional subparsers).
+    - Generic dataclasses (including nested generics).
 
 Args:
-    cls: Dataclass type to instantiate.
+    f: Callable.
 
 Keyword Args:
     description: Description text for the parser, displayed when the --help flag is
-        passed in. If not specified, the dataclass docstring is used. Mirrors argument
+        passed in. If not specified, `f`'s docstring is used. Mirrors argument
         from `argparse.ArgumentParser()`.
     args: If set, parse arguments from a sequence of strings instead of the
         commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`.
-    default_instance: An instance of `T` to use for default values. Helpful for overriding fields
-        in an existing instance; if not specified, the field defaults are used instead.
+    default_instance: An instance of `T` to use for default values; only supported
+        if `T` is a dataclass type. Helpful for merging CLI arguments with values loaded
+        from elsewhere. (for example, a config object loaded from a yaml file)
 
 Returns:
-    Instantiated dataclass.

+    The output of `f(...)`.
-**Example usage** +Notably, `dcargs.cli()` supports _nested_ classes and dataclasses, which enable +expressive hierarchical configuration objects built on standard Python features. + +Our ultimate goal is an interface that's: + +- **Low-effort.** Type annotations, docstrings, and default values can be used + to automatically generate argument parsers with informative helptext. This + includes bells and whistles like enums, containers, etc. +- **Strongly typed.** Unlike dynamic configuration namespaces produced by + libraries like `argparse`, `YACS`, `abseil`, `hydra`, or `ml_collections`, + statically typed outputs mean that IDE-assisted autocomplete, rename, + refactor, go-to-definition operations work out-of-the-box, as do static + checking tools like `mypy` and `pyright`. +- **Modular.** Most approaches to configuration objects require a centralized + definition of all configurable fields. Supporting hierarchically nested + configuration classes/dataclasses, however, makes it easy to distribute + definitions, defaults, and documentation of configurable fields across modules + or source files. A model configuration dataclass, for example, can be + co-located in its entirety with the model implementation and dropped into any + experiment configuration with an import — this eliminates redundancy and makes + the entire module easy to port across codebases. + +## Examples + +A series of example scripts can be found in [./examples](./examples). -If you're familiar with dataclasses, writing a script with `dcargs.parse()` is -simple! +### Functions ```python -# examples/simple.py +# examples/0_simple_function.py +import dcargs + + +def main( + field1: str, + field2: int, + flag: bool = False, +) -> None: + """Function, whose arguments will be populated from a CLI interface. + + Args: + field1: First field. + field2: Second field. + flag: Boolean flag that we can set to true. + """ + print(field1, field2, flag) + + +if __name__ == "__main__": + dcargs.cli(main) +``` + +--- + +```console +$ python 0_simple_function.py --help +usage: 0_simple_function.py [-h] --field1 STR --field2 INT [--flag] + +Function, whose arguments will be populated from a CLI interface. + +required arguments: + --field1 STR First field. + --field2 INT Second field. + +optional arguments: + -h, --help show this help message and exit + --flag Boolean flag that we can set to true. +``` + +### Dataclasses + +```python +# examples/1_simple_dataclass.py import dataclasses import dcargs @@ -96,21 +170,27 @@ import dcargs @dataclasses.dataclass class Args: + """Description. + This should show up in the helptext!""" + field1: str # A string field. field2: int # A numeric field. - flag: bool = False # A boolean flag. + flag: bool = False # A boolean flag. if __name__ == "__main__": - args = dcargs.parse(Args) + args = dcargs.cli(Args) print(args) ``` -We can run this to get: +--- -``` -$ python simple.py --help -usage: simple.py [-h] --field1 STR --field2 INT [--flag] +```console +$ python 1_simple_dataclass.py --help +usage: 1_simple_dataclass.py [-h] --field1 STR --field2 INT [--flag] + +Description. +This should show up in the helptext! required arguments: --field1 STR A string field. @@ -121,20 +201,91 @@ optional arguments: --flag A boolean flag. ``` +### Nested dataclasses + +```python +# examples/6_nested_dataclasses.py +import dataclasses +import enum + +import dcargs + + +class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + +@dataclasses.dataclass(frozen=True) +class OptimizerConfig: + # Gradient-based optimizer to use. + algorithm: OptimizerType = OptimizerType.ADAM + + # Learning rate to use. + learning_rate: float = 3e-4 + + # Coefficient for L2 regularization. + weight_decay: float = 1e-2 + + +@dataclasses.dataclass(frozen=True) +class ExperimentConfig: + """A nested experiment configuration. Note that the argument parser description is + pulled from this docstring by default, but can also be overrided with + `dcargs.cli()`'s `description=` argument.""" + + # Experiment name to use. + experiment_name: str + + # Various configurable options for our optimizer. + optimizer: OptimizerConfig + + # Random seed. This is helpful for making sure that our experiments are all + # reproducible! + seed: int = 0 + + +if __name__ == "__main__": + config = dcargs.cli(ExperimentConfig) + print(config) + print(dcargs.to_yaml(config)) ``` -$ python simple.py --field1 string --field2 4 -Args(field1='string', field2=4, flag=False) -``` -Note that we also support significantly more complex structures and annotations, -including nested dataclasses, container types, generics, optional and default -arguments, enums, and more. Examples of additional features can be found in the -[examples](./examples/) and [unit tests](./tests/); a -[feature list](#feature-list) is also included below. +--- + +```console +usage: 6_nested_dataclasses.py [-h] --experiment-name STR [--optimizer.algorithm {ADAM,SGD}] + [--optimizer.learning-rate FLOAT] [--optimizer.weight-decay FLOAT] + [--seed INT] -### Serialization +A nested experiment configuration. Note that the argument parser description is +pulled from this docstring by default, but can also be overrided with +`dcargs.cli()`'s `description=` flag. -As a secondary feature, we also introduce two functions for human-readable +required arguments: + --experiment-name STR + Experiment name to use. + +optional arguments: + -h, --help show this help message and exit + --seed INT Random seed. This is helpful for making sure that our experiments are all + reproducible! (default: 0) + +optional optimizer arguments: + Various configurable options for our optimizer. + + --optimizer.algorithm {ADAM,SGD} + Gradient-based optimizer to use. (default: ADAM) + --optimizer.learning-rate FLOAT + Learning rate to use. (default: 0.0003) + --optimizer.weight-decay FLOAT + Coefficient for L2 regularization. (default: 0.01) +``` + +## Serialization + +As a secondary feature aimed at enabling the use of `dcargs.cli()` for general +configuration use cases, we also introduce functions for human-readable dataclass serialization: - dcargs.from_yaml(cls: Type[T], stream: Union[str, @@ -148,67 +299,9 @@ PyYAML, etc), explicit type references enable custom tags that are robust against code reorganization and refactor, while a PyYAML backend enables serialization of arbitrary Python objects. -Particularly for cases where serialized dataclasses need to exit the Python -ecosystem, [dacite](https://github.com/konradhalas/dacite) is also a good option -(at the cost of a little bit of flexibility). - -### Feature list - -The parse function supports a wide range of dataclass definitions, while -automatically generating helptext from comments/docstrings. A selection of -features are shown in the [examples](./examples/). - -Our unit tests cover many more complex type annotations, including classes -containing: - -- Types natively accepted by `argparse`: str, int, float, pathlib.Path, etc -- Default values for optional parameters -- Booleans, which are automatically converted to flags when provided a default - value (eg `action="store_true"` or `action="store_false"`; in the latter case, - we prefix names with `no-`) -- Enums (via `enum.Enum`; argparse's `choices` is populated and arguments are - converted automatically) -- Various container types. Some examples: - - `typing.ClassVar` types (omitted from parser) - - `typing.Optional` types - - `typing.Literal` types (populates argparse's `choices`) - - `typing.Sequence` types (populates argparse's `nargs`) - - `typing.List` types (populates argparse's `nargs`) - - `typing.Tuple` types, such as `typing.Tuple[T1, T2, T3]` or - `typing.Tuple[T, ...]` (populates argparse's `nargs`, and converts - automatically) - - `typing.Set` types (populates argparse's `nargs`, and converts - automatically) - - `typing.Final` types and `typing.Annotated` (for parsing, these are - effectively no-ops) - - Nested combinations of the above: `Optional[Literal[T]]`, - `Final[Optional[Sequence[T]]]`, etc -- Nested dataclasses - - Simple nesting (see `OptimizerConfig` in - [./examples/nested.py](./examples/nested.py)) - - Unions over nested dataclasses (subparsers, see - [./examples/subparsers.py](./examples/subparsers.py)) - - Optional unions over nested dataclasses (optional subparsers) -- Generic dataclasses (including nested generics, see - [./examples/generics.py](./examples/generics.py)) - -Some other design decisions that we've put effort into: - -- Strong typing: we actively avoid relying on strings or dynamic namespace - objects (eg `argparse.Namespace`). -- Simplicity + strict abstractions: we're focused on a single function API, and - don't leak any argparse implementation details to the user level. We also - intentionally don't offer any way to add argument parsing-specific logic to - dataclass definitions. (in contrast, some of the libaries above rely heavily - on dataclass field metadata, or on the more extreme end inheritance+decorators - to make parsing-specific dataclasses) -- Hyphens over underscores. For example, field names with underscores - (`argument_name`) are parsed with hyphens (`--argument-name`). - ([why?](https://stackoverflow.com/questions/1253679/should-command-line-options-in-posix-style-operating-systems-be-underscore-style)) - -### Alternative tools - -The core functionality of `dcargs` --- generating an argument parser from type +## Alternative tools + +The core functionality of `dcargs` --- generating argument parsers from type annotations --- can be found as a subset of the features offered by many other libraries. A summary of some distinguishing features: @@ -225,8 +318,8 @@ libraries. A summary of some distinguishing features: | **[hf_argparser](https://github.com/huggingface/transformers/blob/master/src/transformers/hf_argparser.py)** | | | | | | ✓ | | **[pyrallis](https://github.com/eladrich/pyrallis/)** | | | ✓ | ✓ | | ✓ | -Note that most of these other libraries offer other features that you might find -useful, such as `attrs` support (`datargs`, `clout`), registration for custom -types (`pyrallis`), different approaches for serialization and config files -(`tap`, `pyrallis`), simultaneous parsing of multiple dataclasses -(`simple-parsing`), etc. +Note that most of these other libraries are generally aimed specifically at +_dataclasses_ rather than general typed callables, but offer other features that +you might find useful, such as registration for custom types (`pyrallis`), +different approaches for serialization and config files (`tap`, `pyrallis`), +simultaneous parsing of multiple dataclasses (`simple-parsing`), etc. diff --git a/dcargs/__init__.py b/dcargs/__init__.py index c99bae5bc..7673553ce 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,10 +1,30 @@ +from ._cli import cli from ._instantiators import UnsupportedTypeAnnotationError -from ._parse import parse from ._serialization import from_yaml, to_yaml __all__ = [ "UnsupportedTypeAnnotationError", - "parse", + "cli", "from_yaml", "to_yaml", ] + +from typing import TYPE_CHECKING + +if not TYPE_CHECKING: + + def parse(*args, **kwargs): + # API breaking transition plan: + # (v0.1.0) Rename dcargs.parse() to dcargs.cli(). Keep the former as an alias. + # ( ) Enable deprecation warning. + # ( ) Remove dcargs.parse() alias. + # + # import warnings + # + # warnings.warn( + # "`dcargs.parse()` has been renamed `dcargs.cli()`. It will be removed" + # " soon.", + # DeprecationWarning, + # stacklevel=2, + # ) + return cli(*args, **kwargs) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 350ecdec7..9229f66bb 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -1,23 +1,24 @@ +from __future__ import annotations + import argparse import dataclasses import enum import shlex -from typing import Any, Dict, Optional, Set, Tuple, Type, TypeVar, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Tuple, Type, TypeVar, Union + +from . import _instantiators -from . import _docstrings, _instantiators +if TYPE_CHECKING: + from . import _parsers @dataclasses.dataclass(frozen=True) class ArgumentDefinition: """Options for defining arguments. Contains all necessary arguments for argparse's - add_argument() method. - - TODO: this class (as well as major other parts of this library) has succumbed a bit - to entropy and could benefit from some refactoring.""" + add_argument() method.""" prefix: str # Prefix for nesting. - field: dataclasses.Field # Corresponding dataclass field. - parent_class: Type # Class that this field belongs to. + field: _parsers.Field # Corresponding dataclass field. # Action that is called on parsed arguments. This handles conversions from strings # to our desired types. @@ -59,9 +60,8 @@ def add_argument( if "choices" in kwargs: kwargs["choices"] = list(map(str, kwargs["choices"])) - kwargs.pop("field") - kwargs.pop("parent_class") kwargs.pop("prefix") + kwargs.pop("field") kwargs.pop("instantiator") kwargs.pop("name") @@ -70,34 +70,31 @@ def add_argument( def get_flag(self) -> str: """Get --flag representation, with a prefix applied for nested dataclasses.""" - return "--" + (self.prefix + self.name).replace("_", "-") + if self.field.positional: + return (self.prefix + self.name).replace("_", "-") + else: + return "--" + (self.prefix + self.name).replace("_", "-") @staticmethod - def make_from_field( - parent_class: Type, - field: dataclasses.Field, + def from_field( + field: _parsers.Field, type_from_typevar: Dict[TypeVar, Type], - default: Optional[Any], - ) -> "ArgumentDefinition": - """Create an argument definition from a field. Also returns a field action, which - specifies special instructions for reconstruction.""" - - assert field.init, "Field must be in class constructor" - + ) -> ArgumentDefinition: + """""" arg = ArgumentDefinition( prefix="", field=field, - parent_class=parent_class, instantiator=None, name=field.name, - type=field.type, - default=default, + type=field.typ, + default=field.default, ) arg = _transform_required_if_default_set(arg) arg = _transform_handle_boolean_flags(arg) arg = _transform_recursive_instantiator_from_type(arg, type_from_typevar) arg = _transform_generate_helptext(arg) arg = _transform_convert_defaults_to_strings(arg) + arg = _transform_positional_special_handling(arg) return arg @@ -107,7 +104,8 @@ def _transform_required_if_default_set(arg: ArgumentDefinition) -> ArgumentDefin # Mark arg as required if a default is set. if arg.default is None: return dataclasses.replace(arg, required=True) - return arg + + return dataclasses.replace(arg) def _transform_handle_boolean_flags(arg: ArgumentDefinition) -> ArgumentDefinition: @@ -166,12 +164,16 @@ def _transform_recursive_instantiator_from_type( def _transform_generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: """Generate helptext from docstring and argument name.""" help_parts = [] - docstring_help = _docstrings.get_field_docstring(arg.parent_class, arg.field.name) - if docstring_help is not None: + + docstring_help = arg.field.helptext + + if docstring_help is not None and docstring_help != "": # Note that the percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string docstring_help = docstring_help.replace("%", "%%") help_parts.append(docstring_help) + elif arg.field.positional: + help_parts.append(str(arg.metavar)) if arg.action is not None: # Don't show defaults for boolean flags. @@ -213,7 +215,23 @@ def as_str(x: Any) -> str: if arg.default is None or arg.action is not None: return arg - elif arg.nargs is not None: + elif arg.nargs is not None and arg.nargs != "?": return dataclasses.replace(arg, default=tuple(map(as_str, arg.default))) else: return dataclasses.replace(arg, default=as_str(arg.default)) + + +def _transform_positional_special_handling( + arg: ArgumentDefinition, +) -> ArgumentDefinition: + """Special handling for positional args.""" + + if not arg.field.positional: + return arg + + return dataclasses.replace( + arg, + metavar=(arg.prefix + arg.name).upper(), + required=None, + nargs="?" if not arg.required else arg.nargs, + ) diff --git a/dcargs/_construction.py b/dcargs/_calling.py similarity index 70% rename from dcargs/_construction.py rename to dcargs/_calling.py index 203e7d933..b89ad5009 100644 --- a/dcargs/_construction.py +++ b/dcargs/_calling.py @@ -1,15 +1,13 @@ -"""Core functionality for instantiating dataclasses from argparse namespaces.""" +"""Core functionality for calling functions with arguments specified by argparse +namespaces.""" -from typing import TYPE_CHECKING, Any, Dict, Set, Tuple, Type, TypeVar +from __future__ import annotations -from typing_extensions import get_args - -from . import _resolver, _strings +from typing import Any, Callable, Dict, List, Set, Tuple, TypeVar -if TYPE_CHECKING: - from . import _parsers +from typing_extensions import get_args -T = TypeVar("T") +from . import _parsers, _resolver, _strings class FieldActionValueError(Exception): @@ -17,23 +15,22 @@ class FieldActionValueError(Exception): the CLI are invalid.""" -DataclassType = TypeVar("DataclassType") +T = TypeVar("T") -def construct_dataclass( - cls: Type[DataclassType], - parser_definition: "_parsers.ParserSpecification", +def call_from_args( + f: Callable[..., T], + parser_definition: _parsers.ParserSpecification, value_from_arg: Dict[str, Any], field_name_prefix: str = "", -) -> Tuple[DataclassType, Set[str]]: - """Construct a dataclass object from a dictionary of values from argparse. - - Returns dataclass object and set of used arguments.""" +) -> Tuple[T, Set[str]]: + """Call `f` with arguments specified by a dictionary of values from argparse. - assert _resolver.is_dataclass(cls) + Returns the output of `f` and a set of used arguments.""" - cls, type_from_typevar = _resolver.resolve_generic_classes(cls) + f, type_from_typevar = _resolver.resolve_generic_types(f) + args: List[str] = [] kwargs: Dict[str, Any] = {} consumed_keywords: Set[str] = set() @@ -49,18 +46,15 @@ def get_value_from_arg(arg: str) -> Any: for arg in parser_definition.args: arg_from_prefixed_field_name[arg.prefix + arg.field.name] = arg - for field in _resolver.resolved_fields(cls): # type: ignore - if not field.init: - continue - + for field in _parsers._fields_from_callable(f, default_instance=None): # type: ignore value: Any prefixed_field_name = field_name_prefix + field.name # Resolve field type. field_type = ( - type_from_typevar[field.type] # type: ignore - if field.type in type_from_typevar - else field.type + type_from_typevar[field.typ] # type: ignore + if field.typ in type_from_typevar + else field.typ ) if prefixed_field_name in arg_from_prefixed_field_name: @@ -77,10 +71,10 @@ def get_value_from_arg(arg: str) -> Any: ) elif ( prefixed_field_name - in parser_definition.helptext_from_nested_dataclass_field_name + in parser_definition.helptext_from_nested_class_field_name ): - # Nested dataclasses. - value, consumed_keywords_child = construct_dataclass( + # Nested callable. + value, consumed_keywords_child = call_from_args( field_type, parser_definition, value_from_arg, @@ -111,19 +105,22 @@ def get_value_from_arg(arg: str) -> Any: lambda x: x if x not in type_from_typevar else type_from_typevar[x], get_args(field_type), ) - chosen_cls = None + chosen_f = None for option in options: if _strings.subparser_name_from_type(option) == subparser_name: - chosen_cls = option + chosen_f = option break - assert chosen_cls is not None - value, consumed_keywords_child = construct_dataclass( - chosen_cls, + assert chosen_f is not None + value, consumed_keywords_child = call_from_args( + chosen_f, parser_definition.subparsers.parser_from_name[subparser_name], value_from_arg, ) consumed_keywords |= consumed_keywords_child - kwargs[field.name] = value + if field.positional: + args.append(value) + else: + kwargs[field.name] = value - return cls(**kwargs), consumed_keywords # type: ignore + return f(*args, **kwargs), consumed_keywords # type: ignore diff --git a/dcargs/_cli.py b/dcargs/_cli.py new file mode 100644 index 000000000..035161604 --- /dev/null +++ b/dcargs/_cli.py @@ -0,0 +1,99 @@ +import argparse +from typing import Callable, Optional, Sequence, TypeVar + +from . import _calling, _docstrings, _parsers, _resolver + +T = TypeVar("T") + + +def cli( + f: Callable[..., T], + *, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + default_instance: Optional[T] = None, +) -> T: + """Call `f(...)`, with arguments populated from an automatically generated CLI + interface. + + `f` should have type-annotated inputs, and can be a function, class, or dataclass. + Note that if `f` is a class, `dcargs.parse()` returns an instance. + + The parser is generated by populating helptext from docstrings and types from + annotations; a broad range of core type annotations are supported... + - Types natively accepted by `argparse`: str, int, float, pathlib.Path, etc. + - Default values for optional parameters. + - Booleans, which are automatically converted to flags when provided a default + value. + - Enums (via `enum.Enum`). + - Various container types. Some examples: + - `typing.ClassVar`. + - `typing.Optional`. + - `typing.Literal`. + - `typing.Sequence`. + - `typing.List`. + - `typing.Tuple`, such as `typing.Tuple[T1, T2, T3]` or + `typing.Tuple[T, ...]`. + - `typing.Set`. + - `typing.Final` and `typing.Annotated`. + - Nested combinations of the above: `Optional[Literal[T]]`, + `Final[Optional[Sequence[T]]]`, etc. + - Nested dataclasses. + - Simple nesting. + - Unions over nested dataclasses (subparsers). + - Optional unions over nested dataclasses (optional subparsers). + - Generic dataclasses (including nested generics). + + Args: + f: Callable. + + Keyword Args: + description: Description text for the parser, displayed when the --help flag is + passed in. If not specified, `f`'s docstring is used. Mirrors argument from + `argparse.ArgumentParser()`. + args: If set, parse arguments from a sequence of strings instead of the + commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. + default_instance: An instance of `T` to use for default values; only supported + if `T` is a dataclass type. Helpful for merging CLI arguments with values loaded + from elsewhere. (for example, a config object loaded from a yaml file) + + Returns: + The output of `f(...)`. + """ + + # Map a callable to the relevant CLI arguments + subparsers. + assert default_instance is None or _resolver.is_dataclass( + f + ), "Default instance specification is only supported for dataclasses!" + parser_definition = _parsers.ParserSpecification.from_callable( + f, + parent_classes=set(), # Used for recursive calls. + parent_type_from_typevar=None, # Used for recursive calls. + default_instance=default_instance, # Overrides for default values. + ) + + if description is None: + description = _docstrings.get_callable_description(f) + + # Parse using argparse! + parser = argparse.ArgumentParser( + description=description, + formatter_class=argparse.RawTextHelpFormatter, + ) + parser_definition.apply(parser) + value_from_arg = vars(parser.parse_args(args=args)) + + try: + # Attempt to call `f` using whatever was passed in. + out, consumed_keywords = _calling.call_from_args( + f, parser_definition, value_from_arg + ) + except _calling.FieldActionValueError as e: + # Emulate argparse's error behavior when invalid arguments are passed in. + parser.print_usage() + print() + print(e.args[0]) + raise SystemExit() + + assert consumed_keywords == value_from_arg.keys() + return out diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index f3ee65429..da057701b 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -1,12 +1,12 @@ """Helpers for parsing dataclass docstrings. Used for helptext generation.""" - import dataclasses import functools import inspect import io import tokenize -from typing import Dict, List, Optional, Type +from typing import Callable, Dict, List, Optional, Type +import docstring_parser from typing_extensions import get_origin from . import _resolver, _strings @@ -102,15 +102,12 @@ def get_class_tokenization_with_field( # Search for token in this class + all parents. found_field: bool = False classes_to_search = cls.mro() + tokenization = None for search_cls in classes_to_search: # Inherited generics seem challenging for now. # https://github.com/python/typing/issues/777 assert get_origin(search_cls) is None - # Skip parent classes that aren't dataclasses. - if not dataclasses.is_dataclass(search_cls): - continue - try: tokenization = _ClassTokenization.make(search_cls) # type: ignore except OSError as e: @@ -128,10 +125,11 @@ def get_class_tokenization_with_field( found_field = True break - assert found_field, ( - "Docstring parsing error -- this usually means that there are multiple" - " dataclasses in the same file with the same name but different scopes." - ) + if dataclasses.is_dataclass(cls): + assert found_field, ( + "Docstring parsing error -- this usually means that there are multiple" + " dataclasses in the same file with the same name but different scopes." + ) return tokenization @@ -209,20 +207,34 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: return None -def get_dataclass_docstring(cls: Type) -> str: - """Get dataclass docstring, but only if it is hand-specified. +def get_callable_description(f: Callable) -> str: + """Get description associated with a callable via docstring parsing. Note that the `dataclasses.dataclass` will automatically populate __doc__ based on the fields of the class if a docstring is not specified; this helper will ignore these docstrings.""" - cls, _unused = _resolver.resolve_generic_classes(cls) - - # Docstring should never be `None` with a dataclass. - assert cls.__doc__ is not None + f, _unused = _resolver.resolve_generic_types(f) # Ignore any default docstrings, as generated by `dataclasses.dataclass`. - default_doc = cls.__name__ + str(inspect.signature(cls)).replace(" -> None", "") - if cls.__doc__ == default_doc: + docstring = inspect.getdoc(f) + if docstring is None: return "" - return cls.__doc__ + if dataclasses.is_dataclass(f): + default_doc = f.__name__ + str(inspect.signature(f)).replace(" -> None", "") + if docstring == default_doc: + return "" + return docstring + else: + parsed_docstring = docstring_parser.parse(docstring) + return "\n".join( + list( + filter( + lambda x: x is not None, # type: ignore + [ + parsed_docstring.short_description, + parsed_docstring.long_description, + ], + ) + ) + ) diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index 8424f03f1..2b7332e92 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -84,14 +84,6 @@ def instantiator_from_type( type_from_typevar[typ], # type: ignore type_from_typevar, ) - elif isinstance(typ, TypeVar): - # Found an unbound TypeVar. This could be because inheriting from generics is - # currently not implemented. It's unclear whether this is feasible, because - # generics are lost in the mro: https://github.com/python/typing/issues/777 - raise UnsupportedTypeAnnotationError( - f"Found unbound TypeVar {typ}. Note that inheritance from generic dataclass" - " types is currently not supported." - ) # Address container types. If a matching container is found, this will recursively # call instantiator_from_type(). diff --git a/dcargs/_parse.py b/dcargs/_parse.py deleted file mode 100644 index bb864e2c0..000000000 --- a/dcargs/_parse.py +++ /dev/null @@ -1,67 +0,0 @@ -import argparse -from typing import Optional, Sequence, Type, TypeVar - -from . import _construction, _docstrings, _parsers, _strings - -DataclassType = TypeVar("DataclassType") - - -def parse( - cls: Type[DataclassType], - *, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default_instance: Optional[DataclassType] = None, -) -> DataclassType: - """Generate a CLI containing fields for a dataclass, and use it to create an - instance of the class. Gracefully handles nested dataclasses, container types, - generics, optional and default arguments, enums, and more. - - Args: - cls: Dataclass type to instantiate. - - Keyword Args: - description: Description text for the parser, displayed when the --help flag is - passed in. If not specified, the dataclass docstring is used. Mirrors argument - from `argparse.ArgumentParser()`. - args: If set, parse arguments from a sequence of strings instead of the - commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. - default_instance: An instance of `T` to use for default values. Helpful for overriding fields - in an existing instance; if not specified, the field defaults are used instead. - - Returns: - Instantiated dataclass. - """ - # Map a dataclass to the relevant CLI arguments + subparsers. - parser_definition = _parsers.ParserSpecification.from_dataclass( - cls, - parent_dataclasses=set(), # Used for recursive calls. - parent_type_from_typevar=None, # Used for recursive calls. - default_instance=default_instance, # Overrides for default values. - ) - - if description is None: - description = _docstrings.get_dataclass_docstring(cls) - - # Parse using argparse! - parser = argparse.ArgumentParser( - description=_strings.dedent(description), - formatter_class=argparse.RawTextHelpFormatter, - ) - parser_definition.apply(parser) - value_from_arg = vars(parser.parse_args(args=args)) - - try: - # Attempt to construct a dataclass from whatever was passed in. - out, consumed_keywords = _construction.construct_dataclass( - cls, parser_definition, value_from_arg - ) - except _construction.FieldActionValueError as e: - # Emulate argparse's error behavior when invalid arguments are passed in. - parser.print_usage() - print() - print(e.args[0]) - raise SystemExit() - - assert consumed_keywords == value_from_arg.keys() - return out diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 5a1643a03..a6af5f2a9 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -1,11 +1,31 @@ """Abstractions for creating argparse parsers from a dataclass definition.""" +from __future__ import annotations + import argparse +import collections.abc import dataclasses +import inspect +import itertools import warnings -from typing import Any, Dict, Generic, List, Optional, Set, Tuple, Type, TypeVar, Union - +from typing import ( + Any, + Callable, + Dict, + Hashable, + List, + Optional, + Set, + Type, + TypeVar, + Union, + cast, + get_type_hints, +) + +import docstring_parser import termcolor +import typing_extensions from typing_extensions import get_args, get_origin from . import _arguments, _docstrings, _instantiators, _resolver, _strings @@ -28,6 +48,46 @@ def _ensure_dataclass_instance_used_as_default_is_frozen( ) +_known_parsable_types = set( + filter( + lambda x: isinstance(x, Hashable), # type: ignore + itertools.chain( + __builtins__.values(), # type: ignore + vars(typing_extensions).values(), + vars(collections.abc).values(), + ), + ) +) + + +def _is_parsable_type(typ: Any) -> bool: + """Heuristics for determining whether a type can be treated as a single argument. + This is true for built-in types like ints, strings, bools, etc, as well as + pathlib.Path.""" + + origin = get_origin(typ) + if origin is not None: + typ = origin + + # Known parsable types: builtins like int/str/float/bool, collections, annotations. + if typ in _known_parsable_types: + return True + + # Simple heuristic: dataclasses should be treated as nested objects and are not + # parsable. + if dataclasses.is_dataclass(typ): + return False + + # Non-parsable types like nested (data)classes should have fully type-annotated + # inputs. If any inputs are unannotated (for example, in the case of pathlib.Path), + # we can assume the type is parsable. + for param in inspect.signature(typ).parameters.values(): + if param.annotation is inspect.Parameter.empty: + return True + + return False + + def _get_field_default( field: dataclasses.Field, parent_default_instance: Any ) -> Optional[Any]: @@ -46,19 +106,183 @@ def _get_field_default( field_default_instance = field.default_factory() if parent_default_instance is not None: - # Populate default from some parent, eg `default_instance` in `dcargs.parse()`. + # Populate default from some parent, eg `default_instance` in `dcargs.cli()`. field_default_instance = getattr(parent_default_instance, field.name) return field_default_instance +@dataclasses.dataclass(frozen=True) +class Field: + name: str + typ: Type + default: Any + helptext: Optional[str] + positional: bool + + +def _fields_from_callable(f: Callable, default_instance: Any) -> List[Field]: + """Generate a list of generic 'field' objects corresponding to an input callable. + + `f` can be from a dataclass type, regular class type, or function.""" + out = [] + f, type_from_typevar = _resolver.resolve_generic_types(f) + if _resolver.is_dataclass(f): + for field in _resolver.resolved_fields(f): + if not field.init: + continue + out.append( + Field( + name=field.name, + typ=field.type, + default=_get_field_default(field, default_instance), + helptext=_docstrings.get_field_docstring(cast(Type, f), field.name), + positional=False, + ) + ) + else: + if isinstance(f, type): + hints = get_type_hints(f.__init__) # type: ignore + docstring = inspect.getdoc(f.__init__) # type: ignore + else: + hints = get_type_hints(f) + docstring = inspect.getdoc(f) + + docstring_from_name = {} + if docstring is not None: + for param_doc in docstring_parser.parse(docstring).params: + docstring_from_name[param_doc.arg_name] = param_doc.description + + for param in inspect.signature(f).parameters.values(): + field_docstring = ( + _docstrings.get_field_docstring(cast(Type, f), param.name) + if isinstance(f, type) and hasattr(f, "mro") + else None + ) + out.append( + Field( + name=param.name, + # Note that param.annotation does not resolve forward references. + typ=hints[param.name], + default=param.default + if param.default is not inspect.Parameter.empty + else None, + helptext=docstring_from_name.get( + param.name, + field_docstring, + ), + positional=param.kind is inspect.Parameter.POSITIONAL_ONLY, + ) + ) + return out + + @dataclasses.dataclass(frozen=True) class ParserSpecification: """Each parser contains a list of arguments and optionally some subparsers.""" - cls: Type + f: Callable args: List[_arguments.ArgumentDefinition] - helptext_from_nested_dataclass_field_name: Dict[str, Optional[str]] - subparsers: Optional["SubparsersSpecification"] + helptext_from_nested_class_field_name: Dict[str, Optional[str]] + subparsers: Optional[SubparsersSpecification] + + @staticmethod + def from_callable( + f: Callable, + parent_classes: Set[Type], + parent_type_from_typevar: Optional[Dict[TypeVar, Type]], + default_instance: Optional[T], + ) -> ParserSpecification: + """Create a parser definition from a callable.""" + + # Resolve generic types. + f, type_from_typevar = _resolver.resolve_generic_types(f) + if parent_type_from_typevar is not None: + for typevar, typ in type_from_typevar.items(): + if typ in parent_type_from_typevar: + type_from_typevar[typevar] = parent_type_from_typevar[typ] # type: ignore + + # Cycle detection. + if f in parent_classes: + raise _instantiators.UnsupportedTypeAnnotationError( + f"Found a cyclic dataclass dependency with type {f}." + ) + + # TODO: we are abusing the (minor) distinctions between types, classes, and + # callables throughout the code. This is mostly for legacy reasons, could be + # cleaned up. + parent_classes = parent_classes | {cast(Type, f)} + + args = [] + helptext_from_nested_class_field_name = {} + subparsers = None + for field in _fields_from_callable(f=f, default_instance=default_instance): + + field = dataclasses.replace( + field, + typ=type_from_typevar.get(field.typ, field.typ), # type: ignore + ) + if isinstance(field.typ, TypeVar): + # Found an unbound TypeVar. This could be because inheriting from + # generics is currently not implemented. It's unclear whether this is + # feasible, because generics are lost in the mro: + # https://github.com/python/typing/issues/777 + raise _instantiators.UnsupportedTypeAnnotationError( + f"Field {field.name} has an unbound TypeVar: {field.typ}. Note that" + " inheriting from generics is currently not implemented. It's" + " unclear whether this is feasible, because generics are lost in" + " the mro: https://github.com/python/typing/issues/777" + ) + + get_origin(field.typ) + # (1) Handle Unions over callables; these result in subparsers. + subparsers_attempt = SubparsersSpecification.from_field( + field, + type_from_typevar=type_from_typevar, + parent_classes=parent_classes, + ) + if subparsers_attempt is not None: + if subparsers is not None: + raise _instantiators.UnsupportedTypeAnnotationError( + "Only one set of subparsers is allowed." + ) + subparsers = subparsers_attempt + continue + + # (2) Handle primitive types. These produce a single argument! + if _is_parsable_type(field.typ): + args.append( + _arguments.ArgumentDefinition.from_field(field, type_from_typevar) + ) + continue + + # (3) Handle nested callables. + nested_parser = ParserSpecification.from_callable( + field.typ, + parent_classes=parent_classes, + parent_type_from_typevar=type_from_typevar, + default_instance=field.default, + ) + for arg in nested_parser.args: + args.append( + dataclasses.replace( + arg, + prefix=field.name + + _strings.NESTED_DATACLASS_DELIMETER + + arg.prefix, + ) + ) + for k, v in nested_parser.helptext_from_nested_class_field_name.items(): + helptext_from_nested_class_field_name[ + field.name + _strings.NESTED_DATACLASS_DELIMETER + k + ] = v + helptext_from_nested_class_field_name[field.name] = field.helptext + + return ParserSpecification( + f=f, + args=args, + helptext_from_nested_class_field_name=helptext_from_nested_class_field_name, + subparsers=subparsers, + ) def apply(self, parser: argparse.ArgumentParser) -> None: """Create defined arguments and subparsers.""" @@ -68,16 +292,18 @@ def format_group_name(nested_field_name: str, required: bool) -> str: prefix = termcolor.colored("required", attrs=["bold"]) else: prefix = termcolor.colored("optional", attrs=["bold", "dark"]) - suffix = " arguments" - if nested_field_name == "": - suffix = suffix[1:] - - return ( - prefix - + " " - + nested_field_name.replace("_", " ").replace(".", " • ") - + suffix - ) + suffix = termcolor.colored("arguments", attrs=["bold"]) + + if nested_field_name != "": + return " ".join( + [ + prefix, + termcolor.colored(nested_field_name, attrs=["bold"]), + suffix, + ] + ) + else: + return " ".join([prefix, suffix]) optional_group_from_prefix: Dict[str, argparse._ArgumentGroup] = { "": parser._action_groups[1], @@ -86,12 +312,19 @@ def format_group_name(nested_field_name: str, required: bool) -> str: "": parser.add_argument_group(format_group_name("", required=True)), } - # Break some API boundaries to rename the optional group, and + # Break some API boundaries to rename the optional group. parser._action_groups[1].title = format_group_name("", required=False) + positional_group = parser.add_argument_group( + termcolor.colored("positional arguments", attrs=["bold"]) + ) parser._action_groups = parser._action_groups[::-1] # Add each argument. for arg in self.args: + if arg.field.positional: + arg.add_argument(positional_group) + continue + if arg.required: target_groups, other_groups = ( required_group_from_prefix, @@ -108,7 +341,7 @@ def format_group_name(nested_field_name: str, required: bool) -> str: target_groups[arg.prefix] = parser.add_argument_group( format_group_name(nested_field_name, required=arg.required), # Add a description, but only to the first group for a field. - description=self.helptext_from_nested_dataclass_field_name[ + description=self.helptext_from_nested_class_field_name[ nested_field_name ] if arg.prefix not in other_groups @@ -134,101 +367,10 @@ def format_group_name(nested_field_name: str, required: bool) -> str: for name, subparser_def in self.subparsers.parser_from_name.items(): subparser = argparse_subparsers.add_parser( name, - description=_docstrings.get_dataclass_docstring(subparser_def.cls), + description=_docstrings.get_callable_description(subparser_def.f), ) subparser_def.apply(subparser) - @staticmethod - def from_dataclass( - cls: Type[T], - parent_dataclasses: Set[Type], - parent_type_from_typevar: Optional[Dict[TypeVar, Type]], - default_instance: Optional[T], - ) -> "ParserSpecification": - """Create a parser definition from a dataclass.""" - - assert _resolver.is_dataclass(cls) - - cls, type_from_typevar = _resolver.resolve_generic_classes(cls) - - if parent_type_from_typevar is not None: - for typevar, typ in type_from_typevar.items(): - if typ in parent_type_from_typevar: - type_from_typevar[typevar] = parent_type_from_typevar[typ] # type: ignore - - if cls in parent_dataclasses: - raise _instantiators.UnsupportedTypeAnnotationError( - f"Found a cyclic dataclass dependency with type {cls}." - ) - parent_dataclasses = parent_dataclasses | {cls} - - args = [] - helptext_from_nested_dataclass_field_name = {} - subparsers = None - for field in _resolver.resolved_fields(cls): # type: ignore - - # Ignore fields not in constructor - if not field.init: - continue - - field_default_instance = _get_field_default(field, default_instance) - nested_handler = _NestedDataclassHandler( - cls, - field, - type_from_typevar, - parent_dataclasses, - field_default_instance, - ) - - # Try to create subparsers from this field. - subparsers_out = nested_handler.handle_unions_over_dataclasses() - if subparsers_out is not None: - if subparsers is not None: - raise _instantiators.UnsupportedTypeAnnotationError( - "Only one subparser (union over dataclasses) is allowed per" - " class." - ) - - subparsers = subparsers_out - continue - - # Try to interpret field as a nested dataclass. - nested_out = nested_handler.handle_nested_dataclasses() - if nested_out is not None: - child_args, child_nested_field_names = nested_out - args.extend(child_args) - helptext_from_nested_dataclass_field_name.update( - child_nested_field_names - ) - helptext_from_nested_dataclass_field_name[ - field.name - ] = _docstrings.get_field_docstring(cls, field.name) - continue - - # Handle simple fields! - try: - arg = _arguments.ArgumentDefinition.make_from_field( - cls, - field, - type_from_typevar, - default=field_default_instance, - ) - except _instantiators.UnsupportedTypeAnnotationError as e: - # Catch unsupported annotation errors, and make the error message more - # informative. - raise _instantiators.UnsupportedTypeAnnotationError( - f"Error when parsing {cls.__name__}.{field.name} of type" - f" {field.type}: {e.args[0]}" - ) - args.append(arg) - - return ParserSpecification( - cls=cls, - args=args, - helptext_from_nested_dataclass_field_name=helptext_from_nested_dataclass_field_name, - subparsers=subparsers, - ) - @dataclasses.dataclass(frozen=True) class SubparsersSpecification: @@ -240,95 +382,41 @@ class SubparsersSpecification: required: bool default_instance: Optional[Any] - -@dataclasses.dataclass(frozen=True) -class _NestedDataclassHandler(Generic[T]): - """Helper for handling nested dataclasses, which are converted to either subparsers - or prefixed fields.""" - - cls: Type[T] - field: dataclasses.Field - type_from_typevar: Dict[TypeVar, Type] - parent_dataclasses: Set[Type] - default_instance: Optional[T] - - def handle_unions_over_dataclasses( - self, - ) -> Optional["SubparsersSpecification"]: - """Handle unions over dataclasses, which are converted to subparsers.. Returns - `None` if not applicable.""" - - # Union of dataclasses should create subparsers. - if get_origin(self.field.type) is not Union: + @staticmethod + def from_field( + field: Field, + type_from_typevar: Dict[TypeVar, Type], + parent_classes: Set[Type], + ) -> Optional[SubparsersSpecification]: + # Union of classes should create subparsers. + if get_origin(field.typ) is not Union: return None # We don't use sets here to retain order of subcommands. - options = [ - self.type_from_typevar.get(typ, typ) for typ in get_args(self.field.type) - ] + options = [type_from_typevar.get(typ, typ) for typ in get_args(field.typ)] options_no_none = [o for o in options if o != type(None)] # noqa - if len(options_no_none) < 2 or not all( - map(_resolver.is_dataclass, options_no_none) - ): + if len(options_no_none) < 2 or any(map(_is_parsable_type, options_no_none)): return None parser_from_name: Dict[str, ParserSpecification] = {} for option in options_no_none: subparser_name = _strings.subparser_name_from_type(option) - - parser_from_name[subparser_name] = ParserSpecification.from_dataclass( + parser_from_name[subparser_name] = ParserSpecification.from_callable( option, - self.parent_dataclasses, - parent_type_from_typevar=self.type_from_typevar, + parent_classes, + parent_type_from_typevar=type_from_typevar, default_instance=None, ) return SubparsersSpecification( - name=self.field.name, + name=field.name, # If we wanted, we could add information about the default instance # automatically, as is done for normal fields. But for now we just rely on # the user to include it in the docstring. - description=_docstrings.get_field_docstring(self.cls, self.field.name), + description=field.helptext, parser_from_name=parser_from_name, # Required if: type hint is not Optional[], or a default instance is # provided. - required=(options == options_no_none) and self.default_instance is None, - default_instance=self.default_instance, - ) - - def handle_nested_dataclasses( - self, - ) -> Optional[Tuple[List[_arguments.ArgumentDefinition], Dict[str, Optional[str]]]]: - """Handle nested dataclasses. Returns `None` if not applicable.""" - # Resolve field type - field_type = ( - self.type_from_typevar[self.field.type] # type: ignore - if self.field.type in self.type_from_typevar # type: ignore - else self.field.type + required=(options == options_no_none) and field.default is None, + default_instance=field.default, ) - if not _resolver.is_dataclass(field_type): - return None - - # Add arguments for nested dataclasses. - child_definition = ParserSpecification.from_dataclass( - field_type, - self.parent_dataclasses, - parent_type_from_typevar=self.type_from_typevar, - default_instance=self.default_instance, - ) - - child_args = child_definition.args - for i, arg in enumerate(child_args): - child_args[i] = dataclasses.replace( - arg, - prefix=self.field.name - + _strings.NESTED_DATACLASS_DELIMETER - + arg.prefix, - ) - - helptext_from_nested_dataclass_field_name = { - self.field.name + _strings.NESTED_DATACLASS_DELIMETER + x: y - for x, y in child_definition.helptext_from_nested_dataclass_field_name.items() - } - - return child_args, helptext_from_nested_dataclass_field_name diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 2d8d990d9..43e920eed 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -3,20 +3,23 @@ import copy import dataclasses import functools -from typing import Dict, List, Tuple, Type, TypeVar +from typing import Callable, Dict, List, Tuple, Type, TypeVar, Union from typing_extensions import get_args, get_origin, get_type_hints -def is_dataclass(cls: Type) -> bool: +def is_dataclass(cls: Union[Type, Callable]) -> bool: """Same as `dataclasses.is_dataclass`, but also handles generic aliases.""" origin_cls = get_origin(cls) return dataclasses.is_dataclass(cls if origin_cls is None else origin_cls) -def resolve_generic_classes( - cls: Type, -) -> Tuple[Type, Dict[TypeVar, Type]]: +TypeOrCallable = TypeVar("TypeOrCallable", Type, Callable) + + +def resolve_generic_types( + cls: TypeOrCallable, +) -> Tuple[TypeOrCallable, Dict[TypeVar, Type]]: """If the input is a class: no-op. If it's a generic alias: returns the origin class, and a mapping from typevars to concrete types.""" diff --git a/dcargs/_serialization.py b/dcargs/_serialization.py index 924430ba0..1559d94b4 100644 --- a/dcargs/_serialization.py +++ b/dcargs/_serialization.py @@ -43,7 +43,7 @@ def _get_contained_special_types_from_type( else _parent_contained_dataclasses ) - cls, type_from_typevar = _resolver.resolve_generic_classes(cls) + cls, type_from_typevar = _resolver.resolve_generic_types(cls) contained_dataclasses = {cls} diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 96357121c..9cfbc78d4 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -1,5 +1,4 @@ """Utilities for working with strings.""" - import enum import functools import re @@ -32,7 +31,7 @@ def hyphen_separated_from_camel_case(name: str) -> str: def subparser_name_from_type(cls: Type) -> str: - cls, type_from_typevar = _resolver.resolve_generic_classes(cls) + cls, type_from_typevar = _resolver.resolve_generic_types(cls) if len(type_from_typevar) == 0: assert hasattr(cls, "__name__") return hyphen_separated_from_camel_case(cls.__name__) # type: ignore diff --git a/dcargs/py.typed b/dcargs/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/0_simple_function.py b/examples/0_simple_function.py new file mode 100644 index 000000000..cd2398743 --- /dev/null +++ b/examples/0_simple_function.py @@ -0,0 +1,20 @@ +import dcargs + + +def main( + field1: str, + field2: int, + flag: bool = False, +) -> None: + """Function, whose arguments will be populated from a CLI interface. + + Args: + field1: First field. + field2: Second field. + flag: Boolean flag that we can set to true. + """ + print(field1, field2, flag) + + +if __name__ == "__main__": + dcargs.cli(main) diff --git a/examples/simple.py b/examples/1_simple_dataclass.py similarity index 74% rename from examples/simple.py rename to examples/1_simple_dataclass.py index 06691177f..90bebc3ff 100644 --- a/examples/simple.py +++ b/examples/1_simple_dataclass.py @@ -5,13 +5,16 @@ @dataclasses.dataclass class Args: + """Description. + This should show up in the helptext!""" + field1: str # A string field. field2: int # A numeric field. flag: bool = False # A boolean flag. if __name__ == "__main__": - args = dcargs.parse(Args) + args = dcargs.cli(Args) print(args) print() print(dcargs.to_yaml(args)) diff --git a/examples/2_enums_and_collections.py b/examples/2_enums_and_collections.py new file mode 100644 index 000000000..8c52badbb --- /dev/null +++ b/examples/2_enums_and_collections.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import dataclasses +import enum +import pathlib +from typing import Tuple + +import dcargs + + +def main( + source: pathlib.Path, + dest: pathlib.Path, + /, # Mark the end of positional arguments. + optimizer: OptimizerConfig, + force: bool = False, + verbose: bool = False, + background_rgb: Tuple[float, float, float] = (1.0, 0.0, 0.0), +) -> None: + """Command-line interface defined using a function signature. Note that this + docstring is parsed to generate helptext. + + Args: + source: Path to move data from. + dest: Path to move data to. + optimizer: Configuration for our optimizer object. + force: Do not prompt before overwriting. + verbose: Explain what is being done. + background_rgb: Background color. Red by default. + """ + print( + f"{source.absolute()=}" + "\n" + f"{dest.absolute()=}" + "\n" + f"{optimizer=}" + "\n" + f"{force=}" + "\n" + f"{verbose=}" + "\n" + f"{background_rgb=}" + ) + + +class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + +@dataclasses.dataclass(frozen=True) +class OptimizerConfig: + algorithm: OptimizerType = OptimizerType.ADAM + """Gradient-based optimizer to use.""" + + learning_rate: float = 3e-4 + """Learning rate to use.""" + + weight_decay: float = 1e-2 + """Coefficient for L2 regularization.""" + + +if __name__ == "__main__": + dcargs.cli(main) diff --git a/examples/flags.py b/examples/3_flags.py similarity index 77% rename from examples/flags.py rename to examples/3_flags.py index b0cfc7e58..f054280d9 100644 --- a/examples/flags.py +++ b/examples/3_flags.py @@ -1,4 +1,5 @@ import dataclasses +from typing import Optional import dcargs @@ -12,12 +13,13 @@ class Args: # Boolean values. flag: bool + optional_flag: Optional[bool] flag1: bool = False flag2: bool = True if __name__ == "__main__": - args = dcargs.parse(Args) + args = dcargs.cli(Args) print(args) print() print(dcargs.to_yaml(args)) diff --git a/examples/literals.py b/examples/4_literals.py similarity index 95% rename from examples/literals.py rename to examples/4_literals.py index 0676e8b56..0737e5add 100644 --- a/examples/literals.py +++ b/examples/4_literals.py @@ -24,7 +24,7 @@ class Args: if __name__ == "__main__": - args = dcargs.parse(Args) + args = dcargs.cli(Args) print(args) print() print(dcargs.to_yaml(args)) diff --git a/examples/5_simple_class.py b/examples/5_simple_class.py new file mode 100644 index 000000000..cdef1c978 --- /dev/null +++ b/examples/5_simple_class.py @@ -0,0 +1,23 @@ +import dcargs + + +class Args: + def __init__( + self, + field1: str, + field2: int, + flag: bool = False, + ): + """Arguments. + + Args: + field1: A string field. + field2: A numeric field. + flag: A boolean flag. + """ + self.data = [field1, field2, flag] + + +if __name__ == "__main__": + args = dcargs.cli(Args) + print(args.data) diff --git a/examples/nested.py b/examples/6_nested_dataclasses.py similarity index 86% rename from examples/nested.py rename to examples/6_nested_dataclasses.py index 12e7ce71b..c92f8af31 100644 --- a/examples/nested.py +++ b/examples/6_nested_dataclasses.py @@ -8,8 +8,6 @@ import dataclasses import enum -from typing_extensions import Annotated - import dcargs @@ -24,7 +22,7 @@ class OptimizerConfig: algorithm: OptimizerType = OptimizerType.ADAM # Learning rate to use. - learning_rate: Annotated[float, "Learning rate"] = 3e-4 + learning_rate: float = 3e-4 # Coefficient for L2 regularization. weight_decay: float = 1e-2 @@ -34,7 +32,7 @@ class OptimizerConfig: class ExperimentConfig: """A nested experiment configuration. Note that the argument parser description is pulled from this docstring by default, but can also be overrided with - `dcargs.parse`'s `description=` flag.""" + `dcargs.cli()`'s `description=` argument.""" # Experiment name to use. experiment_name: str @@ -48,6 +46,6 @@ class ExperimentConfig: if __name__ == "__main__": - config = dcargs.parse(ExperimentConfig) + config = dcargs.cli(ExperimentConfig) print(config) print(dcargs.to_yaml(config)) diff --git a/examples/7_subparsers.py b/examples/7_subparsers.py new file mode 100644 index 000000000..add12a267 --- /dev/null +++ b/examples/7_subparsers.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import dataclasses +from typing import Union + +import dcargs + + +def main(command: Union[Checkout, Commit]) -> None: + pass + + +@dataclasses.dataclass(frozen=True) +class Checkout: + """Checkout a branch.""" + + branch: str + + +@dataclasses.dataclass(frozen=True) +class Commit: + """Commit changes.""" + + message: str + all: bool = False + + +if __name__ == "__main__": + dcargs.cli(main) diff --git a/examples/generics_alt.py b/examples/8_generics.py similarity index 93% rename from examples/generics_alt.py rename to examples/8_generics.py index 77270a057..08321b79d 100644 --- a/examples/generics_alt.py +++ b/examples/8_generics.py @@ -30,5 +30,5 @@ class Args(Generic[ShapeType]): if __name__ == "__main__": - args = dcargs.parse(Args[Triangle]) + args = dcargs.cli(Args[Triangle]) print(args) diff --git a/examples/generics.py b/examples/generics.py deleted file mode 100644 index 819ca359c..000000000 --- a/examples/generics.py +++ /dev/null @@ -1,43 +0,0 @@ -import dataclasses -from typing import Generic, Optional, TypeVar - -import dcargs - -ScalarType = TypeVar("ScalarType") - - -@dataclasses.dataclass(frozen=True) -class Point3(Generic[ScalarType]): - x: ScalarType - y: ScalarType - z: ScalarType - frame_id: str - - -@dataclasses.dataclass(frozen=True) -class Triangle(Generic[ScalarType]): - a: Point3[ScalarType] - b: Point3[ScalarType] - c: Point3[ScalarType] - - -@dataclasses.dataclass(frozen=True) -class Args: - point_continuous: Point3[float] - point_discrete: Point3[int] - - triangle_continuous: Triangle[float] - triangle_discrete: Triangle[int] - - triangle_optional_coords: Triangle[Optional[float]] - - triangle_with_default: Triangle[int] = Triangle( - a=Point3(1, 2, 3, "world"), - b=Point3(1, 2, 3, "world"), - c=Point3(1, 2, 3, "world"), - ) - - -if __name__ == "__main__": - args = dcargs.parse(Args) - print(args) diff --git a/examples/subparsers.py b/examples/subparsers.py deleted file mode 100644 index 0e95f5d27..000000000 --- a/examples/subparsers.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import annotations - -import dataclasses -from typing import Union - -import dcargs - - -@dataclasses.dataclass(frozen=True) -class Args: - """Example of a version control-style subcommand interface.""" - - # Subcommand to use; we support "checkout" and "commit". - # - # If desired, we also support default values for subparsers, eg - # command: Union[Checkout, Commit] = Checkout("main") - command: Union[Checkout, Commit] - - -@dataclasses.dataclass(frozen=True) -class Checkout: - """Checkout a branch.""" - - branch: str - - -@dataclasses.dataclass(frozen=True) -class Commit: - """Commit changes.""" - - message: str - all: bool = False - - -if __name__ == "__main__": - args = dcargs.parse(Args) - print(args) diff --git a/setup.py b/setup.py index f849f32d4..32f8ff577 100644 --- a/setup.py +++ b/setup.py @@ -5,8 +5,8 @@ setup( name="dcargs", - version="0.0.21", - description="Portable, reusable, strongly typed CLIs from dataclass definitions", + version="0.1.0", + description="Strongly typed, zero effort CLIs", long_description=long_description, long_description_content_type="text/markdown", url="http://github.com/brentyi/dcargs", @@ -16,11 +16,17 @@ packages=find_packages(), package_data={"dcargs": ["py.typed"]}, python_requires=">=3.7", - install_requires=["typing_extensions>=4.0.0", "pyyaml", "termcolor"], + install_requires=[ + "docstring_parser", + "typing_extensions>=4.0.0", + "pyyaml", + "termcolor", + ], extras_require={ "testing": [ "pytest", "pytest-cov", + "attrs", ], "type-checking": [ "mypy", diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..7b7e4729b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,4 @@ +import sys + +if sys.version_info.major == 3 and sys.version_info.minor == 7: + collect_ignore_glob = ["*_ignore_py37.py"] diff --git a/tests/test_attrs.py b/tests/test_attrs.py new file mode 100644 index 000000000..dc2e373b2 --- /dev/null +++ b/tests/test_attrs.py @@ -0,0 +1,77 @@ +"""Tests for attrs. This is not officially supported, but should work.""" + +import contextlib +import io +import pathlib + +import attr +import pytest + +import dcargs + + +def test_attrs_basic(): + @attr.s + class ManyTypesA: + i: int = attr.ib() + s: str = attr.ib() + f: float = attr.ib() + p: pathlib.Path = attr.ib() + + # We can directly pass a dataclass to `dcargs.cli()`: + assert dcargs.cli( + ManyTypesA, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--p", + "~", + ], + ) == ManyTypesA(i=5, s="5", f=5.0, p=pathlib.Path("~")) + + +def test_attrs_defaults(): + @attr.s + class ManyTypesB: + i: int = attr.ib() + s: str = attr.ib() + f: float = attr.ib(default=1.0) + + # We can directly pass a dataclass to `dcargs.cli()`: + assert dcargs.cli( + ManyTypesB, + args=[ + "--i", + "5", + "--s", + "5", + ], + ) == ManyTypesB(i=5, s="5", f=1.0) + + +def test_attrs_helptext(): + @attr.s + class Helptext: + """This docstring should be printed as a description.""" + + x: int = attr.ib() # Documentation 1 + + # Documentation 2 + y: int = attr.ib() + + z: int = attr.ib(default=3) + """Documentation 3""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli(Helptext, args=["--help"]) + helptext = f.getvalue() + assert Helptext.__doc__ in helptext + assert ":\n --x INT Documentation 1\n" in helptext + assert "--y INT Documentation 2\n" in helptext + assert "--z INT Documentation 3 (default: 3)\n" in helptext diff --git a/tests/test_collections.py b/tests/test_collections.py index b8614adfd..c06fe2760 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -13,11 +13,11 @@ def test_tuples_fixed(): class A: x: Tuple[int, int, int] - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) def test_tuples_fixed_mixed(): @@ -25,11 +25,11 @@ def test_tuples_fixed_mixed(): class A: x: Tuple[int, str, int] - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=(1, "2", 3)) + assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, "2", 3)) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) def test_tuples_with_default(): @@ -37,10 +37,10 @@ def test_tuples_with_default(): class A: x: Tuple[int, int, int] = (0, 1, 2) - assert dcargs.parse(A, args=[]) == A(x=(0, 1, 2)) - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + assert dcargs.cli(A, args=[]) == A(x=(0, 1, 2)) + assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) def test_tuples_fixed_multitype(): @@ -48,11 +48,11 @@ def test_tuples_fixed_multitype(): class A: x: Tuple[int, str, float] - assert dcargs.parse(A, args=["--x", "1", "2", "3.5"]) == A(x=(1, "2", 3.5)) + assert dcargs.cli(A, args=["--x", "1", "2", "3.5"]) == A(x=(1, "2", 3.5)) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) def test_tuples_fixed_bool(): @@ -60,13 +60,13 @@ def test_tuples_fixed_bool(): class A: x: Tuple[bool, bool, bool] - assert dcargs.parse(A, args=["--x", "True", "True", "False"]) == A( + assert dcargs.cli(A, args=["--x", "True", "True", "False"]) == A( x=(True, True, False) ) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) def test_tuples_variable(): @@ -74,11 +74,11 @@ def test_tuples_variable(): class A: x: Tuple[int, ...] - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) def test_tuples_variable_bool(): @@ -86,13 +86,13 @@ def test_tuples_variable_bool(): class A: x: Tuple[bool, ...] - assert dcargs.parse(A, args=["--x", "True", "True", "False"]) == A( + assert dcargs.cli(A, args=["--x", "True", "True", "False"]) == A( x=(True, True, False) ) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) def test_tuples_variable_optional(): @@ -100,10 +100,10 @@ def test_tuples_variable_optional(): class A: x: Optional[Tuple[int, ...]] - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) - assert dcargs.parse(A, args=[]) == A(x=None) + dcargs.cli(A, args=["--x"]) + assert dcargs.cli(A, args=[]) == A(x=None) def test_sequences(): @@ -111,11 +111,11 @@ def test_sequences(): class A: x: Sequence[int] - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) def test_lists(): @@ -123,11 +123,11 @@ def test_lists(): class A: x: List[int] - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) def test_list_with_literal(): @@ -135,13 +135,13 @@ def test_list_with_literal(): class A: x: List[Literal[1, 2, 3]] - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x", "1", "2", "3", "4"]) + dcargs.cli(A, args=["--x", "1", "2", "3", "4"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) def test_list_with_enums(): @@ -154,15 +154,15 @@ class Color(enum.Enum): class A: x: List[Color] - assert dcargs.parse(A, args=["--x", "RED", "RED", "BLUE"]) == A( + assert dcargs.cli(A, args=["--x", "RED", "RED", "BLUE"]) == A( x=[Color.RED, Color.RED, Color.BLUE] ) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x", "RED", "RED", "YELLOW"]) + dcargs.cli(A, args=["--x", "RED", "RED", "YELLOW"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) def test_lists_with_default(): @@ -177,8 +177,8 @@ class A: default_factory=[Color.RED, Color.GREEN].copy ) - assert dcargs.parse(A, args=[]) == A(x=[Color.RED, Color.GREEN]) - assert dcargs.parse(A, args=["--x", "RED", "GREEN", "BLUE"]) == A( + assert dcargs.cli(A, args=[]) == A(x=[Color.RED, Color.GREEN]) + assert dcargs.cli(A, args=["--x", "RED", "GREEN", "BLUE"]) == A( x=[Color.RED, Color.GREEN, Color.BLUE] ) @@ -188,13 +188,13 @@ def test_lists_bool(): class A: x: List[bool] - assert dcargs.parse(A, args=["--x", "True", "False", "True"]) == A( + assert dcargs.cli(A, args=["--x", "True", "False", "True"]) == A( x=[True, False, True] ) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) def test_sets(): @@ -202,11 +202,11 @@ def test_sets(): class A: x: Set[int] - assert dcargs.parse(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) + assert dcargs.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) def test_sets_with_default(): @@ -214,10 +214,10 @@ def test_sets_with_default(): class A: x: Set[int] = dataclasses.field(default_factory={0, 1, 2}.copy) - assert dcargs.parse(A, args=[]) == A(x={0, 1, 2}) - assert dcargs.parse(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) + assert dcargs.cli(A, args=[]) == A(x={0, 1, 2}) + assert dcargs.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) + dcargs.cli(A, args=["--x"]) def test_optional_sequences(): @@ -225,10 +225,10 @@ def test_optional_sequences(): class A: x: Optional[Sequence[int]] - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) - assert dcargs.parse(A, args=[]) == A(x=None) + dcargs.cli(A, args=["--x"]) + assert dcargs.cli(A, args=[]) == A(x=None) def test_optional_lists(): @@ -236,7 +236,7 @@ def test_optional_lists(): class A: x: Optional[List[int]] - assert dcargs.parse(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x"]) - assert dcargs.parse(A, args=[]) == A(x=None) + dcargs.cli(A, args=["--x"]) + assert dcargs.cli(A, args=[]) == A(x=None) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 15bc8600c..4e0dffeb7 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -17,7 +17,8 @@ class ManyTypes: f: float p: pathlib.Path - assert dcargs.parse( + # We can directly pass a dataclass to `dcargs.cli()`: + assert dcargs.cli( ManyTypes, args=[ "--i", @@ -31,6 +32,43 @@ class ManyTypes: ], ) == ManyTypes(i=5, s="5", f=5.0, p=pathlib.Path("~")) + # We can directly pass a function to `dcargs.cli()`: + def function(i: int, s: str, f: float, p: pathlib.Path) -> ManyTypes: + return ManyTypes(i=i, s=s, f=f, p=p) + + assert dcargs.cli( + function, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--p", + "~", + ], + ) == ManyTypes(i=5, s="5", f=5.0, p=pathlib.Path("~")) + + # We can directly pass a generic class to `dcargs.cli()`: + class Wrapper: + def __init__(self, i: int, s: str, f: float, p: pathlib.Path): + self.inner = ManyTypes(i=i, s=s, f=f, p=p) + + assert dcargs.cli( + Wrapper, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--p", + "~", + ], + ).inner == ManyTypes(i=5, s="5", f=5.0, p=pathlib.Path("~")) + def test_init_false(): @dataclasses.dataclass @@ -41,7 +79,7 @@ class InitFalseDataclass: p: pathlib.Path ignored: str = dataclasses.field(default="hello", init=False) - assert dcargs.parse( + assert dcargs.cli( InitFalseDataclass, args=[ "--i", @@ -56,7 +94,7 @@ class InitFalseDataclass: ) == InitFalseDataclass(i=5, s="5", f=5.0, p=pathlib.Path("~")) with pytest.raises(SystemExit): - dcargs.parse( + dcargs.cli( InitFalseDataclass, args=["--i", "5", "--s", "5", "--f", "5", "--p", "~", "--ignored", "blah"], ) @@ -68,7 +106,7 @@ class A: x: int with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) def test_flag(): @@ -79,19 +117,19 @@ class A: x: bool with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) + dcargs.cli(A, args=[]) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x", "1"]) + dcargs.cli(A, args=["--x", "1"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x", "true"]) - assert dcargs.parse(A, args=["--x", "True"]) == A(True) + dcargs.cli(A, args=["--x", "true"]) + assert dcargs.cli(A, args=["--x", "True"]) == A(True) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x", "0"]) + dcargs.cli(A, args=["--x", "0"]) with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x", "false"]) - assert dcargs.parse(A, args=["--x", "False"]) == A(False) + dcargs.cli(A, args=["--x", "false"]) + assert dcargs.cli(A, args=["--x", "False"]) == A(False) def test_flag_default_false(): @@ -101,8 +139,8 @@ def test_flag_default_false(): class A: x: bool = False - assert dcargs.parse(A, args=[]) == A(False) - assert dcargs.parse(A, args=["--x"]) == A(True) + assert dcargs.cli(A, args=[]) == A(False) + assert dcargs.cli(A, args=["--x"]) == A(True) def test_flag_default_true(): @@ -112,8 +150,8 @@ def test_flag_default_true(): class A: x: bool = True - assert dcargs.parse(A, args=[]) == A(True) - assert dcargs.parse(A, args=["--no-x"]) == A(False) + assert dcargs.cli(A, args=[]) == A(True) + assert dcargs.cli(A, args=["--no-x"]) == A(False) def test_flag_default_true_nested(): @@ -127,8 +165,8 @@ class NestedDefaultTrue: class A: x: NestedDefaultTrue - assert dcargs.parse(A, args=[]) == A(NestedDefaultTrue(True)) - assert dcargs.parse(A, args=["--x.no-x"]) == A(NestedDefaultTrue(False)) + assert dcargs.cli(A, args=[]) == A(NestedDefaultTrue(True)) + assert dcargs.cli(A, args=["--x.no-x"]) == A(NestedDefaultTrue(False)) def test_default(): @@ -136,7 +174,7 @@ def test_default(): class A: x: int = 5 - assert dcargs.parse(A, args=[]) == A() + assert dcargs.cli(A, args=[]) == A() def test_default_factory(): @@ -144,7 +182,7 @@ def test_default_factory(): class A: x: int = dataclasses.field(default_factory=lambda: 5) - assert dcargs.parse(A, args=[]) == A() + assert dcargs.cli(A, args=[]) == A() def test_optional(): @@ -152,7 +190,7 @@ def test_optional(): class A: x: Optional[int] - assert dcargs.parse(A, args=[]) == A(x=None) + assert dcargs.cli(A, args=[]) == A(x=None) def test_enum(): @@ -169,10 +207,10 @@ class EnumClassA: class EnumClassB: color: Color = Color.GREEN - assert dcargs.parse(EnumClassA, args=["--color", "RED"]) == EnumClassA( + assert dcargs.cli(EnumClassA, args=["--color", "RED"]) == EnumClassA( color=Color.RED ) - assert dcargs.parse(EnumClassB, args=[]) == EnumClassB() + assert dcargs.cli(EnumClassB, args=[]) == EnumClassB() def test_literal(): @@ -180,9 +218,9 @@ def test_literal(): class A: x: Literal[0, 1, 2] - assert dcargs.parse(A, args=["--x", "1"]) == A(x=1) + assert dcargs.cli(A, args=["--x", "1"]) == A(x=1) with pytest.raises(SystemExit): - assert dcargs.parse(A, args=["--x", "3"]) + assert dcargs.cli(A, args=["--x", "3"]) def test_literal_enum(): @@ -195,10 +233,10 @@ class Color(enum.Enum): class A: x: Literal[Color.RED, Color.GREEN] - assert dcargs.parse(A, args=["--x", "RED"]) == A(x=Color.RED) - assert dcargs.parse(A, args=["--x", "GREEN"]) == A(x=Color.GREEN) + assert dcargs.cli(A, args=["--x", "RED"]) == A(x=Color.RED) + assert dcargs.cli(A, args=["--x", "GREEN"]) == A(x=Color.GREEN) with pytest.raises(SystemExit): - assert dcargs.parse(A, args=["--x", "BLUE"]) + assert dcargs.cli(A, args=["--x", "BLUE"]) def test_optional_literal(): @@ -206,10 +244,10 @@ def test_optional_literal(): class A: x: Optional[Literal[0, 1, 2]] - assert dcargs.parse(A, args=["--x", "1"]) == A(x=1) + assert dcargs.cli(A, args=["--x", "1"]) == A(x=1) with pytest.raises(SystemExit): - assert dcargs.parse(A, args=["--x", "3"]) - assert dcargs.parse(A, args=[]) == A(x=None) + assert dcargs.cli(A, args=["--x", "3"]) + assert dcargs.cli(A, args=[]) == A(x=None) def test_annotated(): @@ -219,7 +257,7 @@ def test_annotated(): class A: x: Annotated[int, "some label"] = 3 - assert dcargs.parse(A, args=["--x", "5"]) == A(x=5) + assert dcargs.cli(A, args=["--x", "5"]) == A(x=5) def test_annotated_optional(): @@ -229,8 +267,8 @@ def test_annotated_optional(): class A: x: Annotated[Optional[int], "some label"] = 3 - assert dcargs.parse(A, args=[]) == A(x=3) - assert dcargs.parse(A, args=["--x", "5"]) == A(x=5) + assert dcargs.cli(A, args=[]) == A(x=3) + assert dcargs.cli(A, args=["--x", "5"]) == A(x=5) def test_optional_annotated(): @@ -240,8 +278,8 @@ def test_optional_annotated(): class A: x: Optional[Annotated[int, "some label"]] = 3 - assert dcargs.parse(A, args=[]) == A(x=3) - assert dcargs.parse(A, args=["--x", "5"]) == A(x=5) + assert dcargs.cli(A, args=[]) == A(x=3) + assert dcargs.cli(A, args=["--x", "5"]) == A(x=5) def test_final(): @@ -251,7 +289,7 @@ def test_final(): class A: x: Final[int] = 3 - assert dcargs.parse(A, args=["--x", "5"]) == A(x=5) + assert dcargs.cli(A, args=["--x", "5"]) == A(x=5) def test_final_optional(): @@ -259,8 +297,8 @@ def test_final_optional(): class A: x: Final[Optional[int]] = 3 - assert dcargs.parse(A, args=[]) == A(x=3) - assert dcargs.parse(A, args=["--x", "5"]) == A(x=5) + assert dcargs.cli(A, args=[]) == A(x=3) + assert dcargs.cli(A, args=["--x", "5"]) == A(x=5) def test_classvar(): @@ -271,33 +309,8 @@ class A: x: ClassVar[int] = 5 with pytest.raises(SystemExit): - dcargs.parse(A, args=["--x", "1"]) - assert dcargs.parse(A, args=[]) == A() - - -# TODO: implement this! -# def test_optional_nested(): -# @dataclasses.dataclass -# class OptionalNestedChild: -# y: int -# z: int -# -# @dataclasses.dataclass -# class OptionalNested: -# x: int -# b: Optional[OptionalNestedChild] -# -# assert dcargs.parse(OptionalNested, args=["--x", "1"]) == OptionalNested( -# x=1, b=None -# ) -# with pytest.raises(SystemExit): -# dcargs.parse(OptionalNested, args=["--x", "1", "--b.y", "3"]) -# with pytest.raises(SystemExit): -# dcargs.parse(OptionalNested, args=["--x", "1", "--b.z", "3"]) -# -# assert dcargs.parse( -# OptionalNested, args=["--x", "1", "--b.y", "2", "--b.z", "3"] -# ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) + dcargs.cli(A, args=["--x", "1"]) + assert dcargs.cli(A, args=[]) == A() def test_parse_empty_description(): @@ -307,4 +320,4 @@ def test_parse_empty_description(): class A: x: int = 0 - assert dcargs.parse(A, description=None, args=[]) == A(x=0) + assert dcargs.cli(A, description=None, args=[]) == A(x=0) diff --git a/tests/test_dynamic_dataclasses.py b/tests/test_dynamic_dataclasses.py index 639f5b230..c46093914 100644 --- a/tests/test_dynamic_dataclasses.py +++ b/tests/test_dynamic_dataclasses.py @@ -10,5 +10,5 @@ def test_dynamic(): A = make_dataclass("A", [("b", B, field())]) with pytest.raises(SystemExit): - dcargs.parse(A, args=[]) - assert dcargs.parse(A, args=["--b.c", "5"]) == A(b=B(c=5)) + dcargs.cli(A, args=[]) + assert dcargs.cli(A, args=["--b.c", "5"]) == A(b=B(c=5)) diff --git a/tests/test_errors.py b/tests/test_errors.py index 3eff2ad77..9ca95844a 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -15,14 +15,14 @@ def test_choices_in_tuples(): class A: x: Tuple[bool, bool] - assert dcargs.parse(A, args=["--x", "True", "False"]) == A((True, False)) + assert dcargs.cli(A, args=["--x", "True", "False"]) == A((True, False)) # OK @dataclasses.dataclass class A: x: Tuple[bool, Literal["True", "False"]] - assert dcargs.parse(A, args=["--x", "True", "False"]) == A((True, "False")) + assert dcargs.cli(A, args=["--x", "True", "False"]) == A((True, "False")) # Not OK: same argument, different choices. @dataclasses.dataclass @@ -30,7 +30,7 @@ class A: x: Tuple[bool, Literal["True", "False", "None"]] with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.parse(A, args=["--x", "True", "False"]) + dcargs.cli(A, args=["--x", "True", "False"]) def test_nested_sequence_types(): @@ -41,7 +41,7 @@ class A: x: Tuple[Tuple[int, ...], ...] with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.parse(A, args=["--x", "0", "1"]) + dcargs.cli(A, args=["--x", "0", "1"]) def test_nested_optional_types(): @@ -57,7 +57,7 @@ class A: x: Tuple[Optional[int], ...] with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.parse(A, args=["--x", "0", "1"]) + dcargs.cli(A, args=["--x", "0", "1"]) def test_multiple_subparsers(): @@ -77,7 +77,7 @@ class MultipleSubparsers: y: Union[Subcommmand1, Subcommand2] with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.parse(MultipleSubparsers) + dcargs.cli(MultipleSubparsers) # Must be global. @@ -88,7 +88,7 @@ class _CycleDataclass: def test_cycle(): with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.parse(_CycleDataclass) + dcargs.cli(_CycleDataclass) def test_generic_inherited(): @@ -116,4 +116,4 @@ class ChildClass(UnrelatedParentClass, ActualParentClass[int]): pass with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.parse(ChildClass, args=["--x", 1, "--y", 2, "--z", 3]) + dcargs.cli(ChildClass, args=["--x", 1, "--y", 2, "--z", 3]) diff --git a/tests/test_forward_ref.py b/tests/test_forward_ref.py index d794ec183..05120796f 100644 --- a/tests/test_forward_ref.py +++ b/tests/test_forward_ref.py @@ -30,21 +30,21 @@ class C: def test_forward_ref_1(): - assert dcargs.parse(A1, args=["--x", "1", "b", "--y", "3"]) == A1(x=1, bc=B(y=3)) - assert dcargs.parse(A1, args=["--x", "1", "c", "--z", "3"]) == A1(x=1, bc=C(z=3)) + assert dcargs.cli(A1, args=["--x", "1", "b", "--y", "3"]) == A1(x=1, bc=B(y=3)) + assert dcargs.cli(A1, args=["--x", "1", "c", "--z", "3"]) == A1(x=1, bc=C(z=3)) with pytest.raises(SystemExit): - dcargs.parse(A1, args=["--x", "1", "b", "--z", "3"]) + dcargs.cli(A1, args=["--x", "1", "b", "--z", "3"]) with pytest.raises(SystemExit): - dcargs.parse(A1, args=["--x", "1", "c", "--y", "3"]) + dcargs.cli(A1, args=["--x", "1", "c", "--y", "3"]) def test_forward_ref_2(): - assert dcargs.parse(A2, args=["--x", "1", "b", "--y", "3"]) == A2(x=1, bc=B(y=3)) - assert dcargs.parse(A2, args=["--x", "1", "c", "--z", "3"]) == A2(x=1, bc=C(z=3)) + assert dcargs.cli(A2, args=["--x", "1", "b", "--y", "3"]) == A2(x=1, bc=B(y=3)) + assert dcargs.cli(A2, args=["--x", "1", "c", "--z", "3"]) == A2(x=1, bc=C(z=3)) with pytest.raises(SystemExit): - dcargs.parse(A2, args=["--x", "1", "b", "--z", "3"]) + dcargs.cli(A2, args=["--x", "1", "b", "--z", "3"]) with pytest.raises(SystemExit): - dcargs.parse(A2, args=["--x", "1", "c", "--y", "3"]) + dcargs.cli(A2, args=["--x", "1", "c", "--y", "3"]) diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index fd63d4a8b..18065c3ca 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -21,7 +21,7 @@ def test_tuple_generic_variable(): class TupleGenericVariable(Generic[ScalarType]): xyz: Tuple[ScalarType, ...] - assert dcargs.parse( + assert dcargs.cli( TupleGenericVariable[int], args=["--xyz", "1", "2", "3"] ) == TupleGenericVariable((1, 2, 3)) @@ -31,7 +31,7 @@ def test_tuple_generic_fixed(): class TupleGenericFixed(Generic[ScalarType]): xyz: Tuple[ScalarType, ScalarType, ScalarType] - assert dcargs.parse( + assert dcargs.cli( TupleGenericFixed[int], args=["--xyz", "1", "2", "3"] ) == TupleGenericFixed((1, 2, 3)) @@ -55,7 +55,7 @@ class SimpleGeneric: point_continuous: Point3[float] point_discrete: Point3[int] - parsed_instance = dcargs.parse( + parsed_instance = dcargs.cli( SimpleGeneric, args=[ "--point-continuous.x", @@ -84,7 +84,7 @@ class SimpleGeneric: with pytest.raises(SystemExit): # Accidentally pass in floats instead of ints for discrete - dcargs.parse( + dcargs.cli( SimpleGeneric, args=[ "--point-continuous.x", @@ -114,7 +114,7 @@ class Triangle(Generic[ScalarType]): b: Point3[ScalarType] c: Point3[ScalarType] - parsed_instance = dcargs.parse( + parsed_instance = dcargs.cli( Triangle[float], args=[ "--a.x", @@ -163,7 +163,7 @@ class Child: class DataclassGeneric(Generic[T]): child: T - parsed_instance = dcargs.parse( + parsed_instance = dcargs.cli( DataclassGeneric[Child], args=["--child.a", "5", "--child.b", "7"] ) assert parsed_instance == DataclassGeneric(Child(5, 7)) @@ -186,13 +186,13 @@ class CommandTwo: class Subparser(Generic[T1, T2]): command: Union[T1, T2] - parsed_instance = dcargs.parse( + parsed_instance = dcargs.cli( Subparser[CommandOne, CommandTwo], args="command-one --a 5".split(" ") ) assert parsed_instance == Subparser(CommandOne(5)) _check_serialization_identity(Subparser[CommandOne, CommandTwo], parsed_instance) - parsed_instance = dcargs.parse( + parsed_instance = dcargs.cli( Subparser[CommandOne, CommandTwo], args="command-two --b 7".split(" ") ) assert parsed_instance == Subparser(CommandTwo(7)) @@ -213,7 +213,7 @@ class Command(Generic[T]): class Subparser(Generic[T1, T2]): command: Union[T1, T2] - parsed_instance = dcargs.parse( + parsed_instance = dcargs.cli( Subparser[Command[int], Command[float]], args="command-int --a 5 3".split(" ") ) assert parsed_instance == Subparser(Command([5, 3])) and isinstance( @@ -223,7 +223,7 @@ class Subparser(Generic[T1, T2]): Subparser[Command[int], Command[float]], parsed_instance ) - parsed_instance = dcargs.parse( + parsed_instance = dcargs.cli( Subparser[Command[int], Command[float]], args="command-float --a 7 2".split(" ") ) assert parsed_instance == Subparser(Command([7.0, 2.0])) and isinstance( @@ -236,27 +236,29 @@ class Subparser(Generic[T1, T2]): # Not implemented. It's unclear whether this is feasible, because generics are lost in # the mro: https://github.com/python/typing/issues/777 -# -# def test_generic_inherited(): -# class UnrelatedParentClass: -# pass -# -# T = TypeVar("T") -# -# @dataclasses.dataclass -# class ActualParentClass(Generic[T]): -# x: T # Documentation 1 -# -# # Documentation 2 -# y: T -# -# z: T = 3 -# """Documentation 3""" -# -# @dataclasses.dataclass -# class ChildClass(UnrelatedParentClass, ActualParentClass[int]): -# pass -# -# assert dcargs.parse(ChildClass, args=["--x", 1, "--y", 2, "--z", 3]) == ChildClass( -# 1, 2, 3 -# ) + + +def test_generic_inherited(): + class UnrelatedParentClass: + pass + + T = TypeVar("T") + + @dataclasses.dataclass + class ActualParentClass(Generic[T]): + x: T # Documentation 1 + + # Documentation 2 + y: T + + z: T = 3 + """Documentation 3""" + + @dataclasses.dataclass + class ChildClass(UnrelatedParentClass, ActualParentClass[int]): + pass + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + assert dcargs.cli( + ChildClass, args=["--x", 1, "--y", 2, "--z", 3] + ) == ChildClass(1, 2, 3) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 92edb3f07..63b251758 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -27,7 +27,7 @@ class Helptext: f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(Helptext, args=["--help"]) + dcargs.cli(Helptext, args=["--help"]) helptext = f.getvalue() assert Helptext.__doc__ in helptext assert ":\n --x INT Documentation 1\n" in helptext @@ -53,7 +53,6 @@ class ActualParentClass: z: int = 3 def some_method(self) -> None: # noqa """Coverage stress test.""" - pass # fmt: on @dataclasses.dataclass @@ -63,7 +62,7 @@ class ChildClass(UnrelatedParentClass, ActualParentClass): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(ChildClass, args=["--help"]) + dcargs.cli(ChildClass, args=["--help"]) helptext = f.getvalue() assert ":\n --x INT Documentation 1\n" in helptext assert "--y INT Documentation 2\n" in helptext @@ -83,7 +82,7 @@ class HelptextWithVariousDefaults: f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(HelptextWithVariousDefaults, args=["--help"]) + dcargs.cli(HelptextWithVariousDefaults, args=["--help"]) helptext = f.getvalue() assert ( "show this help message and exit\n --x PATH (default:" @@ -110,7 +109,7 @@ class HelptextMultiline: f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(HelptextMultiline, args=["--help"]) + dcargs.cli(HelptextMultiline, args=["--help"]) helptext = f.getvalue() assert " --x INT Documentation 1\n" in helptext assert ( @@ -127,7 +126,7 @@ def test_grouped_helptext(): @dataclasses.dataclass class HelptextGrouped: x: int # Documentation 1 - + pass # Description of both y and z. y: int z: int = 3 @@ -135,7 +134,7 @@ class HelptextGrouped: f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(HelptextGrouped, args=["--help"]) + dcargs.cli(HelptextGrouped, args=["--help"]) helptext = f.getvalue() assert " --x INT Documentation 1\n" in helptext assert " --y INT Description of both y and z.\n" in helptext @@ -151,7 +150,7 @@ class Config: f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(Config, args=["--help"]) + dcargs.cli(Config, args=["--help"]) helptext = f.getvalue() assert " --x INT An optional variable. (default: None)\n" in helptext @@ -172,7 +171,7 @@ class HelptextHardString: f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(HelptextHardString, args=["--help"]) + dcargs.cli(HelptextHardString, args=["--help"]) helptext = f.getvalue() assert "--x Helptext. 2% milk.\n" in helptext @@ -194,7 +193,7 @@ class Child(Parent): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(Child, args=["--help"]) + dcargs.cli(Child, args=["--help"]) helptext = f.getvalue() assert ( "--x STR Helptext. (default: 'This docstring may be tougher to parse!')\n" @@ -224,7 +223,7 @@ class Child2(Parent2): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(Child2, args=["--help"]) + dcargs.cli(Child2, args=["--help"]) helptext = f.getvalue() assert ( "--x STR Helptext! (default: 'This docstring may be tougher to parse?')\n" @@ -240,7 +239,7 @@ class TupleHelptext: f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(TupleHelptext, args=["--help"]) + dcargs.cli(TupleHelptext, args=["--help"]) helptext = f.getvalue() assert "--x INT STR FLOAT\n" in helptext @@ -253,7 +252,7 @@ class TupleHelptextDefaults: f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(TupleHelptextDefaults, args=["--help"]) + dcargs.cli(TupleHelptextDefaults, args=["--help"]) helptext = f.getvalue() assert "--x INT STR STR (default: 5 'hello world' hello)\n" in helptext @@ -268,7 +267,7 @@ class GenericTupleHelptext(Generic[T]): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(GenericTupleHelptext[int], args=["--help"]) + dcargs.cli(GenericTupleHelptext[int], args=["--help"]) helptext = f.getvalue() assert "--x INT\n" in helptext @@ -283,7 +282,7 @@ class GenericTupleHelptext(Generic[T]): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(GenericTupleHelptext[int], args=["--help"]) + dcargs.cli(GenericTupleHelptext[int], args=["--help"]) helptext = f.getvalue() assert "--x INT INT INT\n" in helptext @@ -298,7 +297,7 @@ class GenericTupleHelptext(Generic[T]): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(GenericTupleHelptext[int], args=["--help"]) + dcargs.cli(GenericTupleHelptext[int], args=["--help"]) helptext = f.getvalue() assert "--x INT [INT ...]\n" in helptext @@ -312,7 +311,7 @@ class LiteralHelptext: f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(LiteralHelptext, args=["--help"]) + dcargs.cli(LiteralHelptext, args=["--help"]) helptext = f.getvalue() assert "--x {1,2,3} A number.\n" in helptext @@ -326,6 +325,6 @@ class OptionalLiteralHelptext: f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.parse(OptionalLiteralHelptext, args=["--help"]) + dcargs.cli(OptionalLiteralHelptext, args=["--help"]) helptext = f.getvalue() assert "--x {1,2,3} A number. (default: None)\n" in helptext diff --git a/tests/test_nested.py b/tests/test_nested.py index e6e48dd42..9348e6b21 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -16,11 +16,9 @@ class Nested: x: int b: B - assert dcargs.parse(Nested, args=["--x", "1", "--b.y", "3"]) == Nested( - x=1, b=B(y=3) - ) + assert dcargs.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) with pytest.raises(SystemExit): - dcargs.parse(Nested, args=["--x", "1"]) + dcargs.cli(Nested, args=["--x", "1"]) def test_nested_default(): @@ -35,10 +33,10 @@ class Nested: assert ( Nested(x=1, b=B(y=3)) - == dcargs.parse(Nested, args=["--x", "1", "--b.y", "3"]) - == dcargs.parse(Nested, args=[], default_instance=Nested(x=1, b=B(y=3))) + == dcargs.cli(Nested, args=["--x", "1", "--b.y", "3"]) + == dcargs.cli(Nested, args=[], default_instance=Nested(x=1, b=B(y=3))) ) - assert dcargs.parse(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=3)) + assert dcargs.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=3)) def test_default_nested(): @@ -51,10 +49,8 @@ class Nested: x: int b: B = B(y=5) - assert dcargs.parse(Nested, args=["--x", "1", "--b.y", "3"]) == Nested( - x=1, b=B(y=3) - ) - assert dcargs.parse(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) + assert dcargs.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) + assert dcargs.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) def test_double_default_nested(): @@ -71,10 +67,10 @@ class Grandparent: x: int b: Parent = Parent(Child(y=5)) - assert dcargs.parse(Grandparent, args=["--x", "1", "--b.c.y", "3"]) == Grandparent( + assert dcargs.cli(Grandparent, args=["--x", "1", "--b.c.y", "3"]) == Grandparent( x=1, b=Parent(Child(y=3)) ) - assert dcargs.parse(Grandparent, args=["--x", "1"]) == Grandparent( + assert dcargs.cli(Grandparent, args=["--x", "1"]) == Grandparent( x=1, b=Parent(Child(y=5)) ) @@ -89,10 +85,8 @@ class Nested: x: int b: B = dataclasses.field(default_factory=lambda: B(y=5)) - assert dcargs.parse(Nested, args=["--x", "1", "--b.y", "3"]) == Nested( - x=1, b=B(y=3) - ) - assert dcargs.parse(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) + assert dcargs.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) + assert dcargs.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) # TODO: implement this! @@ -107,15 +101,15 @@ class Nested: # x: int # b: Optional[OptionalNestedChild] # -# assert dcargs.parse(OptionalNested, args=["--x", "1"]) == OptionalNested( +# assert dcargs.cli(OptionalNested, args=["--x", "1"]) == OptionalNested( # x=1, b=None # ) # with pytest.raises(SystemExit): -# dcargs.parse(OptionalNested, args=["--x", "1", "--b.y", "3"]) +# dcargs.cli(OptionalNested, args=["--x", "1", "--b.y", "3"]) # with pytest.raises(SystemExit): -# dcargs.parse(OptionalNested, args=["--x", "1", "--b.z", "3"]) +# dcargs.cli(OptionalNested, args=["--x", "1", "--b.z", "3"]) # -# assert dcargs.parse( +# assert dcargs.cli( # OptionalNested, args=["--x", "1", "--b.y", "2", "--b.z", "3"] # ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) @@ -134,22 +128,22 @@ class Subparser: x: int bc: Union[HTTPServer, SMTPServer] - assert dcargs.parse( + assert dcargs.cli( Subparser, args=["--x", "1", "http-server", "--y", "3"] ) == Subparser(x=1, bc=HTTPServer(y=3)) - assert dcargs.parse( + assert dcargs.cli( Subparser, args=["--x", "1", "smtp-server", "--z", "3"] ) == Subparser(x=1, bc=SMTPServer(z=3)) with pytest.raises(SystemExit): # Missing subcommand. - dcargs.parse(Subparser, args=["--x", "1"]) + dcargs.cli(Subparser, args=["--x", "1"]) with pytest.raises(SystemExit): # Wrong field. - dcargs.parse(Subparser, args=["--x", "1", "http-server", "--z", "3"]) + dcargs.cli(Subparser, args=["--x", "1", "http-server", "--z", "3"]) with pytest.raises(SystemExit): # Wrong field. - dcargs.parse(Subparser, args=["--x", "1", "smtp-server", "--y", "3"]) + dcargs.cli(Subparser, args=["--x", "1", "smtp-server", "--y", "3"]) def test_subparser_with_default(): @@ -169,20 +163,20 @@ class DefaultSubparser: ) assert ( - dcargs.parse( + dcargs.cli( DefaultSubparser, args=["--x", "1", "default-http-server", "--y", "5"] ) - == dcargs.parse(DefaultSubparser, args=["--x", "1"]) + == dcargs.cli(DefaultSubparser, args=["--x", "1"]) == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=5)) ) - assert dcargs.parse( + assert dcargs.cli( DefaultSubparser, args=["--x", "1", "default-smtp-server", "--z", "3"] ) == DefaultSubparser(x=1, bc=DefaultSMTPServer(z=3)) assert ( - dcargs.parse( + dcargs.cli( DefaultSubparser, args=["--x", "1", "default-http-server", "--y", "8"] ) - == dcargs.parse( + == dcargs.cli( DefaultSubparser, args=[], default_instance=DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)), @@ -191,9 +185,9 @@ class DefaultSubparser: ) with pytest.raises(SystemExit): - dcargs.parse(DefaultSubparser, args=["--x", "1", "b", "--z", "3"]) + dcargs.cli(DefaultSubparser, args=["--x", "1", "b", "--z", "3"]) with pytest.raises(SystemExit): - dcargs.parse(DefaultSubparser, args=["--x", "1", "c", "--y", "3"]) + dcargs.cli(DefaultSubparser, args=["--x", "1", "c", "--y", "3"]) def test_optional_subparser(): @@ -210,23 +204,23 @@ class OptionalSubparser: x: int bc: Optional[Union[OptionalHTTPServer, OptionalSMTPServer]] - assert dcargs.parse( + assert dcargs.cli( OptionalSubparser, args=["--x", "1", "optional-http-server", "--y", "3"] ) == OptionalSubparser(x=1, bc=OptionalHTTPServer(y=3)) - assert dcargs.parse( + assert dcargs.cli( OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--z", "3"] ) == OptionalSubparser(x=1, bc=OptionalSMTPServer(z=3)) - assert dcargs.parse(OptionalSubparser, args=["--x", "1"]) == OptionalSubparser( + assert dcargs.cli(OptionalSubparser, args=["--x", "1"]) == OptionalSubparser( x=1, bc=None ) with pytest.raises(SystemExit): # Wrong field. - dcargs.parse( + dcargs.cli( OptionalSubparser, args=["--x", "1", "optional-http-server", "--z", "3"] ) with pytest.raises(SystemExit): # Wrong field. - dcargs.parse( + dcargs.cli( OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--y", "3"] ) diff --git a/tests/test_positional_ignore_py37.py b/tests/test_positional_ignore_py37.py new file mode 100644 index 000000000..0cdfa6c24 --- /dev/null +++ b/tests/test_positional_ignore_py37.py @@ -0,0 +1,57 @@ +from typing import List, Tuple + +import pytest + +import dcargs + + +def test_positional(): + def main( + x: int, + y: int, + /, + # Note: it's generally a bad idea to have a mutable object (like a list) as a + # default value. But it should still work. + z: List[int] = [1, 2, 3], + ) -> Tuple[int, int, int]: + """main. + + Args: + x: x + y: y + z: z + + Returns: + Tuple[int, int, int]: Output. + """ + return (x, y, z[0]) + + assert dcargs.cli(main, args="1 2 --z 3".split(" ")) == (1, 2, 3) + with pytest.raises(SystemExit): + assert dcargs.cli(main, args="--x 1 --y 2 --z 3".split(" ")) == (1, 2, 3) + + +def test_nested_positional(): + class A: + def __init__(self, a: int, b: int, /, c: int): + pass + + def nest1(a: int, b: int, thing: A, /, c: int): + return thing + + assert isinstance(dcargs.cli(nest1, args="0 1 2 3 --thing.c 4 --c 4".split(" ")), A) + with pytest.raises(SystemExit): + dcargs.cli(nest1, args="0 1 2 3 4 --thing.c 4 --c 4".split(" ")) + + +def test_nested_positional_alt(): + class B: + def __init__(self, a: int, b: int, /, c: int): + pass + + def nest2(a: int, b: int, /, thing: B, c: int): + return thing + + assert isinstance(dcargs.cli(nest2, args="0 1 2 3 --thing.c 4 --c 4".split(" ")), B) + with pytest.raises(SystemExit): + dcargs.cli(nest2, args="0 1 2 3 4 --thing.c 4 --c 4".split(" ")) From b0b22754cd52ad36de5868c732b98f50cc1992ac Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 19 Jun 2022 21:29:38 -0700 Subject: [PATCH 042/491] Code hygiene, error handling, tests --- .coverage | Bin 53248 -> 53248 bytes .gitignore | 1 + README.md | 12 ++-- dcargs/__init__.py | 26 ++------ dcargs/_arguments.py | 11 ++-- dcargs/_calling.py | 4 +- dcargs/_cli.py | 26 ++++++-- dcargs/_fields.py | 135 +++++++++++++++++++++++++++++++++++++++ dcargs/_instantiators.py | 24 ++++--- dcargs/_parsers.py | 135 ++++++--------------------------------- dcargs/_resolver.py | 2 - setup.py | 2 +- tests/test_errors.py | 16 +++++ tests/test_helptext.py | 1 - 14 files changed, 222 insertions(+), 173 deletions(-) create mode 100644 dcargs/_fields.py diff --git a/.coverage b/.coverage index 38c7029b9bba90d95f8bb110594d9023add2a7b8..273636ba87c95200f5b974068c51e3193a0f53b5 100644 GIT binary patch delta 668 zcmYk2Ur19?9LMk3xqo(d+xaO}#1viV%zxZDvC=_|UP3B%rI;exl`svlrfj{a;Z~$V z4@sN{Um|_-p`<>TwYzlF3umlkphBHtF0(nEG$Pn=nM>ID_A|_R#4PKRcy>@EX za(NkZE%dCGM;kONGE-eELlV0SAb2Uz_;l95)-)Jjw)+NJDz1Ae3U?4kQ2m=l`}wC);#8b6DcwJG_SkMBxU6 zp$9x*BO7FiIm{48zQQBft@L-t2X1S>lAU#7d_S}QFvHr#gzmYG!^hlped- zx;59h+r)6Wsd1BwNStvZg)M7V5%IE|<0lpd6qx2<8#dqze1cVY4bNd79zhJI*^0(M qGE|VFL}TOiY@ZM}!kh37)*;RIy22V3AqjC7z6%0YFlVL7h3enn(9vxG delta 680 zcmYk2e@IhN6vyxLzPIhY^qx78erP#TEK~Q?CCMoIBMK>avjRiZ$&evY)_hBIDw_}_ z8d&f^354x0h3XG$2?|4{z(7I^X&N<+#Dp@nR+>&`_nA@muX8?p&OKjFiyyc6ai3EZ z3XQpDi@DHJP!MPo+x2?{A|VJtAKZp=$OS|~DVSlR!&5qt;V!qH3s>Ma@jQhJv%2IE2v3gM*$?=6Ls3 zZFgiQ*6=pg&~b8RzA0xVg3~y46SvWBeztxmKG+*tSv%C}4GqPNi=OlI{fPsHa?ze@ zVX0Nmeg+gYkYBf)qs}(>o6WYZyYHe);^4871^-R^t0vcatl9V8^R(7I;XdaX?uu$6 zvAyObOlPz41xI>|X?bkYyUEeDyph$oq zf++leukZL0=KQD2%JhDtv=QSb&cZrrIQoQ{@RL;}iM60oGda AD*ylh diff --git a/.gitignore b/.gitignore index d2104f166..3abe36ed2 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,6 @@ __pycache__ .hypothesis .ipynb_checkpoints .cache +.coverage build/ dist/ diff --git a/README.md b/README.md index ba7e48a2e..6260ae0eb 100644 --- a/README.md +++ b/README.md @@ -93,16 +93,16 @@ Returns: Notably, `dcargs.cli()` supports _nested_ classes and dataclasses, which enable expressive hierarchical configuration objects built on standard Python features. -Our ultimate goal is an interface that's: +Our ultimate goal is the ability build interfaces that are: - **Low-effort.** Type annotations, docstrings, and default values can be used to automatically generate argument parsers with informative helptext. This includes bells and whistles like enums, containers, etc. - **Strongly typed.** Unlike dynamic configuration namespaces produced by libraries like `argparse`, `YACS`, `abseil`, `hydra`, or `ml_collections`, - statically typed outputs mean that IDE-assisted autocomplete, rename, - refactor, go-to-definition operations work out-of-the-box, as do static - checking tools like `mypy` and `pyright`. + typed outputs mean that IDE-assisted autocomplete, rename, refactor, + go-to-definition operations work out-of-the-box, as do static checking tools + like `mypy` and `pyright`. - **Modular.** Most approaches to configuration objects require a centralized definition of all configurable fields. Supporting hierarchically nested configuration classes/dataclasses, however, makes it easy to distribute @@ -301,8 +301,8 @@ serialization of arbitrary Python objects. ## Alternative tools -The core functionality of `dcargs` --- generating argument parsers from type -annotations --- can be found as a subset of the features offered by many other +The core functionality of `dcargs` — generating argument parsers from type +annotations — can be found as a subset of the features offered by many other libraries. A summary of some distinguishing features: | | Choices from literals | Generics | Docstrings as helptext | Nesting | Subparsers | Containers | diff --git a/dcargs/__init__.py b/dcargs/__init__.py index 7673553ce..a4e1a4656 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,30 +1,12 @@ -from ._cli import cli +from ._cli import cli, parse from ._instantiators import UnsupportedTypeAnnotationError from ._serialization import from_yaml, to_yaml __all__ = [ - "UnsupportedTypeAnnotationError", "cli", + # Deprecated. + # "parse", + "UnsupportedTypeAnnotationError", "from_yaml", "to_yaml", ] - -from typing import TYPE_CHECKING - -if not TYPE_CHECKING: - - def parse(*args, **kwargs): - # API breaking transition plan: - # (v0.1.0) Rename dcargs.parse() to dcargs.cli(). Keep the former as an alias. - # ( ) Enable deprecation warning. - # ( ) Remove dcargs.parse() alias. - # - # import warnings - # - # warnings.warn( - # "`dcargs.parse()` has been renamed `dcargs.cli()`. It will be removed" - # " soon.", - # DeprecationWarning, - # stacklevel=2, - # ) - return cli(*args, **kwargs) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 9229f66bb..87ca80383 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -4,12 +4,9 @@ import dataclasses import enum import shlex -from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Tuple, Type, TypeVar, Union +from typing import Any, Dict, Optional, Set, Tuple, Type, TypeVar, Union -from . import _instantiators - -if TYPE_CHECKING: - from . import _parsers +from . import _fields, _instantiators @dataclasses.dataclass(frozen=True) @@ -18,7 +15,7 @@ class ArgumentDefinition: add_argument() method.""" prefix: str # Prefix for nesting. - field: _parsers.Field # Corresponding dataclass field. + field: _fields.Field # Corresponding dataclass field. # Action that is called on parsed arguments. This handles conversions from strings # to our desired types. @@ -77,7 +74,7 @@ def get_flag(self) -> str: @staticmethod def from_field( - field: _parsers.Field, + field: _fields.Field, type_from_typevar: Dict[TypeVar, Type], ) -> ArgumentDefinition: """""" diff --git a/dcargs/_calling.py b/dcargs/_calling.py index b89ad5009..733f6648e 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -7,7 +7,7 @@ from typing_extensions import get_args -from . import _parsers, _resolver, _strings +from . import _fields, _parsers, _resolver, _strings class FieldActionValueError(Exception): @@ -46,7 +46,7 @@ def get_value_from_arg(arg: str) -> Any: for arg in parser_definition.args: arg_from_prefixed_field_name[arg.prefix + arg.field.name] = arg - for field in _parsers._fields_from_callable(f, default_instance=None): # type: ignore + for field in _fields.field_list_from_callable(f, default_instance=None): # type: ignore value: Any prefixed_field_name = field_name_prefix + field.name diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 035161604..d5abf9123 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -1,11 +1,30 @@ import argparse from typing import Callable, Optional, Sequence, TypeVar -from . import _calling, _docstrings, _parsers, _resolver +from . import _calling, _parsers, _resolver T = TypeVar("T") +def parse( + f: Callable[..., T], + *, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + default_instance: Optional[T] = None, +) -> T: # pragma: no cover + """Deprecated alias for `dcargs.cli()`.""" + # import warnings + # + # warnings.warn( + # "`dcargs.parse()` has been renamed `dcargs.cli()`. It will be removed" + # " soon.", + # DeprecationWarning, + # stacklevel=2, + # ) + return cli(f, description=description, args=args, default_instance=default_instance) + + def cli( f: Callable[..., T], *, @@ -67,17 +86,14 @@ def cli( ), "Default instance specification is only supported for dataclasses!" parser_definition = _parsers.ParserSpecification.from_callable( f, + description=description, parent_classes=set(), # Used for recursive calls. parent_type_from_typevar=None, # Used for recursive calls. default_instance=default_instance, # Overrides for default values. ) - if description is None: - description = _docstrings.get_callable_description(f) - # Parse using argparse! parser = argparse.ArgumentParser( - description=description, formatter_class=argparse.RawTextHelpFormatter, ) parser_definition.apply(parser) diff --git a/dcargs/_fields.py b/dcargs/_fields.py new file mode 100644 index 000000000..7b8c581fa --- /dev/null +++ b/dcargs/_fields.py @@ -0,0 +1,135 @@ +"""Abstractions for pulling out 'field' abstractions, which specify inputs, from +general callables.""" +import dataclasses +import inspect +import warnings +from typing import Any, Callable, List, Optional, Type, TypeVar + +import docstring_parser +from typing_extensions import get_type_hints + +from . import _docstrings, _resolver + + +@dataclasses.dataclass(frozen=True) +class Field: + name: str + typ: Type + default: Any + helptext: Optional[str] + positional: bool + + +T = TypeVar("T") + + +def field_list_from_callable( + f: Callable[..., T], default_instance: Optional[T] +) -> List[Field]: + """Generate a list of generic 'field' objects corresponding to an input callable. + + `f` can be from a dataclass type, regular class type, or function.""" + + # Unwrap generics. + f, _unused_type_from_typevar = _resolver.resolve_generic_types(f) + + # Handling for class inputs vs function inputs. + ignore_self = False + cls: Optional[Type] = None + if isinstance(f, type): + # If `f` is a type: + # 1. Set cls to the type. + # 2. Consider `f` to be `cls.__init__`. + cls = f + f = cls.__init__ # type: ignore + ignore_self = True + + # Get type annotations, docstrings. + hints = get_type_hints(f) + docstring = inspect.getdoc(f) + docstring_from_arg_name = {} + if docstring is not None: + for param_doc in docstring_parser.parse(docstring).params: + docstring_from_arg_name[param_doc.arg_name] = param_doc.description + del docstring + + # Special handling for dataclasses: take `field(default_factory=...)` and + # `default_instance` input into acocunt. + default_from_dataclass_field_name = {} + if cls is not None and _resolver.is_dataclass(cls): + for field in _resolver.resolved_fields(cls): + default_from_dataclass_field_name[ + field.name + ] = _get_dataclass_field_default(field, default_instance) + else: + assert ( + default_instance is None + ), "`default_instance` is only supported for dataclass types." + + # Generate field list from function signature. + field_list = [] + for param in inspect.signature(f).parameters.values(): + # For `__init__`, skip self parameter. + if ignore_self: + ignore_self = False + continue + + # Get default value. + default = default_from_dataclass_field_name.get(param.name, param.default) + if default is inspect.Parameter.empty: + default = None + + # Get helptext from docstring. + helptext = docstring_from_arg_name.get(param.name) + if cls is not None: + helptext = _docstrings.get_field_docstring(cls, param.name) + + field_list.append( + Field( + name=param.name, + # Note that param.annotation does not resolve forward references. + typ=hints[param.name], + default=default, + helptext=helptext, + positional=param.kind is inspect.Parameter.POSITIONAL_ONLY, + ) + ) + return field_list + + +def _ensure_dataclass_instance_used_as_default_is_frozen( + field: dataclasses.Field, default_instance: Any +) -> None: + """Ensure that a dataclass type used directly as a default value is marked as + frozen.""" + assert dataclasses.is_dataclass(default_instance) + cls = type(default_instance) + if not cls.__dataclass_params__.frozen: + warnings.warn( + f"Mutable type {cls} is used as a default value for `{field.name}`. This is" + " dangerous! Consider using `dataclasses.field(default_factory=...)` or" + f" marking {cls} as frozen." + ) + + +def _get_dataclass_field_default( + field: dataclasses.Field, parent_default_instance: Any +) -> Optional[Any]: + """Helper for getting the default instance for a field.""" + field_default_instance = None + if field.default is not dataclasses.MISSING: + # Populate default from usual default value, or + # `dataclasses.field(default=...)`. + field_default_instance = field.default + if dataclasses.is_dataclass(field_default_instance): + _ensure_dataclass_instance_used_as_default_is_frozen( + field, field_default_instance + ) + elif field.default_factory is not dataclasses.MISSING: + # Populate default from `dataclasses.field(default_factory=...)`. + field_default_instance = field.default_factory() + + if parent_default_instance is not None: + # Populate default from some parent, eg `default_instance` in `dcargs.cli()`. + field_default_instance = getattr(parent_default_instance, field.name) + return field_default_instance diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index 2b7332e92..d1c591a54 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -38,7 +38,7 @@ import enum from typing import Any, Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union -from typing_extensions import Final, Literal, get_args, get_origin +from typing_extensions import Annotated, Final, Literal, get_args, get_origin from . import _strings @@ -115,9 +115,9 @@ def _instantiator_from_container_type( if type_origin is None: return None - # Unwrap Final types. - if type_origin is Final: - (contained_type,) = get_args(typ) + # Unwrap Annotated and Final types. + if type_origin in (Annotated, Final): + contained_type = get_args(typ)[0] return instantiator_from_type(contained_type, type_from_typevar) # List, tuples, and sequences. @@ -192,9 +192,12 @@ def _instantiator_from_container_type( # Optionals. if type_origin is Union: options = set(get_args(typ)) - assert ( - len(options) == 2 and type(None) in options - ), "Union must be either over dataclasses (for subparsers) or Optional" + if len(options) != 2 or type(None) not in options: + # Note that the subparsers logic happens much earlier. + raise UnsupportedTypeAnnotationError( + "Union must be either over dataclasses (for subparsers) or Optional" + " (Union[T, None])" + ) (typ,) = options - {type(None)} instantiator, metadata = _instantiator_from_type_inner( typ, type_from_typevar, allow_sequences=True @@ -204,10 +207,11 @@ def _instantiator_from_container_type( # Literals. if type_origin is Literal: choices = get_args(typ) + if not len(set(map(type, choices))) == 1: + raise UnsupportedTypeAnnotationError( + "All choices in literal must have the same type!" + ) contained_type = type(next(iter(choices))) - assert all( - map(lambda c: type(c) == contained_type, choices) - ), "All choices in literal must have the same type!" if issubclass(contained_type, enum.Enum): choices = tuple(map(lambda x: x.name, choices)) instantiator, metadata = _instantiator_from_type_inner( diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index a6af5f2a9..04102626f 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -7,7 +7,6 @@ import dataclasses import inspect import itertools -import warnings from typing import ( Any, Callable, @@ -20,34 +19,17 @@ TypeVar, Union, cast, - get_type_hints, ) -import docstring_parser import termcolor import typing_extensions from typing_extensions import get_args, get_origin -from . import _arguments, _docstrings, _instantiators, _resolver, _strings +from . import _arguments, _docstrings, _fields, _instantiators, _resolver, _strings T = TypeVar("T") -def _ensure_dataclass_instance_used_as_default_is_frozen( - field: dataclasses.Field, default_instance: Any -) -> None: - """Ensure that a dataclass type used directly as a default value is marked as - frozen.""" - assert dataclasses.is_dataclass(default_instance) - cls = type(default_instance) - if not cls.__dataclass_params__.frozen: - warnings.warn( - f"Mutable type {cls} is used as a default value for `{field.name}`. This is" - " dangerous! Consider using `dataclasses.field(default_factory=...)` or" - f" marking {cls} as frozen." - ) - - _known_parsable_types = set( filter( lambda x: isinstance(x, Hashable), # type: ignore @@ -88,106 +70,19 @@ def _is_parsable_type(typ: Any) -> bool: return False -def _get_field_default( - field: dataclasses.Field, parent_default_instance: Any -) -> Optional[Any]: - """Helper for getting the default instance for a field.""" - field_default_instance = None - if field.default is not dataclasses.MISSING: - # Populate default from usual default value, or - # `dataclasses.field(default=...)`. - field_default_instance = field.default - if dataclasses.is_dataclass(field_default_instance): - _ensure_dataclass_instance_used_as_default_is_frozen( - field, field_default_instance - ) - elif field.default_factory is not dataclasses.MISSING: - # Populate default from `dataclasses.field(default_factory=...)`. - field_default_instance = field.default_factory() - - if parent_default_instance is not None: - # Populate default from some parent, eg `default_instance` in `dcargs.cli()`. - field_default_instance = getattr(parent_default_instance, field.name) - return field_default_instance - - -@dataclasses.dataclass(frozen=True) -class Field: - name: str - typ: Type - default: Any - helptext: Optional[str] - positional: bool - - -def _fields_from_callable(f: Callable, default_instance: Any) -> List[Field]: - """Generate a list of generic 'field' objects corresponding to an input callable. - - `f` can be from a dataclass type, regular class type, or function.""" - out = [] - f, type_from_typevar = _resolver.resolve_generic_types(f) - if _resolver.is_dataclass(f): - for field in _resolver.resolved_fields(f): - if not field.init: - continue - out.append( - Field( - name=field.name, - typ=field.type, - default=_get_field_default(field, default_instance), - helptext=_docstrings.get_field_docstring(cast(Type, f), field.name), - positional=False, - ) - ) - else: - if isinstance(f, type): - hints = get_type_hints(f.__init__) # type: ignore - docstring = inspect.getdoc(f.__init__) # type: ignore - else: - hints = get_type_hints(f) - docstring = inspect.getdoc(f) - - docstring_from_name = {} - if docstring is not None: - for param_doc in docstring_parser.parse(docstring).params: - docstring_from_name[param_doc.arg_name] = param_doc.description - - for param in inspect.signature(f).parameters.values(): - field_docstring = ( - _docstrings.get_field_docstring(cast(Type, f), param.name) - if isinstance(f, type) and hasattr(f, "mro") - else None - ) - out.append( - Field( - name=param.name, - # Note that param.annotation does not resolve forward references. - typ=hints[param.name], - default=param.default - if param.default is not inspect.Parameter.empty - else None, - helptext=docstring_from_name.get( - param.name, - field_docstring, - ), - positional=param.kind is inspect.Parameter.POSITIONAL_ONLY, - ) - ) - return out - - @dataclasses.dataclass(frozen=True) class ParserSpecification: """Each parser contains a list of arguments and optionally some subparsers.""" - f: Callable + description: str args: List[_arguments.ArgumentDefinition] helptext_from_nested_class_field_name: Dict[str, Optional[str]] subparsers: Optional[SubparsersSpecification] @staticmethod def from_callable( - f: Callable, + f: Callable[..., T], + description: Optional[str], parent_classes: Set[Type], parent_type_from_typevar: Optional[Dict[TypeVar, Type]], default_instance: Optional[T], @@ -215,7 +110,10 @@ def from_callable( args = [] helptext_from_nested_class_field_name = {} subparsers = None - for field in _fields_from_callable(f=f, default_instance=default_instance): + field_list = _fields.field_list_from_callable( + f=f, default_instance=default_instance + ) + for field in field_list: field = dataclasses.replace( field, @@ -258,6 +156,7 @@ def from_callable( # (3) Handle nested callables. nested_parser = ParserSpecification.from_callable( field.typ, + description=None, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, default_instance=field.default, @@ -278,7 +177,9 @@ def from_callable( helptext_from_nested_class_field_name[field.name] = field.helptext return ParserSpecification( - f=f, + description=description + if description is not None + else _docstrings.get_callable_description(f), args=args, helptext_from_nested_class_field_name=helptext_from_nested_class_field_name, subparsers=subparsers, @@ -287,6 +188,8 @@ def from_callable( def apply(self, parser: argparse.ArgumentParser) -> None: """Create defined arguments and subparsers.""" + parser.description = self.description + def format_group_name(nested_field_name: str, required: bool) -> str: if required: prefix = termcolor.colored("required", attrs=["bold"]) @@ -365,10 +268,7 @@ def format_group_name(nested_field_name: str, required: bool) -> str: metavar=metavar, ) for name, subparser_def in self.subparsers.parser_from_name.items(): - subparser = argparse_subparsers.add_parser( - name, - description=_docstrings.get_callable_description(subparser_def.f), - ) + subparser = argparse_subparsers.add_parser(name) subparser_def.apply(subparser) @@ -384,7 +284,7 @@ class SubparsersSpecification: @staticmethod def from_field( - field: Field, + field: _fields.Field, type_from_typevar: Dict[TypeVar, Type], parent_classes: Set[Type], ) -> Optional[SubparsersSpecification]: @@ -403,7 +303,8 @@ def from_field( subparser_name = _strings.subparser_name_from_type(option) parser_from_name[subparser_name] = ParserSpecification.from_callable( option, - parent_classes, + description=None, + parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, default_instance=None, ) diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 43e920eed..90f003e13 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -2,7 +2,6 @@ import copy import dataclasses -import functools from typing import Callable, Dict, List, Tuple, Type, TypeVar, Union from typing_extensions import get_args, get_origin, get_type_hints @@ -33,7 +32,6 @@ def resolve_generic_types( return cls, {} -@functools.lru_cache(maxsize=16) def resolved_fields(cls: Type) -> List[dataclasses.Field]: """Similar to dataclasses.fields, but resolves forward references.""" diff --git a/setup.py b/setup.py index 32f8ff577..3d70c8bb5 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.1.0", + version="0.1.1", description="Strongly typed, zero effort CLIs", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_errors.py b/tests/test_errors.py index 9ca95844a..5b099f2ec 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -60,6 +60,22 @@ class A: dcargs.cli(A, args=["--x", "0", "1"]) +def test_unsupported_union(): + def main(x: Union[int, str]) -> None: + return + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli(main) + + +def test_unsupported_literal(): + def main(x: Literal[0, "5"]) -> None: + return + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli(main) + + def test_multiple_subparsers(): """argparse doesn't support multiple subparsers.""" diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 63b251758..fe9d32e42 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -126,7 +126,6 @@ def test_grouped_helptext(): @dataclasses.dataclass class HelptextGrouped: x: int # Documentation 1 - pass # Description of both y and z. y: int z: int = 3 From 0452a91246701b9fe4a32fd53c1b39e40d7db986 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 21 Jun 2022 18:24:07 -0700 Subject: [PATCH 043/491] Much more thorough checks for unsupported types --- .coverage | Bin 53248 -> 53248 bytes dcargs/_instantiators.py | 62 +++++++++++++++++++++++++++++-- dcargs/_parsers.py | 77 ++++++++++++++++++++++----------------- setup.py | 4 +- tests/test_dcargs.py | 12 +++++- tests/test_errors.py | 54 ++++++++++++++++++++++++--- 6 files changed, 163 insertions(+), 46 deletions(-) diff --git a/.coverage b/.coverage index 273636ba87c95200f5b974068c51e3193a0f53b5..cd4de178a5a90e4571a00f7f517c61edc5ed8350 100644 GIT binary patch delta 171 zcmV;c095~gpaX!Q1F&x|3oZ4$iX?5U>uD){Y)6D+&Yw30MjO0002+0iY`4XC4mV;e7b>a`--ubKct@FWVo_ z@yE^O`&!`V2a_6)B{44u1OW+P2%yS-=I_tX{7JVTZCu`S Z-jd(%+&7rlf5r@xZ;yQeva{`v6F{pAN-Y2Y delta 163 zcmV;U09^lopaX!Q1F&x|3o$t$GBG+dFgi3bvwAO#P#D$#5BLw_56cg;4~`Fj4{r}Y z4;l{i4%V{~5V#JL&W;`?Bnkup2}%kA0002+sv>^I0X&@lah&ts{&?B`c#c1AF5lMz zKR=Thk0mfJ2m}EMUI^+309XDXoH1PP4d8Ro*M&jv&E@y!`;%@z+PJ*uyd}Tixo T` type converter, as expected by argparse. + if typ in _builtin_set: + pass + elif not callable(typ): + raise UnsupportedTypeAnnotationError( + f"Expected {typ} to be a `(arg: str) -> T` type converter, but is not" + " callable." + ) + else: + param_count = 0 + has_var_positional = False + signature = inspect.signature(typ) + for i, param in enumerate(signature.parameters.values()): + if i == 0 and param.annotation not in (str, inspect.Parameter.empty): + raise UnsupportedTypeAnnotationError( + f"Expected {typ} to be a `(arg: str) -> T` type converter, but got" + f" {signature}. You may have a nested type in a container, which is" + " unsupported." + ) + if param.kind is inspect.Parameter.VAR_POSITIONAL: + has_var_positional = True + elif param.default is inspect.Parameter.empty and param.kind is not ( + inspect.Parameter.VAR_KEYWORD + ): + param_count += 1 + + # Raise an error if parameters look wrong. + if not (param_count == 1 or (param_count == 0 and has_var_positional)): + raise UnsupportedTypeAnnotationError( + f"Expected {typ} to be a `(arg: str) -> T` type converter, but got" + f" {signature}. You may have a nested type in a container, which is" + " unsupported." + ) + + # Special case `choices` for some types, as implemented in `instance_from_string()`. auto_choices: Optional[Tuple[str, ...]] = None if typ is bool: auto_choices = ("True", "False") elif issubclass(typ, enum.Enum): auto_choices = tuple(x.name for x in typ) + return lambda arg: _strings.instance_from_string(typ, arg), InstantiatorMetadata( nargs=None, metavar=typ.__name__.upper(), diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 04102626f..e6aa50232 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -42,32 +42,39 @@ ) -def _is_parsable_type(typ: Any) -> bool: - """Heuristics for determining whether a type can be treated as a single argument. - This is true for built-in types like ints, strings, bools, etc, as well as - pathlib.Path.""" +def _is_possibly_nested_type(typ: Any) -> bool: + """Heuristics for determining whether a type can be treated as a 'nested type', + where a single field has multiple corresponding argumentsi (eg for nested + dataclasses or classes). + + Examples of when we return False: int, str, List[int], List[str], pathlib.Path, etc. + """ origin = get_origin(typ) if origin is not None: typ = origin + # Nested types should be callable. + if not callable(typ): + return False + # Known parsable types: builtins like int/str/float/bool, collections, annotations. if typ in _known_parsable_types: - return True + return False # Simple heuristic: dataclasses should be treated as nested objects and are not # parsable. if dataclasses.is_dataclass(typ): - return False + return True # Non-parsable types like nested (data)classes should have fully type-annotated # inputs. If any inputs are unannotated (for example, in the case of pathlib.Path), # we can assume the type is parsable. for param in inspect.signature(typ).parameters.values(): if param.annotation is inspect.Parameter.empty: - return True + return False - return False + return True @dataclasses.dataclass(frozen=True) @@ -146,35 +153,35 @@ def from_callable( subparsers = subparsers_attempt continue - # (2) Handle primitive types. These produce a single argument! - if _is_parsable_type(field.typ): - args.append( - _arguments.ArgumentDefinition.from_field(field, type_from_typevar) + # (2) Handle nested callables. + if _is_possibly_nested_type(field.typ): + nested_parser = ParserSpecification.from_callable( + field.typ, + description=None, + parent_classes=parent_classes, + parent_type_from_typevar=type_from_typevar, + default_instance=field.default, ) + for arg in nested_parser.args: + args.append( + dataclasses.replace( + arg, + prefix=field.name + + _strings.NESTED_DATACLASS_DELIMETER + + arg.prefix, + ) + ) + for k, v in nested_parser.helptext_from_nested_class_field_name.items(): + helptext_from_nested_class_field_name[ + field.name + _strings.NESTED_DATACLASS_DELIMETER + k + ] = v + helptext_from_nested_class_field_name[field.name] = field.helptext continue - # (3) Handle nested callables. - nested_parser = ParserSpecification.from_callable( - field.typ, - description=None, - parent_classes=parent_classes, - parent_type_from_typevar=type_from_typevar, - default_instance=field.default, + # (3) Handle primitive types. These produce a single argument! + args.append( + _arguments.ArgumentDefinition.from_field(field, type_from_typevar) ) - for arg in nested_parser.args: - args.append( - dataclasses.replace( - arg, - prefix=field.name - + _strings.NESTED_DATACLASS_DELIMETER - + arg.prefix, - ) - ) - for k, v in nested_parser.helptext_from_nested_class_field_name.items(): - helptext_from_nested_class_field_name[ - field.name + _strings.NESTED_DATACLASS_DELIMETER + k - ] = v - helptext_from_nested_class_field_name[field.name] = field.helptext return ParserSpecification( description=description @@ -295,7 +302,9 @@ def from_field( # We don't use sets here to retain order of subcommands. options = [type_from_typevar.get(typ, typ) for typ in get_args(field.typ)] options_no_none = [o for o in options if o != type(None)] # noqa - if len(options_no_none) < 2 or any(map(_is_parsable_type, options_no_none)): + if len(options_no_none) < 2 or not all( + map(_is_possibly_nested_type, options_no_none) + ): return None parser_from_name: Dict[str, ParserSpecification] = {} diff --git a/setup.py b/setup.py index 3d70c8bb5..ad682c374 100644 --- a/setup.py +++ b/setup.py @@ -5,8 +5,8 @@ setup( name="dcargs", - version="0.1.1", - description="Strongly typed, zero effort CLIs", + version="0.1.2", + description="Strongly typed, zero-effort CLIs", long_description=long_description, long_description_content_type="text/markdown", url="http://github.com/brentyi/dcargs", diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 4e0dffeb7..4779b902b 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -4,7 +4,7 @@ from typing import ClassVar, Optional import pytest -from typing_extensions import Annotated, Final, Literal # Backward compatibility. +from typing_extensions import Annotated, Final, Literal, TypeAlias import dcargs @@ -321,3 +321,13 @@ class A: x: int = 0 assert dcargs.cli(A, description=None, args=[]) == A(x=0) + + +SomeTypeAlias: TypeAlias = int + + +def test_type_alias(): + def add(a: SomeTypeAlias, b: SomeTypeAlias) -> SomeTypeAlias: + return a + b + + assert dcargs.cli(add, args=["--a", "5", "--b", "7"]) == 12 diff --git a/tests/test_errors.py b/tests/test_errors.py index 5b099f2ec..4a7a7ada4 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Generic, Optional, Tuple, TypeVar, Union +from typing import Generic, List, Optional, Tuple, TypeVar, Union import pytest from typing_extensions import Literal @@ -65,7 +65,7 @@ def main(x: Union[int, str]) -> None: return with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main) + dcargs.cli(main, args=[]) def test_unsupported_literal(): @@ -73,7 +73,7 @@ def main(x: Literal[0, "5"]) -> None: return with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main) + dcargs.cli(main, args=[]) def test_multiple_subparsers(): @@ -93,7 +93,7 @@ class MultipleSubparsers: y: Union[Subcommmand1, Subcommand2] with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(MultipleSubparsers) + dcargs.cli(MultipleSubparsers, args=[]) # Must be global. @@ -104,7 +104,51 @@ class _CycleDataclass: def test_cycle(): with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(_CycleDataclass) + dcargs.cli(_CycleDataclass, args=[]) + + +def test_uncallable_annotation(): + def main(arg: 5) -> None: # type: ignore + pass + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli(main, args=[]) + + +def test_nested_annotation(): + @dataclasses.dataclass + class OneIntArg: + x: int + + def main(arg: List[OneIntArg]) -> List[OneIntArg]: + return arg + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli(main, args=[]) + + @dataclasses.dataclass + class OneStringArg: + x: str + + def main(arg: List[OneStringArg]) -> List[OneStringArg]: + return arg + + assert dcargs.cli(main, args=["--arg", "0", "1", "2"]) == [ + OneStringArg("0"), + OneStringArg("1"), + OneStringArg("2"), + ] + + @dataclasses.dataclass + class TwoStringArg: + x: str + y: str + + def main(arg: List[TwoStringArg]) -> None: + pass + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli(main, args=[]) def test_generic_inherited(): From 1dd7acdc38634f391d6f2f6d27d6cc7e14f846ee Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 24 Jun 2022 03:03:39 -0700 Subject: [PATCH 044/491] Various bug fixes, tests, example improvements - Underscores in positional arguments fixed - Booleans with defaults in positional arguments fixed - Patched docstring corner case for generics --- .coverage | Bin 53248 -> 0 bytes README.md | 801 ++++++++++++++++-- dcargs/_arguments.py | 29 +- dcargs/_calling.py | 8 +- dcargs/_cli.py | 2 +- dcargs/_fields.py | 14 +- dcargs/_parsers.py | 9 +- examples/01_simple_functions.py | 23 + ..._dataclass.py => 02_simple_dataclasses.py} | 4 +- examples/03_enums_and_containers.py | 40 + examples/04_flags.py | 28 + examples/05_hierarchical_configs.py | 68 ++ examples/{4_literals.py => 06_literals.py} | 3 + ...d_collections.py => 07_positional_args.py} | 6 +- ...simple_class.py => 08_standard_classes.py} | 3 + .../{7_subparsers.py => 09_subparsers.py} | 4 +- examples/0_simple_function.py | 20 - examples/{8_generics.py => 10_generics.py} | 2 + examples/3_flags.py | 25 - examples/6_nested_dataclasses.py | 51 -- examples/_format_examples_for_readme.py | 113 +++ setup.py | 2 +- tests/test_dcargs.py | 9 + tests/test_generics_and_serialization.py | 71 ++ tests/test_helptext.py | 35 + tests/test_nested.py | 28 + tests/test_positional_ignore_py37.py | 32 + 27 files changed, 1245 insertions(+), 185 deletions(-) delete mode 100644 .coverage create mode 100644 examples/01_simple_functions.py rename examples/{1_simple_dataclass.py => 02_simple_dataclasses.py} (72%) create mode 100644 examples/03_enums_and_containers.py create mode 100644 examples/04_flags.py create mode 100644 examples/05_hierarchical_configs.py rename examples/{4_literals.py => 06_literals.py} (90%) rename examples/{2_enums_and_collections.py => 07_positional_args.py} (90%) rename examples/{5_simple_class.py => 08_standard_classes.py} (76%) rename examples/{7_subparsers.py => 09_subparsers.py} (80%) delete mode 100644 examples/0_simple_function.py rename examples/{8_generics.py => 10_generics.py} (90%) delete mode 100644 examples/3_flags.py delete mode 100644 examples/6_nested_dataclasses.py create mode 100644 examples/_format_examples_for_readme.py diff --git a/.coverage b/.coverage deleted file mode 100644 index cd4de178a5a90e4571a00f7f517c61edc5ed8350..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 53248 zcmeI4ZIBd48OLX~cX#&XZW{~F6A0d<60&#P%d&?OI4e<%SW-z9hSx*_%k1vl-6Xp+ z$IL8nBnFPeT=|76wJPzG6PtD37Wm*z>bEPYW`B}q!sx0Sx( zrqE6_JfK*fwm;KuQtI2Vqm^%#TH=pOe0%FXytm~Ct-17TExVH6O+TgFpRCXcY#;yv zAOHd&;1L+RyG3bVus}ZW4Oc5xbl28Oy6uOpcdTEtdgB^(WFTugGNPH4YO>NG}kbPRCibprE88}7Be2D(VQm)`@H;OuV9oZm#Ys^5hJ!y z)$CFA0e!T`69{T^^*yeiK!xbWkSQe8&LDf2YU_i#t(zs?@wMnQ%3Z0hNA{(Z_WASW zaow{?vrF_zH&{anbyL<3Qb8rlE(`68wq}-wb*D$wrYtO3q~{*jx7N0*IoDDvhUuAU zI)-alrmFAJOEp(7U(>)I0WI7J-gIlItR2Fj9e&QBu4#1YnXi#E5DzjBl5(aMVy$Ns zRa0h9XR}r<67u_N)e+G+l*6yswPwmWQ(o5!jQzlEbB(sF+uP8bA#%~8ey*Bn?5yci zjo+izbM|; z z*`lMVRv<{4URQ4u%90Lhv0@b)ea-V2N2d-#V?ii3STjqat2n}jlC?{>wIRJPIO`7! z*V~cEb@Iu0`?FpNsM6lnCXe|&LX1n|8*4B^q}02> z6%)Y+SARrvht;cf7s;e}HWqLHeA`Tl3l$1^5yl@`AMsQkS3LSv8uZ1NHK;67>MGx< zkM{1;Dm0pmXogK)md0Rz22h1yStDVsXw_WLf5oBao?`B)>+2U#KMvWGRF@}C-J2?M z2Eqe1JFGck@Nu$g(XuMKX8KbC39fahD#KB|s;{1mzECg285Ly=?aF$pgma1*-8WKS z3%eEVRI8d<4yGwF`}(EQkT3eAsK*~WD5ILCzUWZTyrRso(BP~XsKWC5MhJx{wW^i< zUM|{VT8X=d^{a~WC!sHEuI?IDT@@mpGM+!})*Q8D>vWQ)(5@_N*7~B63V9KVRdPc7 zkQwWcmG_W4a6f)UUw=)GE*>j{&rxj6=7z1Ro-5LGgl?{Cm=iFFh}CTbUjy9CF05NOy8mXL@aY#IilfVYgZt zAmM`8$!<%LMNxiK;z#+v`CEK)Rs$gb0w4eaAOHd&00JNY0w4eaAOHd&@DU);!WPTn z1&C;ZwaMc8152|-a&WOBl4gsO;e`fq|395sBJn@*r}<-i8(+p_nd6yfGT+Q>qGW6! z00JNY0w4eaAOHd&00JNY0v{&=eQ73@rr(hEZ#3tEsNm?fp;e5%^rjulq(9;JrJBnd zrsHa+YiO=zJ0!m}$)v4K%BQ>9wJO~R7qXoRCT(dVJEGa7jwh)!mvmg4Uacb}yW&i` zw@I0{?pPICJ(Fm6j7i&?h?cEVz1(b+Nt>Do4;p%&9*ixfBO1A*~PXrldZ4+6SGLx z*Xhbqc5kyXr>+0vJ#1Su(T4SZtdnhOCOozNkItfMef`gtuyxIf^w{Tl>800ck)1V8`;KmY_l00ck)1m-jWMh^+Jiu?aEza-HI8wh{^2!H?xfB*=9 z00@8p2!H?xfWRCiz*uY2|NsAl#4qqS_$&NJ{2<@YEq*&+kU5)~n1e>cBoF`r5C8!X z009sH0T2KI5CDNWK;VXSR89`0_dam=!SAp8@5JD${p`o*ZoE~#>)(BGY+v-hMT zWa!)SlS|(i`b_53Umm*fu6^P5gU-q09l7HjN8b6(@BVi2>!(gDHzi5x@m%6*q!nt0(sJM;YLAKMtE_ptT)&HMY#z5MFF(@#|IJOA^|pNvrQ(#Yobq-Pbc ziJ$J+dZsllN0Z5UX2$#fKYvc*7wP@~|KM-(N&aX4d;T*26@QVEv4H>xfB*=900@8p z2!H?xfB*=900>+!0^*v1oKAaDD&=T_y4i}|1e0S4iEqV5C8!X009sH z0T2KI5C8!XxPAn@|NryD;`#sg_&fY8KTYoeI6?mbc$H63GBywZ0T2KI5C8!X009sH r0T2KI5CDOX5&@Q`xB0V_2$CX5h(HlRTm&%@L`A?v5D|fR|Ns91=u}D7 diff --git a/README.md b/README.md index 6260ae0eb..b8a28c52b 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,6 @@ * [Overview](#overview) * [Examples](#examples) - * [Functions](#functions) - * [Dataclasses](#dataclasses) - * [Nested dataclasses](#nested-dataclasses) * [Serialization](#serialization) * [Alternative tools](#alternative-tools) @@ -19,24 +16,30 @@ ## Overview -**`dcargs`** is a library for strongly-typed argument parsers and configuration -objects. - ```bash pip install dcargs ``` -Our core interface generates CLI interfaces from type-annotated callables, which -may be functions, classes, or dataclasses. The goal is a tool that's lightweight -enough for simple interactive scripts, but flexible enough to replace heavier -frameworks typically used to build hierarchical configuration systems. +**`dcargs`** is a library for typed CLI interfaces and configuration objects. + +Our core interface contains a single function for generating an argument parser +from a type-annotated callable _`f`_, which may be a function, class, or +dataclass: + +--- + +```python +dcargs.cli( + f: Callable[..., T], + *, + description: Optional[str]=None, + args: Optional[Sequence[str]]=None, + default_instance: Optional[T]=None, +) -> T +``` -
- - dcargs.cli(f: Callable[..., T], *, description: - Optional[str], args: Optional[Sequence[str]], default_instance: Optional[T]) -> T - +Docstring 
Call `f(...)`, with arguments populated from an automatically generated CLI
@@ -88,12 +91,16 @@ Returns:
-
-Notably, `dcargs.cli()` supports _nested_ classes and dataclasses, which enable -expressive hierarchical configuration objects built on standard Python features. +--- + +The goal is a tool that's lightweight enough for simple interactive scripts, but +flexible enough to replace heavier configuration frameworks like `hydra` and +`ml_collections`. Notably, `dcargs.cli()` supports _nested_ classes and +dataclasses, which enable expressive hierarchical configuration objects built on +standard Python features. -Our ultimate goal is the ability build interfaces that are: +Ultimately, we aim to enable configuration interfaces that are: - **Low-effort.** Type annotations, docstrings, and default values can be used to automatically generate argument parsers with informative helptext. This @@ -114,26 +121,36 @@ Our ultimate goal is the ability build interfaces that are: ## Examples -A series of example scripts can be found in [./examples](./examples). +A series of example scripts covering core features are included below. + + +
+ +1. Simple Functions + + +[examples/01_simple_functions.py](examples/01_simple_functions.py) -### Functions +
```python -# examples/0_simple_function.py +"""CLI generation example from a simple annotated function. `dcargs.cli()` will call +`main()`, with arguments populated from the CLI.""" + import dcargs def main( field1: str, - field2: int, + field2: int = 3, flag: bool = False, ) -> None: """Function, whose arguments will be populated from a CLI interface. Args: - field1: First field. - field2: Second field. - flag: Boolean flag that we can set to true. + field1: A string field. + field2: A numeric field, with a default value. + flag: A boolean flag. """ print(field1, field2, flag) @@ -145,24 +162,38 @@ if __name__ == "__main__": --- ```console -$ python 0_simple_function.py --help -usage: 0_simple_function.py [-h] --field1 STR --field2 INT [--flag] +$ python examples/01_simple_functions.py --help +``` + +``` +usage: 01_simple_functions.py [-h] --field1 STR [--field2 INT] [--flag] Function, whose arguments will be populated from a CLI interface. required arguments: - --field1 STR First field. - --field2 INT Second field. + --field1 STR A string field. optional arguments: -h, --help show this help message and exit - --flag Boolean flag that we can set to true. + --field2 INT A numeric field, with a default value. (default: 3) + --flag A boolean flag. ``` -### Dataclasses +
+
+ +
+ +2. Simple Dataclasses + + +[examples/02_simple_dataclasses.py](examples/02_simple_dataclasses.py) + +
```python -# examples/1_simple_dataclass.py +"""Example using dcargs.cli() to instantiate a dataclass.""" + import dataclasses import dcargs @@ -174,39 +205,202 @@ class Args: This should show up in the helptext!""" field1: str # A string field. - field2: int # A numeric field. + field2: int = 3 # A numeric field, with a default value. flag: bool = False # A boolean flag. if __name__ == "__main__": args = dcargs.cli(Args) print(args) + print() + print(dcargs.to_yaml(args)) ``` --- ```console -$ python 1_simple_dataclass.py --help -usage: 1_simple_dataclass.py [-h] --field1 STR --field2 INT [--flag] +$ python examples/02_simple_dataclasses.py --help +``` + +``` +usage: 02_simple_dataclasses.py [-h] --field1 STR [--field2 INT] [--flag] Description. This should show up in the helptext! required arguments: --field1 STR A string field. - --field2 INT A numeric field. optional arguments: -h, --help show this help message and exit + --field2 INT A numeric field, with a default value. (default: 3) --flag A boolean flag. ``` -### Nested dataclasses +
+
+ +
+ +3. Enums And Containers + + +[examples/03_enums_and_containers.py](examples/03_enums_and_containers.py) + +
```python -# examples/6_nested_dataclasses.py +"""Examples of more advanced type annotations: enums and containers types. + +For collections, we only showcase Tuple here, but List, Sequence, Set, etc are all +supported as well.""" + import dataclasses import enum +import pathlib +from typing import Optional, Tuple + +import dcargs + + +class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + +@dataclasses.dataclass(frozen=True) +class TrainConfig: + # Example of a variable-length tuple: + dataset_sources: Tuple[pathlib.Path, ...] + """Paths to load training data from. This can be multiple!""" + + # Fixed-length tupels are also okay: + image_dimensions: Tuple[int, int] + """Height and width of some image data.""" + + # Enums are handled seamlessly. + optimizer_type: OptimizerType + """Gradient-based optimizer to use.""" + + # We can also explicitly mark arguments as optional. + checkpoint_interval: Optional[int] + """Interval to save checkpoints at.""" + + +if __name__ == "__main__": + config = dcargs.cli(TrainConfig) + print(config) +``` + +--- + +```console +$ python examples/03_enums_and_containers.py --help +``` + +``` +usage: 03_enums_and_containers.py [-h] --dataset-sources PATH [PATH ...] + --image-dimensions INT INT --optimizer-type + {ADAM,SGD} [--checkpoint-interval INT] + +required arguments: + --dataset-sources PATH [PATH ...] + Paths to load training data from. This can be multiple! + --image-dimensions INT INT + Height and width of some image data. + --optimizer-type {ADAM,SGD} + Gradient-based optimizer to use. + +optional arguments: + -h, --help show this help message and exit + --checkpoint-interval INT + Interval to save checkpoints at. (default: None) +``` + +
+
+ +
+ +4. Flags + + +[examples/04_flags.py](examples/04_flags.py) + +
+ +```python +"""Example of how booleans are handled and automatically converted to flags.""" + +import dataclasses +from typing import Optional + +import dcargs + + +@dataclasses.dataclass +class Args: + # Boolean. This expects an explicit "True" or "False". + boolean: bool + + # Optional boolean. Same as above, but can be omitted. + optional_boolean: Optional[bool] + + # Pass --flag-a in to set this value to True. + flag_a: bool = False + + # Pass --no-flag-b in to set this value to False. + flag_b: bool = True + + +if __name__ == "__main__": + args = dcargs.cli(Args) + print(args) + print() + print(dcargs.to_yaml(args)) +``` + +--- + +```console +$ python examples/04_flags.py --help +``` + +``` +usage: 04_flags.py [-h] --boolean {True,False} + [--optional-boolean {True,False}] [--flag-a] [--no-flag-b] + +required arguments: + --boolean {True,False} + Boolean. This expects an explicit "True" or "False". + +optional arguments: + -h, --help show this help message and exit + --optional-boolean {True,False} + Optional boolean. Same as above, but can be omitted. (default: None) + --flag-a Pass --flag-a in to set this value to True. + --no-flag-b Pass --no-flag-b in to set this value to False. +``` + +
+
+ +
+ +5. Hierarchical Configs + + +[examples/05_hierarchical_configs.py](examples/05_hierarchical_configs.py) + +
+ +```python +"""An example of how we can create hierarchical configuration interfaces by nesting +dataclasses.""" + +import dataclasses +import enum +import pathlib import dcargs @@ -230,49 +424,283 @@ class OptimizerConfig: @dataclasses.dataclass(frozen=True) class ExperimentConfig: - """A nested experiment configuration. Note that the argument parser description is - pulled from this docstring by default, but can also be overrided with - `dcargs.cli()`'s `description=` argument.""" - - # Experiment name to use. - experiment_name: str - # Various configurable options for our optimizer. optimizer: OptimizerConfig + # Batch size. + batch_size: int = 32 + + # Total number of training steps. + train_steps: int = 100_000 + # Random seed. This is helpful for making sure that our experiments are all # reproducible! seed: int = 0 -if __name__ == "__main__": - config = dcargs.cli(ExperimentConfig) - print(config) +def train( + out_dir: pathlib.Path, + /, + config: ExperimentConfig, + restore_checkpoint: bool = False, + checkpoint_interval: int = 1000, +) -> None: + """Train a model. + + Args: + out_dir: Where to save logs and checkpoints. + config: Experiment configuration. + restore_checkpoint: Set to restore an existing checkpoint. + checkpoint_interval: Training steps between each checkpoint save. + """ + print(out_dir) + print("---") print(dcargs.to_yaml(config)) + print("---") + print(restore_checkpoint) + print(checkpoint_interval) + + +if __name__ == "__main__": + dcargs.cli(train) ``` --- ```console -usage: 6_nested_dataclasses.py [-h] --experiment-name STR [--optimizer.algorithm {ADAM,SGD}] - [--optimizer.learning-rate FLOAT] [--optimizer.weight-decay FLOAT] - [--seed INT] +$ python examples/05_hierarchical_configs.py --help +``` -A nested experiment configuration. Note that the argument parser description is -pulled from this docstring by default, but can also be overrided with -`dcargs.cli()`'s `description=` flag. +``` +usage: 05_hierarchical_configs.py [-h] + [--config.optimizer.algorithm {ADAM,SGD}] + [--config.optimizer.learning-rate FLOAT] + [--config.optimizer.weight-decay FLOAT] + [--config.batch-size INT] + [--config.train-steps INT] + [--config.seed INT] [--restore-checkpoint] + [--checkpoint-interval INT] + OUT_DIR -required arguments: - --experiment-name STR - Experiment name to use. +Train a model. + +positional arguments: + OUT_DIR Where to save logs and checkpoints. optional arguments: -h, --help show this help message and exit - --seed INT Random seed. This is helpful for making sure that our experiments are all + --restore-checkpoint Set to restore an existing checkpoint. + --checkpoint-interval INT + Training steps between each checkpoint save. (default: 1000) + +optional config.optimizer arguments: + Various configurable options for our optimizer. + + --config.optimizer.algorithm {ADAM,SGD} + Gradient-based optimizer to use. (default: ADAM) + --config.optimizer.learning-rate FLOAT + Learning rate to use. (default: 0.0003) + --config.optimizer.weight-decay FLOAT + Coefficient for L2 regularization. (default: 0.01) + +optional config arguments: + Experiment configuration. + + --config.batch-size INT + Batch size. (default: 32) + --config.train-steps INT + Total number of training steps. (default: 100000) + --config.seed INT Random seed. This is helpful for making sure that our experiments are all reproducible! (default: 0) +``` + +
+
+ +
+ +6. Literals + + +[examples/06_literals.py](examples/06_literals.py) + +
+ +```python +"""typing.Literal[] can be used to specify accepted input choices.""" + +import dataclasses +import enum +from typing import Literal + +import dcargs + + +class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + +@dataclasses.dataclass(frozen=True) +class Args: + enum: Color + restricted_enum: Literal[Color.RED, Color.GREEN] + + integer: Literal[0, 1, 2, 3] + string: Literal["red", "green"] + + restricted_enum_with_default: Literal[Color.RED, Color.GREEN] = Color.GREEN + integer_with_default: Literal[0, 1, 2, 3] = 3 + string_with_Default: Literal["red", "green"] = "red" + + +if __name__ == "__main__": + args = dcargs.cli(Args) + print(args) + print() + print(dcargs.to_yaml(args)) +``` + +--- + +```console +$ python examples/06_literals.py --help +``` + +``` +usage: 06_literals.py [-h] --enum {RED,GREEN,BLUE} --restricted-enum + {RED,GREEN} --integer {0,1,2,3} --string {red,green} + [--restricted-enum-with-default {RED,GREEN}] + [--integer-with-default {0,1,2,3}] + [--string-with-Default {red,green}] + +required arguments: + --enum {RED,GREEN,BLUE} + --restricted-enum {RED,GREEN} + --integer {0,1,2,3} + --string {red,green} + +optional arguments: + -h, --help show this help message and exit + --restricted-enum-with-default {RED,GREEN} + (default: GREEN) + --integer-with-default {0,1,2,3} + (default: 3) + --string-with-Default {red,green} + (default: red) +``` + +
+
+ +
+ +7. Positional Args + + +[examples/07_positional_args.py](examples/07_positional_args.py) + +
+ +```python +"""Positional-only arguments in functions are converted to positional CLI arguments.""" + +from __future__ import annotations + +import dataclasses +import enum +import pathlib +from typing import Tuple + +import dcargs + + +def main( + source: pathlib.Path, + dest: pathlib.Path, + /, # Mark the end of positional arguments. + optimizer: OptimizerConfig, + force: bool = False, + verbose: bool = False, + background_rgb: Tuple[float, float, float] = (1.0, 0.0, 0.0), +) -> None: + """Command-line interface defined using a function signature. Note that this + docstring is parsed to generate helptext. + + Args: + source: Source path. + dest: Destination path. + optimizer: Configuration for our optimizer object. + force: Do not prompt before overwriting. + verbose: Explain what is being done. + background_rgb: Background color. Red by default. + """ + print( + f"{source.absolute()=}" + "\n" + f"{dest.absolute()=}" + "\n" + f"{optimizer=}" + "\n" + f"{force=}" + "\n" + f"{verbose=}" + "\n" + f"{background_rgb=}" + ) + + +class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + +@dataclasses.dataclass(frozen=True) +class OptimizerConfig: + algorithm: OptimizerType = OptimizerType.ADAM + """Gradient-based optimizer to use.""" + + learning_rate: float = 3e-4 + """Learning rate to use.""" + + weight_decay: float = 1e-2 + """Coefficient for L2 regularization.""" + + +if __name__ == "__main__": + dcargs.cli(main) +``` + +--- + +```console +$ python examples/07_positional_args.py --help +``` + +``` +usage: 07_positional_args.py [-h] [--optimizer.algorithm {ADAM,SGD}] + [--optimizer.learning-rate FLOAT] + [--optimizer.weight-decay FLOAT] [--force] + [--verbose] [--background-rgb FLOAT FLOAT FLOAT] + SOURCE DEST + +Command-line interface defined using a function signature. Note that this +docstring is parsed to generate helptext. + +positional arguments: + SOURCE Source path. + DEST Destination path. + +optional arguments: + -h, --help show this help message and exit + --force Do not prompt before overwriting. + --verbose Explain what is being done. + --background-rgb FLOAT FLOAT FLOAT + Background color. Red by default. (default: 1.0 0.0 0.0) optional optimizer arguments: - Various configurable options for our optimizer. + Configuration for our optimizer object. --optimizer.algorithm {ADAM,SGD} Gradient-based optimizer to use. (default: ADAM) @@ -282,6 +710,259 @@ optional optimizer arguments: Coefficient for L2 regularization. (default: 0.01) ``` +
+
+ +
+ +8. Standard Classes + + +[examples/08_standard_classes.py](examples/08_standard_classes.py) + +
+ +```python +"""In addition to functions and dataclasses, we can also generate CLIs from (the +constructors of) standard Python classes.""" + +import dcargs + + +class Args: + def __init__( + self, + field1: str, + field2: int, + flag: bool = False, + ): + """Arguments. + + Args: + field1: A string field. + field2: A numeric field. + flag: A boolean flag. + """ + self.data = [field1, field2, flag] + + +if __name__ == "__main__": + args = dcargs.cli(Args) + print(args.data) +``` + +--- + +```console +$ python examples/08_standard_classes.py --help +``` + +``` +usage: 08_standard_classes.py [-h] --field1 STR --field2 INT [--flag] + +required arguments: + --field1 STR Arguments. + + Args: + field1: A string field. + field2: A numeric field. + flag: A boolean flag. + --field2 INT Arguments. + + Args: + field1: A string field. + field2: A numeric field. + flag: A boolean flag. + +optional arguments: + -h, --help show this help message and exit + --flag Arguments. + + Args: + field1: A string field. + field2: A numeric field. + flag: A boolean flag. +``` + +
+
+ +
+ +9. Subparsers + + +[examples/09_subparsers.py](examples/09_subparsers.py) + +
+ +```python +"""Unions over nested types (classes or dataclasses) will result in subparsers.""" + +from __future__ import annotations + +import dataclasses +from typing import Union + +import dcargs + + +def main(command: Union[Checkout, Commit]) -> None: + print(command) + + +@dataclasses.dataclass(frozen=True) +class Checkout: + """Checkout a branch.""" + + branch: str + + +@dataclasses.dataclass(frozen=True) +class Commit: + """Commit changes.""" + + message: str + all: bool = False + + +if __name__ == "__main__": + dcargs.cli(main) +``` + +--- + +```console +$ python examples/09_subparsers.py --help +``` + +``` +usage: 09_subparsers.py [-h] {checkout,commit} ... + +optional arguments: + -h, --help show this help message and exit + +subcommands: + {checkout,commit} +``` + +
+
+ +
+ +10. Generics + + +[examples/10_generics.py](examples/10_generics.py) + +
+ +```python +"""Example of parsing for generic (~templated) dataclasses.""" + +import dataclasses +from typing import Generic, TypeVar + +import dcargs + +ScalarType = TypeVar("ScalarType") +ShapeType = TypeVar("ShapeType") + + +@dataclasses.dataclass(frozen=True) +class Point3(Generic[ScalarType]): + x: ScalarType + y: ScalarType + z: ScalarType + frame_id: str + + +@dataclasses.dataclass(frozen=True) +class Triangle: + a: Point3[float] + b: Point3[float] + c: Point3[float] + + +@dataclasses.dataclass(frozen=True) +class Args(Generic[ShapeType]): + point_continuous: Point3[float] + point_discrete: Point3[int] + shape: ShapeType + + +if __name__ == "__main__": + args = dcargs.cli(Args[Triangle]) + print(args) +``` + +--- + +```console +$ python examples/10_generics.py --help +``` + +``` +usage: 10_generics.py [-h] --point-continuous.x FLOAT --point-continuous.y + FLOAT --point-continuous.z FLOAT + --point-continuous.frame-id STR --point-discrete.x INT + --point-discrete.y INT --point-discrete.z INT + --point-discrete.frame-id STR --shape.a.x FLOAT + --shape.a.y FLOAT --shape.a.z FLOAT --shape.a.frame-id + STR --shape.b.x FLOAT --shape.b.y FLOAT --shape.b.z + FLOAT --shape.b.frame-id STR --shape.c.x FLOAT + --shape.c.y FLOAT --shape.c.z FLOAT --shape.c.frame-id + STR + +optional arguments: + -h, --help show this help message and exit + +required point_continuous arguments: + Point3(*args, **kwds) + + --point-continuous.x FLOAT + --point-continuous.y FLOAT + --point-continuous.z FLOAT + --point-continuous.frame-id STR + +required point_discrete arguments: + Point3(*args, **kwds) + + --point-discrete.x INT + --point-discrete.y INT + --point-discrete.z INT + --point-discrete.frame-id STR + +required shape.a arguments: + Point3(*args, **kwds) + + --shape.a.x FLOAT + --shape.a.y FLOAT + --shape.a.z FLOAT + --shape.a.frame-id STR + +required shape.b arguments: + Point3(*args, **kwds) + + --shape.b.x FLOAT + --shape.b.y FLOAT + --shape.b.z FLOAT + --shape.b.frame-id STR + +required shape.c arguments: + Point3(*args, **kwds) + + --shape.c.x FLOAT + --shape.c.y FLOAT + --shape.c.z FLOAT + --shape.c.frame-id STR +``` + +
+
+ + ## Serialization As a secondary feature aimed at enabling the use of `dcargs.cli()` for general diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 87ca80383..dc3d38d6c 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -24,7 +24,6 @@ class ArgumentDefinition: # Fields that will be populated initially. # Important: from here on out, all fields correspond 1:1 to inputs to argparse's # add_argument() method. - name: str type: Optional[Union[Type, TypeVar]] default: Optional[Any] @@ -35,7 +34,6 @@ class ArgumentDefinition: choices: Optional[Set[Any]] = None metavar: Optional[Union[str, Tuple[str, ...]]] = None help: Optional[str] = None - dest: Optional[str] = None def add_argument( self, parser: Union[argparse.ArgumentParser, argparse._ArgumentGroup] @@ -44,8 +42,9 @@ def add_argument( kwargs = {k: v for k, v in vars(self).items() if v is not None} # Apply prefix for nested dataclasses. - if "dest" in kwargs: - kwargs["dest"] = self.prefix + kwargs["dest"] + assert "dest" not in kwargs + if not self.field.positional: + kwargs["dest"] = self.prefix + self.field.name # Important: as far as argparse is concerned, all inputs are strings. # @@ -60,29 +59,29 @@ def add_argument( kwargs.pop("prefix") kwargs.pop("field") kwargs.pop("instantiator") - kwargs.pop("name") # Note that the name must be passed in as a position argument. - parser.add_argument(self.get_flag(), **kwargs) + parser.add_argument(self.get_name_or_flag(), **kwargs) - def get_flag(self) -> str: - """Get --flag representation, with a prefix applied for nested dataclasses.""" + def get_name_or_flag(self) -> str: + """Get name (for positional args) or flag (for keyword args), with a prefix + applied for nested dataclasses.""" if self.field.positional: - return (self.prefix + self.name).replace("_", "-") + return self.prefix + self.field.name + elif self.action == "store_false": + return "--" + (self.prefix + "no-" + self.field.name).replace("_", "-") else: - return "--" + (self.prefix + self.name).replace("_", "-") + return "--" + (self.prefix + self.field.name).replace("_", "-") @staticmethod def from_field( field: _fields.Field, type_from_typevar: Dict[TypeVar, Type], ) -> ArgumentDefinition: - """""" arg = ArgumentDefinition( prefix="", field=field, instantiator=None, - name=field.name, type=field.typ, default=field.default, ) @@ -110,7 +109,7 @@ def _transform_handle_boolean_flags(arg: ArgumentDefinition) -> ArgumentDefiniti if arg.type is not bool: return arg - if arg.default is None: + if arg.default is None or arg.field.positional: # If no default is passed in, we treat bools as a normal parameter. return arg elif arg.default is False: @@ -125,8 +124,6 @@ def _transform_handle_boolean_flags(arg: ArgumentDefinition) -> ArgumentDefiniti # Default `True` => --no-flag passed in flips to `False`. return dataclasses.replace( arg, - dest=arg.name, - name="no_" + arg.name, action="store_false", type=None, instantiator=lambda x: x, # argparse will directly give us a bool! @@ -228,7 +225,7 @@ def _transform_positional_special_handling( return dataclasses.replace( arg, - metavar=(arg.prefix + arg.name).upper(), + metavar=(arg.prefix + arg.field.name).upper(), required=None, nargs="?" if not arg.required else arg.nargs, ) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 733f6648e..620bdc400 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -10,8 +10,8 @@ from . import _fields, _parsers, _resolver, _strings -class FieldActionValueError(Exception): - """Exception raised when field actions fail; this typically means that values from +class InstantiationError(Exception): + """Exception raised when instantiation fail; this typically means that values from the CLI are invalid.""" @@ -66,8 +66,8 @@ def get_value_from_arg(arg: str) -> Any: assert arg.instantiator is not None value = arg.instantiator(value) except ValueError as e: - raise FieldActionValueError( - f"Parsing error for {arg.get_flag()}: {e.args[0]}" + raise InstantiationError( + f"Parsing error for {arg.get_name_or_flag()}: {e.args[0]}" ) elif ( prefixed_field_name diff --git a/dcargs/_cli.py b/dcargs/_cli.py index d5abf9123..4a566c587 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -104,7 +104,7 @@ def cli( out, consumed_keywords = _calling.call_from_args( f, parser_definition, value_from_arg ) - except _calling.FieldActionValueError as e: + except _calling.InstantiationError as e: # Emulate argparse's error behavior when invalid arguments are passed in. parser.print_usage() print() diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 7b8c581fa..50c9cc540 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -125,7 +125,19 @@ def _get_dataclass_field_default( _ensure_dataclass_instance_used_as_default_is_frozen( field, field_default_instance ) - elif field.default_factory is not dataclasses.MISSING: + elif field.default_factory is not dataclasses.MISSING and not ( + # Special case to ignore default_factory if we write: + # `field: Dataclass = dataclasses.field(default_factory=Dataclass)`. + # + # In other words, treat it the same way as: + # `field: Dataclass`. + # + # The only time this matters is when we our dataclass has a `__post_init__` + # function that mutates the dataclass. We choose here to use the default values + # before this method is called. + dataclasses.is_dataclass(field.type) + and field.default_factory is field.type + ): # Populate default from `dataclasses.field(default_factory=...)`. field_default_instance = field.default_factory() diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index e6aa50232..e3dd3fb5e 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -138,7 +138,6 @@ def from_callable( " the mro: https://github.com/python/typing/issues/777" ) - get_origin(field.typ) # (1) Handle Unions over callables; these result in subparsers. subparsers_attempt = SubparsersSpecification.from_field( field, @@ -175,7 +174,13 @@ def from_callable( helptext_from_nested_class_field_name[ field.name + _strings.NESTED_DATACLASS_DELIMETER + k ] = v - helptext_from_nested_class_field_name[field.name] = field.helptext + + if field.helptext is not None: + helptext_from_nested_class_field_name[field.name] = field.helptext + else: + helptext_from_nested_class_field_name[field.name] = inspect.getdoc( + _resolver.resolve_generic_types(field.typ)[0] + ) continue # (3) Handle primitive types. These produce a single argument! diff --git a/examples/01_simple_functions.py b/examples/01_simple_functions.py new file mode 100644 index 000000000..6aa357b8e --- /dev/null +++ b/examples/01_simple_functions.py @@ -0,0 +1,23 @@ +"""CLI generation example from a simple annotated function. `dcargs.cli()` will call +`main()`, with arguments populated from the CLI.""" + +import dcargs + + +def main( + field1: str, + field2: int = 3, + flag: bool = False, +) -> None: + """Function, whose arguments will be populated from a CLI interface. + + Args: + field1: A string field. + field2: A numeric field, with a default value. + flag: A boolean flag. + """ + print(field1, field2, flag) + + +if __name__ == "__main__": + dcargs.cli(main) diff --git a/examples/1_simple_dataclass.py b/examples/02_simple_dataclasses.py similarity index 72% rename from examples/1_simple_dataclass.py rename to examples/02_simple_dataclasses.py index 90bebc3ff..52477948a 100644 --- a/examples/1_simple_dataclass.py +++ b/examples/02_simple_dataclasses.py @@ -1,3 +1,5 @@ +"""Example using dcargs.cli() to instantiate a dataclass.""" + import dataclasses import dcargs @@ -9,7 +11,7 @@ class Args: This should show up in the helptext!""" field1: str # A string field. - field2: int # A numeric field. + field2: int = 3 # A numeric field, with a default value. flag: bool = False # A boolean flag. diff --git a/examples/03_enums_and_containers.py b/examples/03_enums_and_containers.py new file mode 100644 index 000000000..d9d4269e4 --- /dev/null +++ b/examples/03_enums_and_containers.py @@ -0,0 +1,40 @@ +"""Examples of more advanced type annotations: enums and containers types. + +For collections, we only showcase Tuple here, but List, Sequence, Set, etc are all +supported as well.""" + +import dataclasses +import enum +import pathlib +from typing import Optional, Tuple + +import dcargs + + +class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + +@dataclasses.dataclass(frozen=True) +class TrainConfig: + # Example of a variable-length tuple: + dataset_sources: Tuple[pathlib.Path, ...] + """Paths to load training data from. This can be multiple!""" + + # Fixed-length tupels are also okay: + image_dimensions: Tuple[int, int] + """Height and width of some image data.""" + + # Enums are handled seamlessly. + optimizer_type: OptimizerType + """Gradient-based optimizer to use.""" + + # We can also explicitly mark arguments as optional. + checkpoint_interval: Optional[int] + """Interval to save checkpoints at.""" + + +if __name__ == "__main__": + config = dcargs.cli(TrainConfig) + print(config) diff --git a/examples/04_flags.py b/examples/04_flags.py new file mode 100644 index 000000000..d5fcd7834 --- /dev/null +++ b/examples/04_flags.py @@ -0,0 +1,28 @@ +"""Example of how booleans are handled and automatically converted to flags.""" + +import dataclasses +from typing import Optional + +import dcargs + + +@dataclasses.dataclass +class Args: + # Boolean. This expects an explicit "True" or "False". + boolean: bool + + # Optional boolean. Same as above, but can be omitted. + optional_boolean: Optional[bool] + + # Pass --flag-a in to set this value to True. + flag_a: bool = False + + # Pass --no-flag-b in to set this value to False. + flag_b: bool = True + + +if __name__ == "__main__": + args = dcargs.cli(Args) + print(args) + print() + print(dcargs.to_yaml(args)) diff --git a/examples/05_hierarchical_configs.py b/examples/05_hierarchical_configs.py new file mode 100644 index 000000000..4324cb79a --- /dev/null +++ b/examples/05_hierarchical_configs.py @@ -0,0 +1,68 @@ +"""An example of how we can create hierarchical configuration interfaces by nesting +dataclasses.""" + +import dataclasses +import enum +import pathlib + +import dcargs + + +class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + +@dataclasses.dataclass(frozen=True) +class OptimizerConfig: + # Gradient-based optimizer to use. + algorithm: OptimizerType = OptimizerType.ADAM + + # Learning rate to use. + learning_rate: float = 3e-4 + + # Coefficient for L2 regularization. + weight_decay: float = 1e-2 + + +@dataclasses.dataclass(frozen=True) +class ExperimentConfig: + # Various configurable options for our optimizer. + optimizer: OptimizerConfig + + # Batch size. + batch_size: int = 32 + + # Total number of training steps. + train_steps: int = 100_000 + + # Random seed. This is helpful for making sure that our experiments are all + # reproducible! + seed: int = 0 + + +def train( + out_dir: pathlib.Path, + /, + config: ExperimentConfig, + restore_checkpoint: bool = False, + checkpoint_interval: int = 1000, +) -> None: + """Train a model. + + Args: + out_dir: Where to save logs and checkpoints. + config: Experiment configuration. + restore_checkpoint: Set to restore an existing checkpoint. + checkpoint_interval: Training steps between each checkpoint save. + """ + print(out_dir) + print("---") + print(dcargs.to_yaml(config)) + print("---") + print(restore_checkpoint) + print(checkpoint_interval) + + +if __name__ == "__main__": + dcargs.cli(train) diff --git a/examples/4_literals.py b/examples/06_literals.py similarity index 90% rename from examples/4_literals.py rename to examples/06_literals.py index 0737e5add..e72ac0605 100644 --- a/examples/4_literals.py +++ b/examples/06_literals.py @@ -1,3 +1,5 @@ +"""typing.Literal[] can be used to specify accepted input choices.""" + import dataclasses import enum from typing import Literal @@ -15,6 +17,7 @@ class Color(enum.Enum): class Args: enum: Color restricted_enum: Literal[Color.RED, Color.GREEN] + integer: Literal[0, 1, 2, 3] string: Literal["red", "green"] diff --git a/examples/2_enums_and_collections.py b/examples/07_positional_args.py similarity index 90% rename from examples/2_enums_and_collections.py rename to examples/07_positional_args.py index 8c52badbb..e2a543f91 100644 --- a/examples/2_enums_and_collections.py +++ b/examples/07_positional_args.py @@ -1,3 +1,5 @@ +"""Positional-only arguments in functions are converted to positional CLI arguments.""" + from __future__ import annotations import dataclasses @@ -21,8 +23,8 @@ def main( docstring is parsed to generate helptext. Args: - source: Path to move data from. - dest: Path to move data to. + source: Source path. + dest: Destination path. optimizer: Configuration for our optimizer object. force: Do not prompt before overwriting. verbose: Explain what is being done. diff --git a/examples/5_simple_class.py b/examples/08_standard_classes.py similarity index 76% rename from examples/5_simple_class.py rename to examples/08_standard_classes.py index cdef1c978..f2ecd5c3f 100644 --- a/examples/5_simple_class.py +++ b/examples/08_standard_classes.py @@ -1,3 +1,6 @@ +"""In addition to functions and dataclasses, we can also generate CLIs from (the +constructors of) standard Python classes.""" + import dcargs diff --git a/examples/7_subparsers.py b/examples/09_subparsers.py similarity index 80% rename from examples/7_subparsers.py rename to examples/09_subparsers.py index add12a267..ed71fb5c8 100644 --- a/examples/7_subparsers.py +++ b/examples/09_subparsers.py @@ -1,3 +1,5 @@ +"""Unions over nested types (classes or dataclasses) will result in subparsers.""" + from __future__ import annotations import dataclasses @@ -7,7 +9,7 @@ def main(command: Union[Checkout, Commit]) -> None: - pass + print(command) @dataclasses.dataclass(frozen=True) diff --git a/examples/0_simple_function.py b/examples/0_simple_function.py deleted file mode 100644 index cd2398743..000000000 --- a/examples/0_simple_function.py +++ /dev/null @@ -1,20 +0,0 @@ -import dcargs - - -def main( - field1: str, - field2: int, - flag: bool = False, -) -> None: - """Function, whose arguments will be populated from a CLI interface. - - Args: - field1: First field. - field2: Second field. - flag: Boolean flag that we can set to true. - """ - print(field1, field2, flag) - - -if __name__ == "__main__": - dcargs.cli(main) diff --git a/examples/8_generics.py b/examples/10_generics.py similarity index 90% rename from examples/8_generics.py rename to examples/10_generics.py index 08321b79d..31168fb9e 100644 --- a/examples/8_generics.py +++ b/examples/10_generics.py @@ -1,3 +1,5 @@ +"""Example of parsing for generic (~templated) dataclasses.""" + import dataclasses from typing import Generic, TypeVar diff --git a/examples/3_flags.py b/examples/3_flags.py deleted file mode 100644 index f054280d9..000000000 --- a/examples/3_flags.py +++ /dev/null @@ -1,25 +0,0 @@ -import dataclasses -from typing import Optional - -import dcargs - - -@dataclasses.dataclass -class Args: - # Color input. - red: int - green: int - blue: int - - # Boolean values. - flag: bool - optional_flag: Optional[bool] - flag1: bool = False - flag2: bool = True - - -if __name__ == "__main__": - args = dcargs.cli(Args) - print(args) - print() - print(dcargs.to_yaml(args)) diff --git a/examples/6_nested_dataclasses.py b/examples/6_nested_dataclasses.py deleted file mode 100644 index c92f8af31..000000000 --- a/examples/6_nested_dataclasses.py +++ /dev/null @@ -1,51 +0,0 @@ -"""An argument parsing example. - -Note that multiple possible documentation styles are supported by the field helptext -generator; we could also use docstring-style triple quote comments, or #-style comments -on the same line. -""" - -import dataclasses -import enum - -import dcargs - - -class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - -@dataclasses.dataclass(frozen=True) -class OptimizerConfig: - # Gradient-based optimizer to use. - algorithm: OptimizerType = OptimizerType.ADAM - - # Learning rate to use. - learning_rate: float = 3e-4 - - # Coefficient for L2 regularization. - weight_decay: float = 1e-2 - - -@dataclasses.dataclass(frozen=True) -class ExperimentConfig: - """A nested experiment configuration. Note that the argument parser description is - pulled from this docstring by default, but can also be overrided with - `dcargs.cli()`'s `description=` argument.""" - - # Experiment name to use. - experiment_name: str - - # Various configurable options for our optimizer. - optimizer: OptimizerConfig - - # Random seed. This is helpful for making sure that our experiments are all - # reproducible! - seed: int = 0 - - -if __name__ == "__main__": - config = dcargs.cli(ExperimentConfig) - print(config) - print(dcargs.to_yaml(config)) diff --git a/examples/_format_examples_for_readme.py b/examples/_format_examples_for_readme.py new file mode 100644 index 000000000..39b1906b9 --- /dev/null +++ b/examples/_format_examples_for_readme.py @@ -0,0 +1,113 @@ +"""Helper script for generating the examples list in the README. + +Generally, invoked as: +`python examples/_format_examples_for_readme.py --readme-path ./README.md` +""" + +import concurrent.futures +import pathlib +import re +import subprocess +from typing import Optional + +import dcargs + +README_MARKER_START = "" +README_MARKER_END = "" +SCRIPT_FORMAT_TEMPLATE = """ +
+ +{index}. {title} + + +[{path}]({path}) + +
+ +```python +{source} +``` + +--- + +```console +$ python {path} --help +``` +``` +{helptext} +``` + +
+
+""" + + +def update_readme(readme_path: pathlib.Path, inner: str) -> None: + """Puts `inner` in between {README_MARKER_START} and {README_MARKER_END}.""" + readme_contents = readme_path.read_text() + assert README_MARKER_START in readme_contents + assert README_MARKER_END in readme_contents + before, _, after = readme_contents.partition(README_MARKER_START) + _, _, after = readme_contents.rpartition(README_MARKER_END) + readme_path.write_text( + "".join([before, README_MARKER_START, inner, README_MARKER_END, after]) + ) + print(f"Wrote update examples to {readme_path}!") + try: + subprocess.run(args=["prettier", "-w", readme_path.as_posix()]) + print("Successfully formatted with `prettier`!") + except FileNotFoundError: + print("Tried to format README with `prettier`, but not in PATH.") + + +def format_script_for_readme(index: int, path: pathlib.Path) -> str: + title = " ".join( + map( + # Capitalize first letter of each word. + lambda lower: lower[0:1].upper() + lower[1:], + # Remove integer prefix. + path.stem.split("_")[1:], + ) + ) + + source = path.read_text().strip() + + helptext = subprocess.run( + args=["python", str(path), "--help"], stdout=subprocess.PIPE, encoding="utf8" + ).stdout + helptext = re.sub( # Strip colorcodes. + r"\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))", "", helptext + ).strip() + + return SCRIPT_FORMAT_TEMPLATE.format( + index=index, title=title, path=path, source=source, helptext=helptext + ) + + +def main(readme_path: pathlib.Path) -> None: + """Helper script for generating the examples list in the README. + + Args: + readme_path: README file to write to; we replace the contents between + the start and end markers. If not specified, output is printed. + """ + script_paths = sorted( + filter( + lambda p: not p.name.startswith("_"), + pathlib.Path(__file__).parent.glob("*.py"), + ) + ) + + with concurrent.futures.ThreadPoolExecutor() as executor: + out_str = "\n".join( + executor.map( + format_script_for_readme, + range(1, len(script_paths) + 1), + script_paths, + ) + ) + update_readme(readme_path, out_str) + + +if __name__ == "__main__": + dcargs.cli(main) diff --git a/setup.py b/setup.py index ad682c374..564bd79d6 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.1.2", + version="0.1.3", description="Strongly typed, zero-effort CLIs", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 4779b902b..678d8c22d 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -9,6 +9,15 @@ import dcargs +def test_no_args(): + def main() -> int: + return 5 + + assert dcargs.cli(main, args=[]) == 5 + with pytest.raises(SystemExit): + dcargs.cli(main, args=["3"]) + + def test_basic(): @dataclasses.dataclass class ManyTypes: diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 18065c3ca..3d76829da 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -1,5 +1,7 @@ +import contextlib import dataclasses import enum +import io from typing import Generic, List, Tuple, Type, TypeVar, Union import pytest @@ -26,6 +28,37 @@ class TupleGenericVariable(Generic[ScalarType]): ) == TupleGenericVariable((1, 2, 3)) +def test_tuple_generic_helptext(): + @dataclasses.dataclass + class TupleGenericVariableHelptext(Generic[ScalarType]): + """Helptext!""" + + xyz: Tuple[ScalarType, ...] + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli(TupleGenericVariableHelptext[int], args=["--help"]) + helptext = f.getvalue() + assert "Helptext!" in helptext + + +def test_tuple_generic_no_helptext(): + @dataclasses.dataclass + class TupleGenericVariableNoHelptext(Generic[ScalarType]): + xyz: Tuple[ScalarType, ...] + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli(TupleGenericVariableNoHelptext[int], args=["--help"]) + helptext = f.getvalue() + assert "Helptext!" not in helptext + + # Check that we don't accidentally grab docstrings from the generic alias! + assert "The central part of internal API" not in helptext + + def test_tuple_generic_fixed(): @dataclasses.dataclass class TupleGenericFixed(Generic[ScalarType]): @@ -151,6 +184,22 @@ class Triangle(Generic[ScalarType]): _check_serialization_identity(Triangle[float], parsed_instance) +def test_multilevel_generic_no_helptext(): + @dataclasses.dataclass + class LineSegment(Generic[ScalarType]): + a: Point3[ScalarType] + b: Point3[ScalarType] + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli(LineSegment[int], args=["--help"]) + helptext = f.getvalue() + + # Check that we don't accidentally grab docstrings from the generic alias! + assert "The central part of internal API" not in helptext + + def test_generic_nested_dataclass(): @dataclasses.dataclass class Child: @@ -170,6 +219,28 @@ class DataclassGeneric(Generic[T]): _check_serialization_identity(DataclassGeneric[Child], parsed_instance) +def test_generic_nested_dataclass_helptext(): + @dataclasses.dataclass + class Child: + a: int + b: int + + T = TypeVar("T") + + @dataclasses.dataclass + class DataclassGeneric(Generic[T]): + child: T + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli(DataclassGeneric[Child], args=["--help"]) + helptext = f.getvalue() + + # Check that we don't accidentally grab docstrings from the generic alias! + assert "The central part of internal API" not in helptext + + def test_generic_subparsers(): @dataclasses.dataclass class CommandOne: diff --git a/tests/test_helptext.py b/tests/test_helptext.py index fe9d32e42..cd745d6f1 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -68,6 +68,41 @@ class ChildClass(UnrelatedParentClass, ActualParentClass): assert "--y INT Documentation 2\n" in helptext +def test_helptext_nested(): + """For nested classes, we should pull helptext from the outermost docstring if + possible. The class docstring can be used as a fallback.""" + + class Inner: + """Documented in class.""" + + def __init__(self, a: int): + pass + + def main_with_docstring(a: Inner) -> None: + """main_with_docstring. + + Args: + a: Documented in function. + """ + + def main_no_docstring(a: Inner) -> None: + pass + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli(main_with_docstring, args=["--help"]) + helptext = f.getvalue() + assert "Documented in function" in helptext and str(Inner.__doc__) not in helptext + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli(main_no_docstring, args=["--help"]) + helptext = f.getvalue() + assert "Documented in function" not in helptext and str(Inner.__doc__) in helptext + + def test_helptext_defaults(): class Color(enum.Enum): RED = enum.auto() diff --git a/tests/test_nested.py b/tests/test_nested.py index 9348e6b21..2b526b4b4 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -224,3 +224,31 @@ class OptionalSubparser: dcargs.cli( OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--y", "3"] ) + + +def test_post_init_default(): + @dataclasses.dataclass + class DataclassWithDynamicDefault: + x: int = 3 + y: Optional[int] = None + + def __post_init__(self): + # If unspecified, set y = x. + if self.y is None: + self.y = self.x + + @dataclasses.dataclass + class NoDefaultPostInitArgs: + inner: DataclassWithDynamicDefault + + @dataclasses.dataclass + class DefaultFactoryPostInitArgs: + inner: DataclassWithDynamicDefault = dataclasses.field( + default_factory=DataclassWithDynamicDefault + ) + + assert ( + dcargs.cli(NoDefaultPostInitArgs, args=["--inner.x", "5"]).inner + == dcargs.cli(DefaultFactoryPostInitArgs, args=["--inner.x", "5"]).inner + == DataclassWithDynamicDefault(x=5, y=5) + ) diff --git a/tests/test_positional_ignore_py37.py b/tests/test_positional_ignore_py37.py index 0cdfa6c24..30efbe879 100644 --- a/tests/test_positional_ignore_py37.py +++ b/tests/test_positional_ignore_py37.py @@ -55,3 +55,35 @@ def nest2(a: int, b: int, /, thing: B, c: int): assert isinstance(dcargs.cli(nest2, args="0 1 2 3 --thing.c 4 --c 4".split(" ")), B) with pytest.raises(SystemExit): dcargs.cli(nest2, args="0 1 2 3 4 --thing.c 4 --c 4".split(" ")) + + +def test_positional_with_underscores(): + """Hyphen replacement works a bit different for positional arguments.""" + + def main(a_multi_word_input: int, /) -> int: + return a_multi_word_input + + assert dcargs.cli(main, args=["5"]) == 5 + + +def test_positional_booleans(): + """Make sure that flag behavior is disabled for positional booleans.""" + + def main( + flag1: bool, + flag2: bool = True, + flag3: bool = False, + /, + ) -> Tuple[bool, bool, bool]: + return flag1, flag2, flag3 + + assert dcargs.cli(main, args=["True"]) == (True, True, False) + assert dcargs.cli(main, args=["True", "False"]) == (True, False, False) + assert dcargs.cli(main, args=["False", "False", "True"]) == (False, False, True) + + with pytest.raises(SystemExit): + dcargs.cli(main, args=["hmm"]) + with pytest.raises(SystemExit): + dcargs.cli(main, args=["true"]) + with pytest.raises(SystemExit): + dcargs.cli(main, args=["True", "false"]) From 9ff43857886a2c3947f387f628064eb78eb7d9a1 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 24 Jun 2022 11:04:09 -0700 Subject: [PATCH 045/491] Helptext fixes, tests for standard classes --- README.md | 36 +++++++------------------ dcargs/_docstrings.py | 14 +++++++++- dcargs/_fields.py | 2 +- dcargs/_parsers.py | 4 ++- examples/_format_examples_for_readme.py | 1 - tests/test_helptext.py | 15 ++++++++--- 6 files changed, 38 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index b8a28c52b..d430dde08 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,10 @@ -* [Overview](#overview) -* [Examples](#examples) -* [Serialization](#serialization) -* [Alternative tools](#alternative-tools) +- [Overview](#overview) +- [Examples](#examples) +- [Serialization](#serialization) +- [Alternative tools](#alternative-tools) @@ -760,28 +760,15 @@ $ python examples/08_standard_classes.py --help ``` usage: 08_standard_classes.py [-h] --field1 STR --field2 INT [--flag] -required arguments: - --field1 STR Arguments. - - Args: - field1: A string field. - field2: A numeric field. - flag: A boolean flag. - --field2 INT Arguments. +Arguments. - Args: - field1: A string field. - field2: A numeric field. - flag: A boolean flag. +required arguments: + --field1 STR A string field. + --field2 INT A numeric field. optional arguments: -h, --help show this help message and exit - --flag Arguments. - - Args: - field1: A string field. - field2: A numeric field. - flag: A boolean flag. + --flag A boolean flag. ``` @@ -919,7 +906,6 @@ optional arguments: -h, --help show this help message and exit required point_continuous arguments: - Point3(*args, **kwds) --point-continuous.x FLOAT --point-continuous.y FLOAT @@ -927,7 +913,6 @@ required point_continuous arguments: --point-continuous.frame-id STR required point_discrete arguments: - Point3(*args, **kwds) --point-discrete.x INT --point-discrete.y INT @@ -935,7 +920,6 @@ required point_discrete arguments: --point-discrete.frame-id STR required shape.a arguments: - Point3(*args, **kwds) --shape.a.x FLOAT --shape.a.y FLOAT @@ -943,7 +927,6 @@ required shape.a arguments: --shape.a.frame-id STR required shape.b arguments: - Point3(*args, **kwds) --shape.b.x FLOAT --shape.b.y FLOAT @@ -951,7 +934,6 @@ required shape.b arguments: --shape.b.frame-id STR required shape.c arguments: - Point3(*args, **kwds) --shape.c.x FLOAT --shape.c.y FLOAT diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index da057701b..2b7077d5f 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -75,9 +75,19 @@ def make(clz) -> "_ClassTokenization": for i, token in enumerate(tokens[:-1]): if token.token_type == tokenize.NAME: # Naive heuristic for field names. + is_first_token = True + for t in tokens_from_logical_line[token.logical_line]: + if t == token: + break + if t.token_type is not tokenize.COMMENT: + is_first_token = False + break + + if not is_first_token: + continue + if ( tokens[i + 1].content == ":" - and tokens[i] == tokens_from_actual_line[tokens[i].actual_line][0] and token.content not in field_data_from_name ): field_data_from_name[token.content] = _FieldData( @@ -217,6 +227,8 @@ def get_callable_description(f: Callable) -> str: # Ignore any default docstrings, as generated by `dataclasses.dataclass`. docstring = inspect.getdoc(f) + if docstring is None and isinstance(f, type): + docstring = inspect.getdoc(f.__init__) # type: ignore if docstring is None: return "" diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 50c9cc540..1c0e62c61 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -81,7 +81,7 @@ def field_list_from_callable( # Get helptext from docstring. helptext = docstring_from_arg_name.get(param.name) - if cls is not None: + if helptext is None and cls is not None: helptext = _docstrings.get_field_docstring(cls, param.name) field_list.append( diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index e3dd3fb5e..7b94f90f2 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -178,7 +178,9 @@ def from_callable( if field.helptext is not None: helptext_from_nested_class_field_name[field.name] = field.helptext else: - helptext_from_nested_class_field_name[field.name] = inspect.getdoc( + helptext_from_nested_class_field_name[ + field.name + ] = _docstrings.get_callable_description( _resolver.resolve_generic_types(field.typ)[0] ) continue diff --git a/examples/_format_examples_for_readme.py b/examples/_format_examples_for_readme.py index 39b1906b9..4598c7631 100644 --- a/examples/_format_examples_for_readme.py +++ b/examples/_format_examples_for_readme.py @@ -8,7 +8,6 @@ import pathlib import re import subprocess -from typing import Optional import dcargs diff --git a/tests/test_helptext.py b/tests/test_helptext.py index cd745d6f1..8355700d3 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -73,9 +73,12 @@ def test_helptext_nested(): possible. The class docstring can be used as a fallback.""" class Inner: - """Documented in class.""" - def __init__(self, a: int): + """Something + + Args: + a (int): Hello world! + """ pass def main_with_docstring(a: Inner) -> None: @@ -86,6 +89,7 @@ def main_with_docstring(a: Inner) -> None: """ def main_no_docstring(a: Inner) -> None: + """main_no_docstring.""" pass f = io.StringIO() @@ -94,13 +98,18 @@ def main_no_docstring(a: Inner) -> None: dcargs.cli(main_with_docstring, args=["--help"]) helptext = f.getvalue() assert "Documented in function" in helptext and str(Inner.__doc__) not in helptext + assert "Args:" not in helptext + assert "Hello world!" in helptext f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(main_no_docstring, args=["--help"]) helptext = f.getvalue() - assert "Documented in function" not in helptext and str(Inner.__doc__) in helptext + print(helptext) + assert "Something" in helptext + assert "Args:" not in helptext + assert "Hello world!" in helptext def test_helptext_defaults(): From 7006e83b8e5358c20a6b3581253b4bbfc8ee7f01 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 30 Jun 2022 01:43:03 -0700 Subject: [PATCH 046/491] NamedTuple+TypedDict support, tests, version bump --- README.md | 322 ++++++++++-------- _update_readme.py | 131 +++++++ dcargs/_arguments.py | 3 +- dcargs/_cli.py | 24 +- dcargs/_docstrings.py | 18 +- dcargs/_fields.py | 160 ++++++--- dcargs/_parsers.py | 22 +- dcargs/_resolver.py | 11 +- ...01_simple_functions.py => 01_functions.py} | 0 ...imple_dataclasses.py => 02_dataclasses.py} | 0 examples/03_enums_and_containers.py | 2 +- examples/11_typed_dictionaries.py | 19 ++ examples/12_named_tuples.py | 21 ++ examples/_format_examples_for_readme.py | 112 ------ setup.py | 2 +- tests/test_attrs.py | 3 +- tests/test_typeddict_namedtuple.py | 161 +++++++++ 17 files changed, 680 insertions(+), 331 deletions(-) create mode 100644 _update_readme.py rename examples/{01_simple_functions.py => 01_functions.py} (100%) rename examples/{02_simple_dataclasses.py => 02_dataclasses.py} (100%) create mode 100644 examples/11_typed_dictionaries.py create mode 100644 examples/12_named_tuples.py delete mode 100644 examples/_format_examples_for_readme.py create mode 100644 tests/test_typeddict_namedtuple.py diff --git a/README.md b/README.md index d430dde08..d8c1eac20 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,8 @@ pip install dcargs **`dcargs`** is a library for typed CLI interfaces and configuration objects. -Our core interface contains a single function for generating an argument parser -from a type-annotated callable _`f`_, which may be a function, class, or -dataclass: +Our core interface generates an argument parser from a type-annotated callable +_`f`_, which may be a function, class, or dataclass: --- @@ -41,12 +40,14 @@ dcargs.cli(
Docstring - -
Call `f(...)`, with arguments populated from an automatically generated CLI
+
+
+```
+Call `f(...)`, with arguments populated from an automatically generated CLI
 interface.
 
-`f` should have type-annotated inputs, and can be a function, class, or dataclass.
-Note that if `f` is a class, `dcargs.cli()` returns an instance.
+`f` should have type-annotated inputs, and can be a function or class. Note that if
+`f` is a class, `dcargs.cli()` returns an instance.
 
 The parser is generated by populating helptext from docstrings and types from
 annotations; a broad range of core type annotations are supported...
@@ -55,7 +56,7 @@ annotations; a broad range of core type annotations are supported...
     - Booleans, which are automatically converted to flags when provided a default
       value.
     - Enums (via `enum.Enum`).
-    - Various container types. Some examples:
+    - Various annotations from the standard typing library. Some examples:
       - `typing.ClassVar`.
       - `typing.Optional`.
       - `typing.Literal`.
@@ -67,28 +68,31 @@ annotations; a broad range of core type annotations are supported...
       - `typing.Final` and `typing.Annotated`.
       - Nested combinations of the above: `Optional[Literal[T]]`,
         `Final[Optional[Sequence[T]]]`, etc.
-    - Nested dataclasses.
+    - Nested structures; dataclasses, TypedDict, NamedTuple, classes.
       - Simple nesting.
-      - Unions over nested dataclasses (subparsers).
-      - Optional unions over nested dataclasses (optional subparsers).
-    - Generic dataclasses (including nested generics).
+      - Unions over nested structures (subparsers).
+      - Optional unions over nested structures (optional subparsers).
+    - Generics (including nested generics).
 
 Args:
     f: Callable.
 
 Keyword Args:
     description: Description text for the parser, displayed when the --help flag is
-        passed in. If not specified, `f`'s docstring is used. Mirrors argument
-        from `argparse.ArgumentParser()`.
+        passed in. If not specified, `f`'s docstring is used. Mirrors argument from
+        `argparse.ArgumentParser()`.
     args: If set, parse arguments from a sequence of strings instead of the
         commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`.
     default_instance: An instance of `T` to use for default values; only supported
-        if `T` is a dataclass type. Helpful for merging CLI arguments with values loaded
-        from elsewhere. (for example, a config object loaded from a yaml file)
+        if `T` is a dataclass, TypedDict, or NamedTuple. Helpful for merging CLI
+        arguments with values loaded from elsewhere. (for example, a config object
+        loaded from a yaml file)
 
 Returns:
-    The output of `f(...)`.
- + The output of `f(...)`. +``` + +
@@ -112,27 +116,23 @@ Ultimately, we aim to enable configuration interfaces that are: like `mypy` and `pyright`. - **Modular.** Most approaches to configuration objects require a centralized definition of all configurable fields. Supporting hierarchically nested - configuration classes/dataclasses, however, makes it easy to distribute - definitions, defaults, and documentation of configurable fields across modules - or source files. A model configuration dataclass, for example, can be - co-located in its entirety with the model implementation and dropped into any - experiment configuration with an import — this eliminates redundancy and makes - the entire module easy to port across codebases. + configuration structures, however, makes it easy to distribute definitions, + defaults, and documentation of configurable fields across modules or source + files. A model configuration dataclass, for example, can be co-located in its + entirety with the model implementation and dropped into any experiment + configuration with an import — this eliminates redundancy and makes the entire + module easy to port across codebases. ## Examples -A series of example scripts covering core features are included below. - - -
+
-1. Simple Functions +1. Functions - -[examples/01_simple_functions.py](examples/01_simple_functions.py) -
+[examples/01_functions.py](examples/01_functions.py) + ```python """CLI generation example from a simple annotated function. `dcargs.cli()` will call `main()`, with arguments populated from the CLI.""" @@ -161,12 +161,9 @@ if __name__ == "__main__": --- -```console -$ python examples/01_simple_functions.py --help -``` - -``` -usage: 01_simple_functions.py [-h] --field1 STR [--field2 INT] [--flag] +
+$ python examples/01_functions.py --help
+usage: 01_functions.py [-h] --field1 STR [--field2 INT] [--flag]
 
 Function, whose arguments will be populated from a CLI interface.
 
@@ -176,21 +173,19 @@ required arguments:
 optional arguments:
   -h, --help    show this help message and exit
   --field2 INT  A numeric field, with a default value. (default: 3)
-  --flag        A boolean flag.
-```
+  --flag        A boolean flag.
+
-
-2. Simple Dataclasses +2. Dataclasses - -[examples/02_simple_dataclasses.py](examples/02_simple_dataclasses.py) -
+[examples/02_dataclasses.py](examples/02_dataclasses.py) + ```python """Example using dcargs.cli() to instantiate a dataclass.""" @@ -218,12 +213,9 @@ if __name__ == "__main__": --- -```console -$ python examples/02_simple_dataclasses.py --help -``` - -``` -usage: 02_simple_dataclasses.py [-h] --field1 STR [--field2 INT] [--flag] +
+$ python examples/02_dataclasses.py --help
+usage: 02_dataclasses.py [-h] --field1 STR [--field2 INT] [--flag]
 
 Description.
 This should show up in the helptext!
@@ -234,21 +226,19 @@ required arguments:
 optional arguments:
   -h, --help    show this help message and exit
   --field2 INT  A numeric field, with a default value. (default: 3)
-  --flag        A boolean flag.
-```
+  --flag        A boolean flag.
+
-
3. Enums And Containers +
[examples/03_enums_and_containers.py](examples/03_enums_and_containers.py) -
- ```python """Examples of more advanced type annotations: enums and containers types. @@ -274,7 +264,7 @@ class TrainConfig: dataset_sources: Tuple[pathlib.Path, ...] """Paths to load training data from. This can be multiple!""" - # Fixed-length tupels are also okay: + # Fixed-length tuples are also okay: image_dimensions: Tuple[int, int] """Height and width of some image data.""" @@ -294,11 +284,8 @@ if __name__ == "__main__": --- -```console -$ python examples/03_enums_and_containers.py --help -``` - -``` +
+$ python examples/03_enums_and_containers.py --help
 usage: 03_enums_and_containers.py [-h] --dataset-sources PATH [PATH ...]
                                   --image-dimensions INT INT --optimizer-type
                                   {ADAM,SGD} [--checkpoint-interval INT]
@@ -314,21 +301,19 @@ required arguments:
 optional arguments:
   -h, --help            show this help message and exit
   --checkpoint-interval INT
-                        Interval to save checkpoints at. (default: None)
-```
+                        Interval to save checkpoints at. (default: None)
+
-
4. Flags +
[examples/04_flags.py](examples/04_flags.py) -
- ```python """Example of how booleans are handled and automatically converted to flags.""" @@ -362,11 +347,8 @@ if __name__ == "__main__": --- -```console -$ python examples/04_flags.py --help -``` - -``` +
+$ python examples/04_flags.py --help
 usage: 04_flags.py [-h] --boolean {True,False}
                    [--optional-boolean {True,False}] [--flag-a] [--no-flag-b]
 
@@ -379,21 +361,19 @@ optional arguments:
   --optional-boolean {True,False}
                         Optional boolean. Same as above, but can be omitted. (default: None)
   --flag-a              Pass --flag-a in to set this value to True.
-  --no-flag-b           Pass --no-flag-b in to set this value to False.
-```
+  --no-flag-b           Pass --no-flag-b in to set this value to False.
+
-
5. Hierarchical Configs +
[examples/05_hierarchical_configs.py](examples/05_hierarchical_configs.py) -
- ```python """An example of how we can create hierarchical configuration interfaces by nesting dataclasses.""" @@ -467,11 +447,8 @@ if __name__ == "__main__": --- -```console -$ python examples/05_hierarchical_configs.py --help -``` - -``` +
+$ python examples/05_hierarchical_configs.py --help
 usage: 05_hierarchical_configs.py [-h]
                                   [--config.optimizer.algorithm {ADAM,SGD}]
                                   [--config.optimizer.learning-rate FLOAT]
@@ -511,21 +488,19 @@ optional config arguments:
   --config.train-steps INT
                         Total number of training steps. (default: 100000)
   --config.seed INT     Random seed. This is helpful for making sure that our experiments are all
-                        reproducible! (default: 0)
-```
+                        reproducible! (default: 0)
+
-
6. Literals +
[examples/06_literals.py](examples/06_literals.py) -
- ```python """typing.Literal[] can be used to specify accepted input choices.""" @@ -564,11 +539,8 @@ if __name__ == "__main__": --- -```console -$ python examples/06_literals.py --help -``` - -``` +
+$ python examples/06_literals.py --help
 usage: 06_literals.py [-h] --enum {RED,GREEN,BLUE} --restricted-enum
                       {RED,GREEN} --integer {0,1,2,3} --string {red,green}
                       [--restricted-enum-with-default {RED,GREEN}]
@@ -588,21 +560,19 @@ optional arguments:
   --integer-with-default {0,1,2,3}
                         (default: 3)
   --string-with-Default {red,green}
-                        (default: red)
-```
+                        (default: red)
+
-
7. Positional Args +
[examples/07_positional_args.py](examples/07_positional_args.py) -
- ```python """Positional-only arguments in functions are converted to positional CLI arguments.""" @@ -674,11 +644,8 @@ if __name__ == "__main__": --- -```console -$ python examples/07_positional_args.py --help -``` - -``` +
+$ python examples/07_positional_args.py --help
 usage: 07_positional_args.py [-h] [--optimizer.algorithm {ADAM,SGD}]
                              [--optimizer.learning-rate FLOAT]
                              [--optimizer.weight-decay FLOAT] [--force]
@@ -707,21 +674,19 @@ optional optimizer arguments:
   --optimizer.learning-rate FLOAT
                         Learning rate to use. (default: 0.0003)
   --optimizer.weight-decay FLOAT
-                        Coefficient for L2 regularization. (default: 0.01)
-```
+                        Coefficient for L2 regularization. (default: 0.01)
+
-
8. Standard Classes + - + - + + + + +
[examples/08_standard_classes.py](examples/08_standard_classes.py) -
- ```python """In addition to functions and dataclasses, we can also generate CLIs from (the constructors of) standard Python classes.""" @@ -753,11 +718,8 @@ if __name__ == "__main__": --- -```console -$ python examples/08_standard_classes.py --help -``` - -``` +
+$ python examples/08_standard_classes.py --help
 usage: 08_standard_classes.py [-h] --field1 STR --field2 INT [--flag]
 
 Arguments.
@@ -768,21 +730,19 @@ required arguments:
 
 optional arguments:
   -h, --help    show this help message and exit
-  --flag        A boolean flag.
-```
+  --flag        A boolean flag.
+
-
9. Subparsers +
[examples/09_subparsers.py](examples/09_subparsers.py) -
- ```python """Unions over nested types (classes or dataclasses) will result in subparsers.""" @@ -819,32 +779,27 @@ if __name__ == "__main__": --- -```console -$ python examples/09_subparsers.py --help -``` - -``` +
+$ python examples/09_subparsers.py --help
 usage: 09_subparsers.py [-h] {checkout,commit} ...
 
 optional arguments:
   -h, --help         show this help message and exit
 
 subcommands:
-  {checkout,commit}
-```
+  {checkout,commit}
+
-
10. Generics +
[examples/10_generics.py](examples/10_generics.py) -
- ```python """Example of parsing for generic (~templated) dataclasses.""" @@ -886,11 +841,8 @@ if __name__ == "__main__": --- -```console -$ python examples/10_generics.py --help -``` - -``` +
+$ python examples/10_generics.py --help
 usage: 10_generics.py [-h] --point-continuous.x FLOAT --point-continuous.y
                       FLOAT --point-continuous.z FLOAT
                       --point-continuous.frame-id STR --point-discrete.x INT
@@ -938,12 +890,114 @@ required shape.c arguments:
   --shape.c.x FLOAT
   --shape.c.y FLOAT
   --shape.c.z FLOAT
-  --shape.c.frame-id STR
+  --shape.c.frame-id STR
+
+ +
+ +
+ +11. Typed Dictionaries + +
+ +[examples/11_typed_dictionaries.py](examples/11_typed_dictionaries.py) + +```python +"""Dictionary inputs can be specified using a TypedDict type. + +TODO: note that setting total=False is not yet (but could be) supported.""" + +from typing import TypedDict + +import dcargs + + +class DictionarySchema(TypedDict): + field1: str # A string field. + field2: int # A numeric field. + field3: bool # A boolean field. + + +if __name__ == "__main__": + x = dcargs.cli(DictionarySchema) + assert isinstance(x, dict) + print(x) ``` +--- + +
+$ python examples/11_typed_dictionaries.py --help
+usage: 11_typed_dictionaries.py [-h] --field1 STR --field2 INT --field3
+                                {True,False}
+
+required arguments:
+  --field1 STR          A string field.
+  --field2 INT          A numeric field.
+  --field3 {True,False}
+                        A boolean field.
+
+optional arguments:
+  -h, --help            show this help message and exit
+
+
- +
+ +12. Named Tuples + +
+ +[examples/12_named_tuples.py](examples/12_named_tuples.py) + +```python +"""Example using dcargs.cli() to instantiate a named tuple.""" + +from typing import NamedTuple + +import dcargs + + +class TupleType(NamedTuple): + """Description. + This should show up in the helptext!""" + + field1: str # A string field. + field2: int = 3 # A numeric field, with a default value. + flag: bool = False # A boolean flag. + + +if __name__ == "__main__": + print(TupleType.__doc__) + x = dcargs.cli(TupleType) + assert isinstance(x, tuple) + print(x) +``` + +--- + +
+$ python examples/12_named_tuples.py --help
+Description.
+    This should show up in the helptext!
+usage: 12_named_tuples.py [-h] --field1 STR [--field2 INT] [--flag]
+
+Description.
+This should show up in the helptext!
+
+required arguments:
+  --field1 STR  A string field.
+
+optional arguments:
+  -h, --help    show this help message and exit
+  --field2 INT  A numeric field, with a default value. (default: 3)
+  --flag        A boolean flag.
+
+ +
+
## Serialization diff --git a/_update_readme.py b/_update_readme.py new file mode 100644 index 000000000..f37639336 --- /dev/null +++ b/_update_readme.py @@ -0,0 +1,131 @@ +"""Helper script for updating the auto-generated parts of the README. (docstring, +examples list)""" + +import concurrent.futures +import dataclasses +import inspect +import pathlib +import re +import subprocess + +import dcargs + + +@dataclasses.dataclass(frozen=True) +class Constants: + docstring_start: str = "" + docstring_marker_end: str = "" + + examples_marker_start: str = "" + examples_marker_end: str = "" + + +def replace_between_markers( + content: str, marker_start: str, marker_end: str, inner: str +) -> str: + """Puts `inner` between `marker_start` and `marker_end`.""" + assert marker_start in content + assert marker_end in content + before, _, after = content.partition(marker_start) + _, _, after = content.rpartition(marker_end) + return "".join([before, marker_start, inner, marker_end, after]) + + +def format_script_for_readme(path: pathlib.Path) -> str: + print("Handling:", path) + + # 01_functions -> 01, _, functions. + index, _, title = path.stem.partition("_") + + # 01 -> 1. + index = str(int(index)) + + # functions -> Functions. + title = title.replace("_", " ").title() + + source = path.read_text().strip() + helptext = subprocess.run( + args=["python", str(path), "--help"], stdout=subprocess.PIPE, encoding="utf8" + ).stdout + helptext = re.sub( # Strip colorcodes. + r"\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))", "", helptext + ).strip() + + return f""" +
+ +{index}. {title} + +
+ +[{path}]({path}) + +```python +{source} +``` + +--- + +
+$ python {path} --help
+{helptext}
+
+ +
+
+ """.strip() + + +def get_examples_str(examples_dir: pathlib.Path, constants: Constants) -> str: + script_paths = sorted(examples_dir.glob("*.py")) + with concurrent.futures.ThreadPoolExecutor() as executor: + return "\n".join( + executor.map( + format_script_for_readme, + script_paths, + ) + ) + + +REPO_ROOT = pathlib.Path(__file__).parent + + +def main( + readme_path: pathlib.Path = REPO_ROOT / "README.md", + examples_dir: pathlib.Path = REPO_ROOT / "examples", + constants: Constants = Constants(), +) -> None: + """Helper script for generating the examples list in the README.""" + + # Read. + content = readme_path.read_text(encoding="utf8") + + # Update examples. + content = replace_between_markers( + content, + constants.examples_marker_start, + constants.examples_marker_end, + get_examples_str(examples_dir, constants), + ) + + # Update docstring. + content = replace_between_markers( + content, + constants.docstring_start, + constants.docstring_marker_end, + f"\n\n```\n{inspect.getdoc(dcargs.cli)}\n```\n\n", + ) + + # Write. + readme_path.write_text(content) + + # Format. + try: + subprocess.run(args=["prettier", "-w", readme_path.as_posix()]) + print("Successfully formatted with `prettier`!") + except FileNotFoundError: + print("Tried to format README with `prettier`, but not in PATH.") + + +if __name__ == "__main__": + dcargs.cli(main) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index dc3d38d6c..959fdcfe9 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -136,7 +136,8 @@ def _transform_recursive_instantiator_from_type( arg: ArgumentDefinition, type_from_typevar: Dict[TypeVar, Type], ) -> ArgumentDefinition: - """The bulkiest bit: recursively analyze the type annotation and use it to determine how""" + """The bulkiest bit: recursively analyze the type annotation and use it to determine + how to instantiate it given some string from the commandline.""" if arg.instantiator is not None: return arg diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 4a566c587..14b4be85f 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -1,7 +1,7 @@ import argparse from typing import Callable, Optional, Sequence, TypeVar -from . import _calling, _parsers, _resolver +from . import _calling, _parsers T = TypeVar("T") @@ -35,8 +35,8 @@ def cli( """Call `f(...)`, with arguments populated from an automatically generated CLI interface. - `f` should have type-annotated inputs, and can be a function, class, or dataclass. - Note that if `f` is a class, `dcargs.parse()` returns an instance. + `f` should have type-annotated inputs, and can be a function or class. Note that if + `f` is a class, `dcargs.cli()` returns an instance. The parser is generated by populating helptext from docstrings and types from annotations; a broad range of core type annotations are supported... @@ -45,7 +45,7 @@ def cli( - Booleans, which are automatically converted to flags when provided a default value. - Enums (via `enum.Enum`). - - Various container types. Some examples: + - Various annotations from the standard typing library. Some examples: - `typing.ClassVar`. - `typing.Optional`. - `typing.Literal`. @@ -57,11 +57,11 @@ def cli( - `typing.Final` and `typing.Annotated`. - Nested combinations of the above: `Optional[Literal[T]]`, `Final[Optional[Sequence[T]]]`, etc. - - Nested dataclasses. + - Nested structures; dataclasses, TypedDict, NamedTuple, classes. - Simple nesting. - - Unions over nested dataclasses (subparsers). - - Optional unions over nested dataclasses (optional subparsers). - - Generic dataclasses (including nested generics). + - Unions over nested structures (subparsers). + - Optional unions over nested structures (optional subparsers). + - Generics (including nested generics). Args: f: Callable. @@ -73,17 +73,15 @@ def cli( args: If set, parse arguments from a sequence of strings instead of the commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. default_instance: An instance of `T` to use for default values; only supported - if `T` is a dataclass type. Helpful for merging CLI arguments with values loaded - from elsewhere. (for example, a config object loaded from a yaml file) + if `T` is a dataclass, TypedDict, or NamedTuple. Helpful for merging CLI + arguments with values loaded from elsewhere. (for example, a config object + loaded from a yaml file) Returns: The output of `f(...)`. """ # Map a callable to the relevant CLI arguments + subparsers. - assert default_instance is None or _resolver.is_dataclass( - f - ), "Default instance specification is only supported for dataclasses!" parser_definition = _parsers.ParserSpecification.from_callable( f, description=description, diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 2b7077d5f..405cf3109 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -7,7 +7,7 @@ from typing import Callable, Dict, List, Optional, Type import docstring_parser -from typing_extensions import get_origin +from typing_extensions import get_origin, is_typeddict from . import _resolver, _strings @@ -225,13 +225,21 @@ def get_callable_description(f: Callable) -> str: these docstrings.""" f, _unused = _resolver.resolve_generic_types(f) - # Ignore any default docstrings, as generated by `dataclasses.dataclass`. - docstring = inspect.getdoc(f) - if docstring is None and isinstance(f, type): - docstring = inspect.getdoc(f.__init__) # type: ignore + # Note inspect.getdoc() causes some corner cases with TypedDicts. + docstring = f.__doc__ + if ( + docstring is None + and isinstance(f, type) + # Ignore TypedDict's __init__ docstring, because it will just be `dict` + and not is_typeddict(f) + and not _resolver.is_namedtuple(f) + ): + docstring = f.__init__.__doc__ # type: ignore if docstring is None: return "" + docstring = _strings.dedent(docstring) + if dataclasses.is_dataclass(f): default_doc = f.__name__ + str(inspect.signature(f)).replace(" -> None", "") if docstring == default_doc: diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 1c0e62c61..2638ae6ff 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -6,7 +6,7 @@ from typing import Any, Callable, List, Optional, Type, TypeVar import docstring_parser -from typing_extensions import get_type_hints +from typing_extensions import get_type_hints, is_typeddict from . import _docstrings, _resolver @@ -26,75 +26,127 @@ class Field: def field_list_from_callable( f: Callable[..., T], default_instance: Optional[T] ) -> List[Field]: - """Generate a list of generic 'field' objects corresponding to an input callable. + """Generate a list of generic 'field' objects corresponding to the inputs of some + annotated callable. `f` can be from a dataclass type, regular class type, or function.""" # Unwrap generics. f, _unused_type_from_typevar = _resolver.resolve_generic_types(f) - # Handling for class inputs vs function inputs. - ignore_self = False + # If `f` is a type: + # 1. Set cls to the type. + # 2. Consider `f` to be `cls.__init__`. cls: Optional[Type] = None if isinstance(f, type): - # If `f` is a type: - # 1. Set cls to the type. - # 2. Consider `f` to be `cls.__init__`. cls = f f = cls.__init__ # type: ignore ignore_self = True - # Get type annotations, docstrings. - hints = get_type_hints(f) - docstring = inspect.getdoc(f) - docstring_from_arg_name = {} - if docstring is not None: - for param_doc in docstring_parser.parse(docstring).params: - docstring_from_arg_name[param_doc.arg_name] = param_doc.description - del docstring - - # Special handling for dataclasses: take `field(default_factory=...)` and - # `default_instance` input into acocunt. - default_from_dataclass_field_name = {} - if cls is not None and _resolver.is_dataclass(cls): - for field in _resolver.resolved_fields(cls): - default_from_dataclass_field_name[ - field.name - ] = _get_dataclass_field_default(field, default_instance) + if cls is not None and is_typeddict(cls): + # Handle typed dictionaries. + field_list = [] + assert default_instance is None or isinstance(default_instance, dict) + for name, typ in get_type_hints(cls).items(): + field_list.append( + Field( + name=name, + typ=typ, + default=None + if default_instance is None + else default_instance.get(name, None), + helptext=_docstrings.get_field_docstring(cls, name), + positional=False, + ) + ) + return field_list + elif cls is not None and _resolver.is_namedtuple(cls): + # Handle NamedTuples. + # + # TODO: in terms of helptext, we currently do display the default NamedTuple + # helptext. But we (intentionally) don't for dataclasses; this is somewhat + # inconsistent. + field_list = [] + field_defaults = getattr(cls, "_field_defaults") + + # Note that _field_types is removed in Python 3.9. + for name, typ in _resolver.get_type_hints(cls).items(): + # Get default, with priority for `default_instance`. + default = field_defaults.get(name) + if default_instance is not None and hasattr(default_instance, name): + default = getattr(default_instance, name) + + field_list.append( + Field( + name=name, + typ=typ, + default=default, + helptext=_docstrings.get_field_docstring(cls, name), + positional=False, + ) + ) + return field_list + elif cls is not None and _resolver.is_dataclass(cls): + # Handle dataclasses. + field_list = [] + for dc_field in filter( + lambda field: field.init, _resolver.resolved_fields(cls) + ): + field_list.append( + Field( + name=dc_field.name, + typ=dc_field.type, + default=_get_dataclass_field_default(dc_field, default_instance), + helptext=_docstrings.get_field_docstring(cls, dc_field.name), + positional=False, + ) + ) + return field_list else: + # Handle general callables. assert ( default_instance is None - ), "`default_instance` is only supported for dataclass types." - - # Generate field list from function signature. - field_list = [] - for param in inspect.signature(f).parameters.values(): - # For `__init__`, skip self parameter. - if ignore_self: - ignore_self = False - continue - - # Get default value. - default = default_from_dataclass_field_name.get(param.name, param.default) - if default is inspect.Parameter.empty: - default = None - - # Get helptext from docstring. - helptext = docstring_from_arg_name.get(param.name) - if helptext is None and cls is not None: - helptext = _docstrings.get_field_docstring(cls, param.name) - - field_list.append( - Field( - name=param.name, - # Note that param.annotation does not resolve forward references. - typ=hints[param.name], - default=default, - helptext=helptext, - positional=param.kind is inspect.Parameter.POSITIONAL_ONLY, + ), "`default_instance` is only supported for dataclass and TypedDict types." + + # Get type annotations, docstrings. + hints = get_type_hints(f) + docstring = inspect.getdoc(f) + docstring_from_arg_name = {} + if docstring is not None: + for param_doc in docstring_parser.parse(docstring).params: + docstring_from_arg_name[param_doc.arg_name] = param_doc.description + del docstring + + # Generate field list from function signature. + field_list = [] + ignore_self = cls is not None + for param in inspect.signature(f).parameters.values(): + # For `__init__`, skip self parameter. + if ignore_self: + ignore_self = False + continue + + # Get default value. + default = param.default + if default is inspect.Parameter.empty: + default = None + + # Get helptext from docstring. + helptext = docstring_from_arg_name.get(param.name) + if helptext is None and cls is not None: + helptext = _docstrings.get_field_docstring(cls, param.name) + + field_list.append( + Field( + name=param.name, + # Note that param.annotation does not resolve forward references. + typ=hints[param.name], + default=default, + helptext=helptext, + positional=param.kind is inspect.Parameter.POSITIONAL_ONLY, + ) ) - ) - return field_list + return field_list def _ensure_dataclass_instance_used_as_default_is_frozen( diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 7b94f90f2..9bd583ed0 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -23,7 +23,7 @@ import termcolor import typing_extensions -from typing_extensions import get_args, get_origin +from typing_extensions import get_args, get_origin, is_typeddict from . import _arguments, _docstrings, _fields, _instantiators, _resolver, _strings @@ -67,9 +67,13 @@ def _is_possibly_nested_type(typ: Any) -> bool: if dataclasses.is_dataclass(typ): return True - # Non-parsable types like nested (data)classes should have fully type-annotated - # inputs. If any inputs are unannotated (for example, in the case of pathlib.Path), - # we can assume the type is parsable. + # TypedDict types can be unpacked. + if is_typeddict(typ): + return True + + # Nested types like nested (data)classes should have fully type-annotated inputs. If + # any inputs are unannotated (for example, in the case of pathlib.Path), we can + # assume the type is not nested. for param in inspect.signature(typ).parameters.values(): if param.annotation is inspect.Parameter.empty: return False @@ -104,6 +108,9 @@ def from_callable( type_from_typevar[typevar] = parent_type_from_typevar[typ] # type: ignore # Cycle detection. + # + # Note that 'parent' here refers to in the nesting hierarchy, not the + # superclass. if f in parent_classes: raise _instantiators.UnsupportedTypeAnnotationError( f"Found a cyclic dataclass dependency with type {f}." @@ -117,11 +124,10 @@ def from_callable( args = [] helptext_from_nested_class_field_name = {} subparsers = None - field_list = _fields.field_list_from_callable( - f=f, default_instance=default_instance - ) - for field in field_list: + for field in _fields.field_list_from_callable( + f=f, default_instance=default_instance + ): field = dataclasses.replace( field, typ=type_from_typevar.get(field.typ, field.typ), # type: ignore diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 90f003e13..e68cfe6a7 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -23,7 +23,7 @@ def resolve_generic_types( class, and a mapping from typevars to concrete types.""" origin_cls = get_origin(cls) - if origin_cls is not None: + if origin_cls is not None and hasattr(origin_cls, "__parameters__"): typevars = origin_cls.__parameters__ typevar_values = get_args(cls) assert len(typevars) == len(typevar_values) @@ -48,3 +48,12 @@ def resolved_fields(cls: Type) -> List[dataclasses.Field]: fields.append(field) return fields + + +def is_namedtuple(cls: Type) -> bool: + return ( + hasattr(cls, "_fields") + # Remove in Python >=3.9. + # and hasattr(cls, "_field_types") + and hasattr(cls, "_field_defaults") + ) diff --git a/examples/01_simple_functions.py b/examples/01_functions.py similarity index 100% rename from examples/01_simple_functions.py rename to examples/01_functions.py diff --git a/examples/02_simple_dataclasses.py b/examples/02_dataclasses.py similarity index 100% rename from examples/02_simple_dataclasses.py rename to examples/02_dataclasses.py diff --git a/examples/03_enums_and_containers.py b/examples/03_enums_and_containers.py index d9d4269e4..3fb86abee 100644 --- a/examples/03_enums_and_containers.py +++ b/examples/03_enums_and_containers.py @@ -22,7 +22,7 @@ class TrainConfig: dataset_sources: Tuple[pathlib.Path, ...] """Paths to load training data from. This can be multiple!""" - # Fixed-length tupels are also okay: + # Fixed-length tuples are also okay: image_dimensions: Tuple[int, int] """Height and width of some image data.""" diff --git a/examples/11_typed_dictionaries.py b/examples/11_typed_dictionaries.py new file mode 100644 index 000000000..f0faabcd0 --- /dev/null +++ b/examples/11_typed_dictionaries.py @@ -0,0 +1,19 @@ +"""Dictionary inputs can be specified using a TypedDict type. + +TODO: note that setting total=False is not yet (but could be) supported.""" + +from typing import TypedDict + +import dcargs + + +class DictionarySchema(TypedDict): + field1: str # A string field. + field2: int # A numeric field. + field3: bool # A boolean field. + + +if __name__ == "__main__": + x = dcargs.cli(DictionarySchema) + assert isinstance(x, dict) + print(x) diff --git a/examples/12_named_tuples.py b/examples/12_named_tuples.py new file mode 100644 index 000000000..bc11bbff4 --- /dev/null +++ b/examples/12_named_tuples.py @@ -0,0 +1,21 @@ +"""Example using dcargs.cli() to instantiate a named tuple.""" + +from typing import NamedTuple + +import dcargs + + +class TupleType(NamedTuple): + """Description. + This should show up in the helptext!""" + + field1: str # A string field. + field2: int = 3 # A numeric field, with a default value. + flag: bool = False # A boolean flag. + + +if __name__ == "__main__": + print(TupleType.__doc__) + x = dcargs.cli(TupleType) + assert isinstance(x, tuple) + print(x) diff --git a/examples/_format_examples_for_readme.py b/examples/_format_examples_for_readme.py deleted file mode 100644 index 4598c7631..000000000 --- a/examples/_format_examples_for_readme.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Helper script for generating the examples list in the README. - -Generally, invoked as: -`python examples/_format_examples_for_readme.py --readme-path ./README.md` -""" - -import concurrent.futures -import pathlib -import re -import subprocess - -import dcargs - -README_MARKER_START = "" -README_MARKER_END = "" -SCRIPT_FORMAT_TEMPLATE = """ -
- -{index}. {title} - - -[{path}]({path}) - -
- -```python -{source} -``` - ---- - -```console -$ python {path} --help -``` -``` -{helptext} -``` - -
-
-""" - - -def update_readme(readme_path: pathlib.Path, inner: str) -> None: - """Puts `inner` in between {README_MARKER_START} and {README_MARKER_END}.""" - readme_contents = readme_path.read_text() - assert README_MARKER_START in readme_contents - assert README_MARKER_END in readme_contents - before, _, after = readme_contents.partition(README_MARKER_START) - _, _, after = readme_contents.rpartition(README_MARKER_END) - readme_path.write_text( - "".join([before, README_MARKER_START, inner, README_MARKER_END, after]) - ) - print(f"Wrote update examples to {readme_path}!") - try: - subprocess.run(args=["prettier", "-w", readme_path.as_posix()]) - print("Successfully formatted with `prettier`!") - except FileNotFoundError: - print("Tried to format README with `prettier`, but not in PATH.") - - -def format_script_for_readme(index: int, path: pathlib.Path) -> str: - title = " ".join( - map( - # Capitalize first letter of each word. - lambda lower: lower[0:1].upper() + lower[1:], - # Remove integer prefix. - path.stem.split("_")[1:], - ) - ) - - source = path.read_text().strip() - - helptext = subprocess.run( - args=["python", str(path), "--help"], stdout=subprocess.PIPE, encoding="utf8" - ).stdout - helptext = re.sub( # Strip colorcodes. - r"\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))", "", helptext - ).strip() - - return SCRIPT_FORMAT_TEMPLATE.format( - index=index, title=title, path=path, source=source, helptext=helptext - ) - - -def main(readme_path: pathlib.Path) -> None: - """Helper script for generating the examples list in the README. - - Args: - readme_path: README file to write to; we replace the contents between - the start and end markers. If not specified, output is printed. - """ - script_paths = sorted( - filter( - lambda p: not p.name.startswith("_"), - pathlib.Path(__file__).parent.glob("*.py"), - ) - ) - - with concurrent.futures.ThreadPoolExecutor() as executor: - out_str = "\n".join( - executor.map( - format_script_for_readme, - range(1, len(script_paths) + 1), - script_paths, - ) - ) - update_readme(readme_path, out_str) - - -if __name__ == "__main__": - dcargs.cli(main) diff --git a/setup.py b/setup.py index 564bd79d6..5d35c7f1d 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.1.3", + version="0.1.4", description="Strongly typed, zero-effort CLIs", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_attrs.py b/tests/test_attrs.py index dc2e373b2..600cc59ad 100644 --- a/tests/test_attrs.py +++ b/tests/test_attrs.py @@ -1,4 +1,5 @@ -"""Tests for attrs. This is not officially supported, but should work.""" +"""Tests for attrs. This is not officially supported, but most features should work. +(exceptions include default factories)""" import contextlib import io diff --git a/tests/test_typeddict_namedtuple.py b/tests/test_typeddict_namedtuple.py new file mode 100644 index 000000000..af3f836d4 --- /dev/null +++ b/tests/test_typeddict_namedtuple.py @@ -0,0 +1,161 @@ +import contextlib +import io +import pathlib +from typing import NamedTuple + +import pytest +from typing_extensions import TypedDict + +import dcargs + + +def test_basic_typeddict(): + class ManyTypesTypedDict(TypedDict): + i: int + s: str + f: float + p: pathlib.Path + + assert dcargs.cli( + ManyTypesTypedDict, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--p", + "~", + ], + ) == dict(i=5, s="5", f=5.0, p=pathlib.Path("~")) + + +def test_nested_typeddict(): + class ChildTypedDict(TypedDict): + y: int + + class NestedTypedDict(TypedDict): + x: int + b: ChildTypedDict + + assert dcargs.cli(NestedTypedDict, args=["--x", "1", "--b.y", "3"]) == dict( + x=1, b=dict(y=3) + ) + with pytest.raises(SystemExit): + dcargs.cli(NestedTypedDict, args=["--x", "1"]) + + +def test_helptext_and_default_instance_typeddict(): + class HelptextTypedDict(TypedDict): + """This docstring should be printed as a description.""" + + x: int # Documentation 1 + + # Documentation 2 + y: int + + z: int + """Documentation 3""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli(HelptextTypedDict, default_instance={"z": 3}, args=["--help"]) + helptext = f.getvalue() + assert HelptextTypedDict.__doc__ in helptext + assert ":\n --x INT Documentation 1\n" in helptext + assert "--y INT Documentation 2\n" in helptext + assert "--z INT Documentation 3 (default: 3)\n" in helptext + + +def test_basic_namedtuple(): + class ManyTypesNamedTuple(NamedTuple): + i: int + s: str + f: float + p: pathlib.Path + + assert dcargs.cli( + ManyTypesNamedTuple, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--p", + "~", + ], + ) == ManyTypesNamedTuple(i=5, s="5", f=5.0, p=pathlib.Path("~")) + + +def test_nested_namedtuple(): + class ChildNamedTuple(NamedTuple): + y: int + + class NestedNamedTuple(NamedTuple): + x: int + b: ChildNamedTuple + + assert dcargs.cli( + NestedNamedTuple, args=["--x", "1", "--b.y", "3"] + ) == NestedNamedTuple(x=1, b=ChildNamedTuple(y=3)) + with pytest.raises(SystemExit): + dcargs.cli(NestedNamedTuple, args=["--x", "1"]) + + +def test_helptext_and_default_namedtuple(): + class HelptextNamedTupleDefault(NamedTuple): + """This docstring should be printed as a description.""" + + x: int # Documentation 1 + + # Documentation 2 + y: int + + z: int = 3 + """Documentation 3""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli(HelptextNamedTupleDefault, args=["--help"]) + helptext = f.getvalue() + assert HelptextNamedTupleDefault.__doc__ in helptext + assert ":\n --x INT Documentation 1\n" in helptext + assert "--y INT Documentation 2\n" in helptext + assert "--z INT Documentation 3 (default: 3)\n" in helptext + + +def test_helptext_and_default_instance_namedtuple(): + class HelptextNamedTuple(NamedTuple): + """This docstring should be printed as a description.""" + + x: int # Documentation 1 + + # Documentation 2 + y: int + + z: int + """Documentation 3""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli( + HelptextNamedTuple, + default_instance=HelptextNamedTuple( + # Sketchy, unsupported behavior... + x=None, # type: ignore + y=None, # type: ignore + z=3, + ), + args=["--help"], + ) + helptext = f.getvalue() + assert HelptextNamedTuple.__doc__ in helptext + assert ":\n --x INT Documentation 1\n" in helptext + assert "--y INT Documentation 2\n" in helptext + assert "--z INT Documentation 3 (default: 3)\n" in helptext From 9ea521ac09faea965f8330f14f48912b1abbabcf Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 4 Jul 2022 18:49:22 -0700 Subject: [PATCH 047/491] Argument def cleanup, nested positional tests --- dcargs/_arguments.py | 290 ++++++++++++++------------- dcargs/_calling.py | 9 +- dcargs/_parsers.py | 12 +- dcargs/_strings.py | 2 +- setup.py | 1 + tests/test_positional_ignore_py37.py | 18 +- 6 files changed, 182 insertions(+), 150 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 959fdcfe9..63bc20ce2 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -3,31 +3,79 @@ import argparse import dataclasses import enum +import functools import shlex from typing import Any, Dict, Optional, Set, Tuple, Type, TypeVar, Union from . import _fields, _instantiators +try: + # Python >=3.8. + from functools import cached_property +except ImportError: + # Python 3.7. + from backports.cached_property import cached_property # type: ignore + @dataclasses.dataclass(frozen=True) class ArgumentDefinition: - """Options for defining arguments. Contains all necessary arguments for argparse's - add_argument() method.""" + """Structure containing everything needed to define an argument.""" prefix: str # Prefix for nesting. - field: _fields.Field # Corresponding dataclass field. + field: _fields.Field + type_from_typevar: Dict[TypeVar, Type] + + def add_argument( + self, parser: Union[argparse.ArgumentParser, argparse._ArgumentGroup] + ) -> None: + """Add a defined argument to a parser.""" + + # Get keyword arguments, with None values removed. + kwargs = dataclasses.asdict(self.lowered) + kwargs = dict(filter(lambda kv: kv[1] is not None, kwargs.items())) + + # Pop lowered properties t + kwargs.pop("instantiator") + name_or_flag = kwargs.pop("name_or_flag") + + # Note that the name must be passed in as a position argument. + parser.add_argument(name_or_flag, **kwargs) + + @cached_property + def lowered(self) -> LoweredArgumentDefinition: + """Lowered argument definition, generated by applying a sequence of rules.""" + rules = ( + _rule_handle_defaults, + _rule_handle_boolean_flags, + _rule_recursive_instantiator_from_type, + _rule_convert_defaults_to_strings, + _rule_generate_helptext, + _rule_set_name_or_flag, + _rule_positional_special_handling, + ) + return functools.reduce( + lambda lowered, rule: rule(self, lowered), + rules, + LoweredArgumentDefinition(), + ) + + +@dataclasses.dataclass(frozen=True) +class LoweredArgumentDefinition: + """Contains fields meant to be passed directly into argparse.""" # Action that is called on parsed arguments. This handles conversions from strings # to our desired types. - instantiator: Optional[_instantiators.Instantiator] + # + # The main reason we use this instead of the standard 'type' argument is to enable + # mixed-type tuples. + instantiator: Optional[_instantiators.Instantiator] = None - # Fields that will be populated initially. - # Important: from here on out, all fields correspond 1:1 to inputs to argparse's + # From here on out, all fields correspond 1:1 to inputs to argparse's # add_argument() method. - type: Optional[Union[Type, TypeVar]] - default: Optional[Any] - - # Fields that will be handled by argument transformations. + name_or_flag: str = "" + default: Optional[Any] = None + dest: Optional[str] = None required: bool = False action: Optional[str] = None nargs: Optional[Union[int, str]] = None @@ -35,129 +83,106 @@ class ArgumentDefinition: metavar: Optional[Union[str, Tuple[str, ...]]] = None help: Optional[str] = None - def add_argument( - self, parser: Union[argparse.ArgumentParser, argparse._ArgumentGroup] - ) -> None: - """Add a defined argument to a parser.""" - kwargs = {k: v for k, v in vars(self).items() if v is not None} - - # Apply prefix for nested dataclasses. - assert "dest" not in kwargs - if not self.field.positional: - kwargs["dest"] = self.prefix + self.field.name - - # Important: as far as argparse is concerned, all inputs are strings. - # - # Conversions from strings to our desired types happen in the "field action"; - # this is a bit more flexible, and lets us handle more complex types like enums - # and multi-type tuples. - if "type" in kwargs: - kwargs["type"] = str - if "choices" in kwargs: - kwargs["choices"] = list(map(str, kwargs["choices"])) - - kwargs.pop("prefix") - kwargs.pop("field") - kwargs.pop("instantiator") - - # Note that the name must be passed in as a position argument. - parser.add_argument(self.get_name_or_flag(), **kwargs) - - def get_name_or_flag(self) -> str: - """Get name (for positional args) or flag (for keyword args), with a prefix - applied for nested dataclasses.""" - if self.field.positional: - return self.prefix + self.field.name - elif self.action == "store_false": - return "--" + (self.prefix + "no-" + self.field.name).replace("_", "-") - else: - return "--" + (self.prefix + self.field.name).replace("_", "-") - - @staticmethod - def from_field( - field: _fields.Field, - type_from_typevar: Dict[TypeVar, Type], - ) -> ArgumentDefinition: - arg = ArgumentDefinition( - prefix="", - field=field, - instantiator=None, - type=field.typ, - default=field.default, - ) - arg = _transform_required_if_default_set(arg) - arg = _transform_handle_boolean_flags(arg) - arg = _transform_recursive_instantiator_from_type(arg, type_from_typevar) - arg = _transform_generate_helptext(arg) - arg = _transform_convert_defaults_to_strings(arg) - arg = _transform_positional_special_handling(arg) - return arg - -def _transform_required_if_default_set(arg: ArgumentDefinition) -> ArgumentDefinition: +def _rule_handle_defaults( + arg: ArgumentDefinition, + lowered: LoweredArgumentDefinition, +) -> LoweredArgumentDefinition: """Set `required=True` if a default value is set.""" - # Mark arg as required if a default is set. - if arg.default is None: - return dataclasses.replace(arg, required=True) + # Mark lowered as required if a default is set. + if arg.field.default is None: + return dataclasses.replace(lowered, required=True) - return dataclasses.replace(arg) + return dataclasses.replace(lowered, default=arg.field.default) -def _transform_handle_boolean_flags(arg: ArgumentDefinition) -> ArgumentDefinition: - """""" - if arg.type is not bool: - return arg +def _rule_handle_boolean_flags( + arg: ArgumentDefinition, + lowered: LoweredArgumentDefinition, +) -> LoweredArgumentDefinition: + if arg.type_from_typevar.get(arg.field.typ, arg.field.typ) is not bool: # type: ignore + return lowered - if arg.default is None or arg.field.positional: + if lowered.default is None or arg.field.positional: # If no default is passed in, we treat bools as a normal parameter. - return arg - elif arg.default is False: + return lowered + elif lowered.default is False: # Default `False` => --flag passed in flips to `True`. return dataclasses.replace( - arg, + lowered, action="store_true", - type=None, instantiator=lambda x: x, # argparse will directly give us a bool! ) - elif arg.default is True: + elif lowered.default is True: # Default `True` => --no-flag passed in flips to `False`. return dataclasses.replace( - arg, + lowered, action="store_false", - type=None, instantiator=lambda x: x, # argparse will directly give us a bool! ) else: assert False, "Invalid default" -def _transform_recursive_instantiator_from_type( +def _rule_recursive_instantiator_from_type( arg: ArgumentDefinition, - type_from_typevar: Dict[TypeVar, Type], -) -> ArgumentDefinition: + lowered: LoweredArgumentDefinition, +) -> LoweredArgumentDefinition: """The bulkiest bit: recursively analyze the type annotation and use it to determine - how to instantiate it given some string from the commandline.""" - if arg.instantiator is not None: - return arg + how to instantiate it given some string from the commandline. + + Important: as far as argparse is concerned, all inputs are strings. + + Conversions from strings to our desired types happen in the instantiator; this is a + bit more flexible, and lets us handle more complex types like enums and multi-type + tuples.""" + if lowered.instantiator is not None: + return lowered instantiator, metadata = _instantiators.instantiator_from_type( - arg.type, # type: ignore - type_from_typevar, + arg.field.typ, # type: ignore + arg.type_from_typevar, ) return dataclasses.replace( - arg, + lowered, instantiator=instantiator, - choices=metadata.choices, + choices=tuple(map(str, metadata.choices)) + if metadata.choices is not None + else None, nargs=metadata.nargs, - required=(not metadata.is_optional) and arg.required, + required=(not metadata.is_optional) and lowered.required, # Ignore metavar if choices is set. metavar=metadata.metavar if metadata.choices is None else None, ) -def _transform_generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: - """Generate helptext from docstring and argument name.""" +def _rule_convert_defaults_to_strings( + arg: ArgumentDefinition, + lowered: LoweredArgumentDefinition, +) -> LoweredArgumentDefinition: + """Sets all default values to strings, as required as input to our instantiator + functions. Special-cased for enums.""" + + def as_str(x: Any) -> str: + if isinstance(x, enum.Enum): + return x.name + else: + return str(x) + + if lowered.default is None or lowered.action is not None: + return lowered + elif lowered.nargs is not None and lowered.nargs != "?": + return dataclasses.replace(lowered, default=tuple(map(as_str, lowered.default))) + else: + return dataclasses.replace(lowered, default=as_str(lowered.default)) + + +def _rule_generate_helptext( + arg: ArgumentDefinition, + lowered: LoweredArgumentDefinition, +) -> LoweredArgumentDefinition: + """Generate helptext from docstring, argument name, default values.""" help_parts = [] docstring_help = arg.field.helptext @@ -168,65 +193,58 @@ def _transform_generate_helptext(arg: ArgumentDefinition) -> ArgumentDefinition: docstring_help = docstring_help.replace("%", "%%") help_parts.append(docstring_help) elif arg.field.positional: - help_parts.append(str(arg.metavar)) + help_parts.append(str(lowered.metavar)) - if arg.action is not None: + if lowered.action is not None: # Don't show defaults for boolean flags. - assert arg.action in ("store_true", "store_false") - elif arg.default is not None and isinstance(arg.default, enum.Enum): - # Special case for enums. - help_parts.append(f"(default: {arg.default.name})") - elif not arg.required: + assert lowered.action in ("store_true", "store_false") + elif not lowered.required: # Include default value in helptext. We intentionally don't use the % template # because the types of all arguments are set to strings, which will cause the # default to be casted to a string and introduce extra quotation marks. - if arg.nargs is not None and hasattr(arg.default, "__iter__"): - # For tuple types, we might have arg.default as (0, 1, 2, 3). - # For list types, we might have arg.default as [0, 1, 2, 3]. - # For set types, we might have arg.default as {0, 1, 2, 3}. + if lowered.nargs is not None and hasattr(lowered.default, "__iter__"): + # For tuple types, we might have lowered.default as (0, 1, 2, 3). + # For list types, we might have lowered.default as [0, 1, 2, 3]. + # For set types, we might have lowered.default as {0, 1, 2, 3}. # # In all cases, we want to display (default: 0 1 2 3), for consistency with # the format that argparse expects when we set nargs. - assert arg.default is not None # Just for type checker. - default_parts = map(shlex.quote, map(str, arg.default)) + assert lowered.default is not None # Just for type checker. + default_parts = map(shlex.quote, map(str, lowered.default)) help_parts.append(f"(default: {' '.join(default_parts)})") else: - help_parts.append(f"(default: {shlex.quote(str(arg.default))})") + help_parts.append(f"(default: {shlex.quote(str(lowered.default))})") - return dataclasses.replace(arg, help=" ".join(help_parts)) + return dataclasses.replace(lowered, help=" ".join(help_parts)) -def _transform_convert_defaults_to_strings( +def _rule_set_name_or_flag( arg: ArgumentDefinition, -) -> ArgumentDefinition: - """Sets all default values to strings, as required as input to our instantiator - functions. Special-cased for enums.""" - - def as_str(x: Any) -> str: - if isinstance(x, enum.Enum): - return x.name - else: - return str(x) - - if arg.default is None or arg.action is not None: - return arg - elif arg.nargs is not None and arg.nargs != "?": - return dataclasses.replace(arg, default=tuple(map(as_str, arg.default))) + lowered: LoweredArgumentDefinition, +) -> LoweredArgumentDefinition: + if arg.field.positional: + name_or_flag = arg.prefix + arg.field.name + elif lowered.action == "store_false": + name_or_flag = "--" + (arg.prefix + "no-" + arg.field.name).replace("_", "-") else: - return dataclasses.replace(arg, default=as_str(arg.default)) + name_or_flag = "--" + (arg.prefix + arg.field.name).replace("_", "-") + return dataclasses.replace( + lowered, name_or_flag=name_or_flag, dest=arg.prefix + arg.field.name + ) -def _transform_positional_special_handling( - arg: ArgumentDefinition, -) -> ArgumentDefinition: - """Special handling for positional args.""" +def _rule_positional_special_handling( + arg: ArgumentDefinition, + lowered: LoweredArgumentDefinition, +) -> LoweredArgumentDefinition: if not arg.field.positional: - return arg + return lowered return dataclasses.replace( - arg, + lowered, + dest=None, metavar=(arg.prefix + arg.field.name).upper(), required=None, - nargs="?" if not arg.required else arg.nargs, + nargs="?" if not lowered.required else lowered.nargs, ) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 620bdc400..70a98d072 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -63,11 +63,11 @@ def get_value_from_arg(arg: str) -> Any: value = get_value_from_arg(prefixed_field_name) if value is not None: try: - assert arg.instantiator is not None - value = arg.instantiator(value) + assert arg.lowered.instantiator is not None + value = arg.lowered.instantiator(value) except ValueError as e: raise InstantiationError( - f"Parsing error for {arg.get_name_or_flag()}: {e.args[0]}" + f"Parsing error for {arg.lowered.name_or_flag}: {e.args[0]}" ) elif ( prefixed_field_name @@ -78,8 +78,7 @@ def get_value_from_arg(arg: str) -> Any: field_type, parser_definition, value_from_arg, - field_name_prefix=prefixed_field_name - + _strings.NESTED_DATACLASS_DELIMETER, + field_name_prefix=prefixed_field_name + _strings.NESTED_FIELD_DELIMETER, ) consumed_keywords |= consumed_keywords_child else: diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 9bd583ed0..72a303894 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -172,13 +172,13 @@ def from_callable( dataclasses.replace( arg, prefix=field.name - + _strings.NESTED_DATACLASS_DELIMETER + + _strings.NESTED_FIELD_DELIMETER + arg.prefix, ) ) for k, v in nested_parser.helptext_from_nested_class_field_name.items(): helptext_from_nested_class_field_name[ - field.name + _strings.NESTED_DATACLASS_DELIMETER + k + field.name + _strings.NESTED_FIELD_DELIMETER + k ] = v if field.helptext is not None: @@ -193,7 +193,9 @@ def from_callable( # (3) Handle primitive types. These produce a single argument! args.append( - _arguments.ArgumentDefinition.from_field(field, type_from_typevar) + _arguments.ArgumentDefinition( + prefix="", field=field, type_from_typevar=type_from_typevar + ) ) return ParserSpecification( @@ -248,7 +250,7 @@ def format_group_name(nested_field_name: str, required: bool) -> str: arg.add_argument(positional_group) continue - if arg.required: + if arg.lowered.required: target_groups, other_groups = ( required_group_from_prefix, optional_group_from_prefix, @@ -262,7 +264,7 @@ def format_group_name(nested_field_name: str, required: bool) -> str: if arg.prefix not in target_groups: nested_field_name = arg.prefix[:-1] target_groups[arg.prefix] = parser.add_argument_group( - format_group_name(nested_field_name, required=arg.required), + format_group_name(nested_field_name, required=arg.lowered.required), # Add a description, but only to the first group for a field. description=self.helptext_from_nested_class_field_name[ nested_field_name diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 9cfbc78d4..173368445 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -9,7 +9,7 @@ from . import _resolver -NESTED_DATACLASS_DELIMETER: str = "." +NESTED_FIELD_DELIMETER: str = "." SUBPARSER_DEST_FMT: str = "{name} (positional)" diff --git a/setup.py b/setup.py index 5d35c7f1d..43446b248 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ "typing_extensions>=4.0.0", "pyyaml", "termcolor", + "backports.cached-property; python_version < '3.8.0'", ], extras_require={ "testing": [ diff --git a/tests/test_positional_ignore_py37.py b/tests/test_positional_ignore_py37.py index 30efbe879..eb1d9d3c9 100644 --- a/tests/test_positional_ignore_py37.py +++ b/tests/test_positional_ignore_py37.py @@ -1,3 +1,5 @@ +import contextlib +import io from typing import List, Tuple import pytest @@ -33,16 +35,26 @@ def main( def test_nested_positional(): class A: - def __init__(self, a: int, b: int, /, c: int): - pass + def __init__(self, a: int, hello_world: int, /, c: int): + self.hello_world = hello_world - def nest1(a: int, b: int, thing: A, /, c: int): + def nest1(a: int, b: int, thing: A, /, c: int) -> A: return thing assert isinstance(dcargs.cli(nest1, args="0 1 2 3 --thing.c 4 --c 4".split(" ")), A) + assert ( + dcargs.cli(nest1, args="0 1 2 3 --thing.c 4 --c 4".split(" ")).hello_world == 3 + ) with pytest.raises(SystemExit): dcargs.cli(nest1, args="0 1 2 3 4 --thing.c 4 --c 4".split(" ")) + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli(nest1, args=["--help"]) + helptext = f.getvalue() + assert "THING.HELLO_WORLD" in helptext + def test_nested_positional_alt(): class B: From 512aa3f566317262d736a97b15fd8fd55fa05706 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 5 Jul 2022 02:16:22 -0700 Subject: [PATCH 048/491] Support Dict[T1,T2], behavior tweaks, tests --- README.md | 72 +++++++++----- dcargs/_arguments.py | 33 ++++--- dcargs/_cli.py | 20 ++-- dcargs/_fields.py | 24 ++++- dcargs/_instantiators.py | 97 ++++++++++++++++--- examples/11_dictionaries.py | 33 +++++++ examples/11_typed_dictionaries.py | 19 ---- setup.py | 5 +- tests/test_dcargs.py | 10 +- ..._namedtuple.py => test_dict_namedtuple.py} | 43 +++++++- tests/test_forward_ref.py | 2 - ...py => test_unsupported_but_should_work.py} | 55 ++++++++++- 12 files changed, 323 insertions(+), 90 deletions(-) create mode 100644 examples/11_dictionaries.py delete mode 100644 examples/11_typed_dictionaries.py rename tests/{test_typeddict_namedtuple.py => test_dict_namedtuple.py} (74%) rename tests/{test_attrs.py => test_unsupported_but_should_work.py} (51%) diff --git a/README.md b/README.md index d8c1eac20..e140e2137 100644 --- a/README.md +++ b/README.md @@ -57,18 +57,20 @@ annotations; a broad range of core type annotations are supported... value. - Enums (via `enum.Enum`). - Various annotations from the standard typing library. Some examples: - - `typing.ClassVar`. - - `typing.Optional`. - - `typing.Literal`. - - `typing.Sequence`. - - `typing.List`. + - `typing.ClassVar[T]`. + - `typing.Optional[T]`. + - `typing.Literal[T]`. + - `typing.Sequence[T]`. + - `typing.List[T]`. + - `typing.Dict[K, V]`. - `typing.Tuple`, such as `typing.Tuple[T1, T2, T3]` or `typing.Tuple[T, ...]`. - - `typing.Set`. - - `typing.Final` and `typing.Annotated`. - - Nested combinations of the above: `Optional[Literal[T]]`, + - `typing.Set[T]`. + - `typing.Final[T]` and `typing.Annotated[T]`. + - Various nested combinations of the above: `Optional[Literal[T]]`, `Final[Optional[Sequence[T]]]`, etc. - - Nested structures; dataclasses, TypedDict, NamedTuple, classes. + - Hierarchical structures via nested dataclasses, TypedDict, NamedTuple, + classes. - Simple nesting. - Unions over nested structures (subparsers). - Optional unions over nested structures (optional subparsers). @@ -897,18 +899,20 @@ required shape.c arguments:
-11. Typed Dictionaries +11. Dictionaries
-[examples/11_typed_dictionaries.py](examples/11_typed_dictionaries.py) +[examples/11_dictionaries.py](examples/11_dictionaries.py) ```python -"""Dictionary inputs can be specified using a TypedDict type. +"""Dictionary inputs can be specified using either a standard Dict[T1, T2] annotation, +or a TypedDict type. -TODO: note that setting total=False is not yet (but could be) supported.""" +Note that setting total=False for TypedDicts is currently not (but reasonably could be) +supported.""" -from typing import TypedDict +from typing import Dict, TypedDict import dcargs @@ -919,27 +923,45 @@ class DictionarySchema(TypedDict): field3: bool # A boolean field. +def main( + standard_dict: Dict[int, bool], + typed_dict: DictionarySchema = { + "field1": "hey", + "field2": 3, + "field3": False, + }, +) -> None: + assert isinstance(standard_dict, dict) + assert isinstance(typed_dict, dict) + print("Standard dict:", standard_dict) + print("Typed dict:", typed_dict) + + if __name__ == "__main__": - x = dcargs.cli(DictionarySchema) - assert isinstance(x, dict) - print(x) + dcargs.cli(main) ``` --- 
-$ python examples/11_typed_dictionaries.py --help
-usage: 11_typed_dictionaries.py [-h] --field1 STR --field2 INT --field3
-                                {True,False}
+$ python examples/11_dictionaries.py --help
+usage: 11_dictionaries.py [-h] --standard-dict INT {True,False}
+                          [INT {True,False} ...] [--typed-dict.field1 STR]
+                          [--typed-dict.field2 INT] [--typed-dict.field3]
 
 required arguments:
-  --field1 STR          A string field.
-  --field2 INT          A numeric field.
-  --field3 {True,False}
-                        A boolean field.
+  --standard-dict INT {True,False} [INT {True,False} ...]
 
 optional arguments:
-  -h, --help            show this help message and exit
+  -h, --help            show this help message and exit
+
+optional typed_dict arguments:
+
+  --typed-dict.field1 STR
+                        A string field. (default: hey)
+  --typed-dict.field2 INT
+                        A numeric field. (default: 3)
+  --typed-dict.field3   A boolean field.
diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 63bc20ce2..3a3120350 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -4,8 +4,9 @@ import dataclasses import enum import functools +import itertools import shlex -from typing import Any, Dict, Optional, Set, Tuple, Type, TypeVar, Union +from typing import Any, Dict, Mapping, Optional, Set, Tuple, Type, TypeVar, Union from . import _fields, _instantiators @@ -104,17 +105,14 @@ def _rule_handle_boolean_flags( if arg.type_from_typevar.get(arg.field.typ, arg.field.typ) is not bool: # type: ignore return lowered - if lowered.default is None or arg.field.positional: - # If no default is passed in, we treat bools as a normal parameter. - return lowered - elif lowered.default is False: + if lowered.default is False and not arg.field.positional: # Default `False` => --flag passed in flips to `True`. return dataclasses.replace( lowered, action="store_true", instantiator=lambda x: x, # argparse will directly give us a bool! ) - elif lowered.default is True: + elif lowered.default is True and not arg.field.positional: # Default `True` => --no-flag passed in flips to `False`. return dataclasses.replace( lowered, @@ -122,7 +120,8 @@ def _rule_handle_boolean_flags( instantiator=lambda x: x, # argparse will directly give us a bool! ) else: - assert False, "Invalid default" + # Treat bools as a normal parameter. + return lowered def _rule_recursive_instantiator_from_type( @@ -147,13 +146,10 @@ def _rule_recursive_instantiator_from_type( return dataclasses.replace( lowered, instantiator=instantiator, - choices=tuple(map(str, metadata.choices)) - if metadata.choices is not None - else None, + choices=metadata.choices, nargs=metadata.nargs, required=(not metadata.is_optional) and lowered.required, - # Ignore metavar if choices is set. - metavar=metadata.metavar if metadata.choices is None else None, + metavar=metadata.metavar, ) @@ -173,7 +169,15 @@ def as_str(x: Any) -> str: if lowered.default is None or lowered.action is not None: return lowered elif lowered.nargs is not None and lowered.nargs != "?": - return dataclasses.replace(lowered, default=tuple(map(as_str, lowered.default))) + if isinstance(lowered.default, Mapping): + return dataclasses.replace( + lowered, + default=tuple(map(as_str, itertools.chain(*lowered.default.items()))), + ) + else: + return dataclasses.replace( + lowered, default=tuple(map(as_str, lowered.default)) + ) else: return dataclasses.replace(lowered, default=as_str(lowered.default)) @@ -197,7 +201,8 @@ def _rule_generate_helptext( if lowered.action is not None: # Don't show defaults for boolean flags. - assert lowered.action in ("store_true", "store_false") + # assert lowered.action in ("store_true", "store_false") + pass elif not lowered.required: # Include default value in helptext. We intentionally don't use the % template # because the types of all arguments are set to strings, which will cause the diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 14b4be85f..fd95c7530 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -46,18 +46,20 @@ def cli( value. - Enums (via `enum.Enum`). - Various annotations from the standard typing library. Some examples: - - `typing.ClassVar`. - - `typing.Optional`. - - `typing.Literal`. - - `typing.Sequence`. - - `typing.List`. + - `typing.ClassVar[T]`. + - `typing.Optional[T]`. + - `typing.Literal[T]`. + - `typing.Sequence[T]`. + - `typing.List[T]`. + - `typing.Dict[K, V]`. - `typing.Tuple`, such as `typing.Tuple[T1, T2, T3]` or `typing.Tuple[T, ...]`. - - `typing.Set`. - - `typing.Final` and `typing.Annotated`. - - Nested combinations of the above: `Optional[Literal[T]]`, + - `typing.Set[T]`. + - `typing.Final[T]` and `typing.Annotated[T]`. + - Various nested combinations of the above: `Optional[Literal[T]]`, `Final[Optional[Sequence[T]]]`, etc. - - Nested structures; dataclasses, TypedDict, NamedTuple, classes. + - Hierarchical structures via nested dataclasses, TypedDict, NamedTuple, + classes. - Simple nesting. - Unions over nested structures (subparsers). - Optional unions over nested structures (optional subparsers). diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 2638ae6ff..caa9e3100 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -164,12 +164,21 @@ def _ensure_dataclass_instance_used_as_default_is_frozen( ) +_missing_types = [dataclasses.MISSING] +try: + import omegaconf + + _missing_types.append(omegaconf.MISSING) +except ImportError: + pass + + def _get_dataclass_field_default( field: dataclasses.Field, parent_default_instance: Any ) -> Optional[Any]: """Helper for getting the default instance for a field.""" field_default_instance = None - if field.default is not dataclasses.MISSING: + if field.default not in _missing_types: # Populate default from usual default value, or # `dataclasses.field(default=...)`. field_default_instance = field.default @@ -177,7 +186,7 @@ def _get_dataclass_field_default( _ensure_dataclass_instance_used_as_default_is_frozen( field, field_default_instance ) - elif field.default_factory is not dataclasses.MISSING and not ( + elif field.default_factory not in _missing_types and not ( # Special case to ignore default_factory if we write: # `field: Dataclass = dataclasses.field(default_factory=Dataclass)`. # @@ -191,9 +200,18 @@ def _get_dataclass_field_default( and field.default_factory is field.type ): # Populate default from `dataclasses.field(default_factory=...)`. + assert callable(field.default_factory) field_default_instance = field.default_factory() if parent_default_instance is not None: # Populate default from some parent, eg `default_instance` in `dcargs.cli()`. - field_default_instance = getattr(parent_default_instance, field.name) + if hasattr(parent_default_instance, field.name): + field_default_instance = getattr(parent_default_instance, field.name) + else: + warnings.warn( + f"Could not find field {field.name} in default instance" + f" {parent_default_instance}, which has" + f" type {type(parent_default_instance)},", + stacklevel=2, + ) return field_default_instance diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index 3e83ed17b..496cf0040 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -37,6 +37,8 @@ import dataclasses import enum import inspect +import warnings +from collections.abc import Mapping from typing import ( Any, Callable, @@ -70,7 +72,7 @@ class InstantiatorMetadata: nargs: Optional[Union[str, int]] metavar: Union[str, Tuple[str, ...]] - choices: Optional[Tuple[Any, ...]] + choices: Optional[Tuple[str, ...]] is_optional: bool @@ -104,6 +106,15 @@ def instantiator_from_type( type_from_typevar, ) + if typ is Any: + warnings.warn("Found field with type `Any`, which will be parsed as a string.") + return lambda arg: arg, InstantiatorMetadata( + nargs=None, + metavar="ANY", + choices=None, + is_optional=False, + ) + # Address container types. If a matching container is found, this will recursively # call instantiator_from_type(). container_out = _instantiator_from_container_type(typ, type_from_typevar) @@ -148,12 +159,14 @@ def instantiator_from_type( auto_choices: Optional[Tuple[str, ...]] = None if typ is bool: auto_choices = ("True", "False") - elif issubclass(typ, enum.Enum): + elif isinstance(typ, type) and issubclass(typ, enum.Enum): auto_choices = tuple(x.name for x in typ) return lambda arg: _strings.instance_from_string(typ, arg), InstantiatorMetadata( nargs=None, - metavar=typ.__name__.upper(), + metavar=typ.__name__.upper() + if auto_choices is None + else "{" + ",".join(map(str, auto_choices)) + "}", choices=auto_choices, is_optional=False, ) @@ -186,7 +199,10 @@ def _instantiator_from_container_type( container_type = list make, inner_meta = _instantiator_from_type_inner( - contained_type, type_from_typevar + contained_type, + type_from_typevar, + allow_sequences=False, + allow_optional=False, ) return lambda strings: container_type( [make(x) for x in strings] @@ -210,7 +226,10 @@ def _instantiator_from_container_type( (contained_type,) = typeset_no_ellipsis make, inner_meta = _instantiator_from_type_inner( - contained_type, type_from_typevar + contained_type, + type_from_typevar, + allow_sequences=False, + allow_optional=False, ) return lambda strings: tuple( [make(x) for x in strings] @@ -224,7 +243,12 @@ def _instantiator_from_container_type( else: instantiators, metas = zip( *map( - lambda t: _instantiator_from_type_inner(t, type_from_typevar), + lambda t: _instantiator_from_type_inner( + t, + type_from_typevar, + allow_sequences=False, + allow_optional=False, + ), types, ) ) @@ -254,7 +278,10 @@ def _instantiator_from_container_type( ) (typ,) = options - {type(None)} instantiator, metadata = _instantiator_from_type_inner( - typ, type_from_typevar, allow_sequences=True + typ, + type_from_typevar, + allow_sequences=True, + allow_optional=False, ) return instantiator, dataclasses.replace(metadata, is_optional=True) @@ -269,14 +296,62 @@ def _instantiator_from_container_type( if issubclass(contained_type, enum.Enum): choices = tuple(map(lambda x: x.name, choices)) instantiator, metadata = _instantiator_from_type_inner( - contained_type, type_from_typevar + contained_type, + type_from_typevar, + allow_sequences=False, + allow_optional=False, ) assert ( # Choices provided by the contained type metadata.choices is None or len(set(choices) - set(metadata.choices)) == 0 ) - return instantiator, dataclasses.replace(metadata, choices=choices) + return instantiator, dataclasses.replace( + metadata, + choices=tuple(map(str, choices)), + metavar="{" + ",".join(map(str, choices)) + "}", + ) + + # Dictionaries. + if type_origin in (dict, Mapping): + key_type, val_type = get_args(typ) + key_instantiator, key_metadata = _instantiator_from_type_inner( + key_type, + type_from_typevar, + allow_sequences=False, + allow_optional=False, + ) + val_instantiator, val_metadata = _instantiator_from_type_inner( + val_type, + type_from_typevar, + allow_sequences=False, + allow_optional=False, + ) + + def dict_instantiator(strings: List[str]) -> Any: + out = {} + if len(strings) % 2 != 0: + raise ValueError("incomplete set of key value pairs!") + for i in range(len(strings) // 2): + k = strings[i * 2] + v = strings[i * 2 + 1] + if key_metadata.choices is not None and k not in key_metadata.choices: + raise ValueError( + f"invalid choice: {k} (choose from {key_metadata.choices}))" + ) + if val_metadata.choices is not None and v not in val_metadata.choices: + raise ValueError( + f"invalid choice: {v} (choose from {val_metadata.choices}))" + ) + out[key_instantiator(k)] = val_instantiator(v) # type: ignore + return out + + return dict_instantiator, InstantiatorMetadata( + nargs="+", + metavar=f"{key_metadata.metavar} {val_metadata.metavar}", + choices=None, + is_optional=False, + ) raise UnsupportedTypeAnnotationError( # pragma: no cover f"Unsupported type {typ} with origin {type_origin}" @@ -286,8 +361,8 @@ def _instantiator_from_container_type( def _instantiator_from_type_inner( typ: Type, type_from_typevar: Dict[TypeVar, Type], - allow_sequences: bool = False, - allow_optional: bool = False, + allow_sequences: bool, + allow_optional: bool, ) -> Tuple[Instantiator, InstantiatorMetadata]: """Thin wrapper over instantiator_from_type, with some extra asserts for catching errors.""" diff --git a/examples/11_dictionaries.py b/examples/11_dictionaries.py new file mode 100644 index 000000000..b20d478f8 --- /dev/null +++ b/examples/11_dictionaries.py @@ -0,0 +1,33 @@ +"""Dictionary inputs can be specified using either a standard Dict[T1, T2] annotation, +or a TypedDict type. + +Note that setting total=False for TypedDicts is currently not (but reasonably could be) +supported.""" + +from typing import Dict, TypedDict + +import dcargs + + +class DictionarySchema(TypedDict): + field1: str # A string field. + field2: int # A numeric field. + field3: bool # A boolean field. + + +def main( + standard_dict: Dict[int, bool], + typed_dict: DictionarySchema = { + "field1": "hey", + "field2": 3, + "field3": False, + }, +) -> None: + assert isinstance(standard_dict, dict) + assert isinstance(typed_dict, dict) + print("Standard dict:", standard_dict) + print("Typed dict:", typed_dict) + + +if __name__ == "__main__": + dcargs.cli(main) diff --git a/examples/11_typed_dictionaries.py b/examples/11_typed_dictionaries.py deleted file mode 100644 index f0faabcd0..000000000 --- a/examples/11_typed_dictionaries.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Dictionary inputs can be specified using a TypedDict type. - -TODO: note that setting total=False is not yet (but could be) supported.""" - -from typing import TypedDict - -import dcargs - - -class DictionarySchema(TypedDict): - field1: str # A string field. - field2: int # A numeric field. - field3: bool # A boolean field. - - -if __name__ == "__main__": - x = dcargs.cli(DictionarySchema) - assert isinstance(x, dict) - print(x) diff --git a/setup.py b/setup.py index 43446b248..6017fdab3 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.1.4", + version="0.1.5", description="Strongly typed, zero-effort CLIs", long_description=long_description, long_description_content_type="text/markdown", @@ -18,7 +18,7 @@ python_requires=">=3.7", install_requires=[ "docstring_parser", - "typing_extensions>=4.0.0", + "typing_extensions>=4.3.0", "pyyaml", "termcolor", "backports.cached-property; python_version < '3.8.0'", @@ -27,6 +27,7 @@ "testing": [ "pytest", "pytest-cov", + "omegaconf", "attrs", ], "type-checking": [ diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 678d8c22d..94ae28e68 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -1,7 +1,7 @@ import dataclasses import enum import pathlib -from typing import ClassVar, Optional +from typing import Any, ClassVar, Optional import pytest from typing_extensions import Annotated, Final, Literal, TypeAlias @@ -340,3 +340,11 @@ def add(a: SomeTypeAlias, b: SomeTypeAlias) -> SomeTypeAlias: return a + b assert dcargs.cli(add, args=["--a", "5", "--b", "7"]) == 12 + + +@pytest.mark.filterwarnings("ignore::Warning") +def test_any(): + def main(x: Any) -> Any: + return x + + assert dcargs.cli(main, args=["--x", "hello"]) == "hello" diff --git a/tests/test_typeddict_namedtuple.py b/tests/test_dict_namedtuple.py similarity index 74% rename from tests/test_typeddict_namedtuple.py rename to tests/test_dict_namedtuple.py index af3f836d4..3eda0d86d 100644 --- a/tests/test_typeddict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -1,14 +1,53 @@ import contextlib import io import pathlib -from typing import NamedTuple +from typing import Any, Dict, Mapping, NamedTuple import pytest -from typing_extensions import TypedDict +from typing_extensions import Literal, TypedDict import dcargs +def test_basic_dict(): + def main(params: Dict[str, int]) -> Dict[str, int]: + return params + + assert dcargs.cli(main, args="--params hey 5 hello 2".split(" ")) == { + "hey": 5, + "hello": 2, + } + assert dcargs.cli(main, args="--params hey 5 hello 2".split(" ")) == { + "hey": 5, + "hello": 2, + } + with pytest.raises(SystemExit): + dcargs.cli(main, args="--params hey 5 hello hey".split(" ")) + with pytest.raises(SystemExit): + dcargs.cli(main, args="--params hey 5 hello".split(" ")) + with pytest.raises(SystemExit): + dcargs.cli(main, args="--params".split(" ")) + + +def test_dict_with_default(): + def main(params: Mapping[Literal[1, 3, 5, 7], bool] = {5: False, 1: True}) -> Any: + return params + + assert dcargs.cli(main, args=[]) == {5: False, 1: True} + assert dcargs.cli(main, args="--params 5 True 3 False".split(" ")) == { + 5: True, + 3: False, + } + with pytest.raises(SystemExit): + dcargs.cli(main, args="--params".split(" ")) + with pytest.raises(SystemExit): + dcargs.cli(main, args="--params 5 Tru 3 False".split(" ")) + with pytest.raises(SystemExit): + dcargs.cli(main, args="--params 5 Tru 3 False".split(" ")) + with pytest.raises(SystemExit): + dcargs.cli(main, args="--params 4 Tru 3 False".split(" ")) + + def test_basic_typeddict(): class ManyTypesTypedDict(TypedDict): i: int diff --git a/tests/test_forward_ref.py b/tests/test_forward_ref.py index 05120796f..5915e2478 100644 --- a/tests/test_forward_ref.py +++ b/tests/test_forward_ref.py @@ -29,7 +29,6 @@ class C: def test_forward_ref_1(): - assert dcargs.cli(A1, args=["--x", "1", "b", "--y", "3"]) == A1(x=1, bc=B(y=3)) assert dcargs.cli(A1, args=["--x", "1", "c", "--z", "3"]) == A1(x=1, bc=C(z=3)) @@ -40,7 +39,6 @@ def test_forward_ref_1(): def test_forward_ref_2(): - assert dcargs.cli(A2, args=["--x", "1", "b", "--y", "3"]) == A2(x=1, bc=B(y=3)) assert dcargs.cli(A2, args=["--x", "1", "c", "--z", "3"]) == A2(x=1, bc=C(z=3)) diff --git a/tests/test_attrs.py b/tests/test_unsupported_but_should_work.py similarity index 51% rename from tests/test_attrs.py rename to tests/test_unsupported_but_should_work.py index 600cc59ad..05e4a6465 100644 --- a/tests/test_attrs.py +++ b/tests/test_unsupported_but_should_work.py @@ -1,16 +1,67 @@ -"""Tests for attrs. This is not officially supported, but most features should work. -(exceptions include default factories)""" +"""Tests for features that are not officially features, but should work. + +Includes things like omegaconf.MISSING, attrs, etc, which mostly work but either likely +have corner cases or just seem sketchy. +""" import contextlib import io import pathlib +from typing import Tuple import attr +import omegaconf import pytest import dcargs +def test_missing(): + """Passing in a 'missing' default; this will mark an argument as required.""" + + def main( + required_a: int, + optional: int = 3, + required_b: int = None, # type: ignore + ) -> Tuple[int, int, int]: + return (required_a, optional, required_b) # type: ignore + + assert dcargs.cli( + main, args="--required-a 3 --optional 4 --required-b 5".split(" ") + ) == (3, 4, 5) + assert dcargs.cli(main, args="--required-a 3 --required-b 5".split(" ")) == ( + 3, + 3, + 5, + ) + + with pytest.raises(SystemExit): + dcargs.cli(main, args="--required-a 3 --optional 4") + with pytest.raises(SystemExit): + dcargs.cli(main, args="--required-a 3") + + def main2( + required_a: int, + optional: int = 3, + required_b: int = omegaconf.MISSING, + ) -> Tuple[int, int, int]: + return (required_a, optional, required_b) + + assert dcargs.cli( + main2, args="--required-a 3 --optional 4 --required-b 5".split(" ") + ) == (3, 4, 5) + assert dcargs.cli(main2, args="--required-a 3 --required-b 5".split(" ")) == ( + 3, + 3, + 5, + ) + + with pytest.raises(SystemExit): + dcargs.cli(main2, args="--required-a 3 --optional 4") + with pytest.raises(SystemExit): + dcargs.cli(main2, args="--required-a 3") + + def test_attrs_basic(): @attr.s class ManyTypesA: From 4488b85803f7ec6a49ecaf3ff1a022f3c268811d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 10 Jul 2022 15:38:36 -0700 Subject: [PATCH 049/491] Features and example for base configs --- README.md | 575 ++++++++++++++++------ _update_readme.py | 103 ++-- dcargs/__init__.py | 2 + dcargs/_arguments.py | 4 +- dcargs/_fields.py | 100 ++-- dcargs/_instantiators.py | 70 ++- dcargs/_parsers.py | 8 +- dcargs/_resolver.py | 17 +- dcargs/_serialization.py | 31 +- examples/01_functions.py | 10 +- examples/02_dataclasses.py | 10 +- examples/03_enums_and_containers.py | 17 +- examples/04_flags.py | 11 +- examples/05_hierarchical_configs.py | 17 +- examples/06_literals.py | 9 +- examples/07_positional_args.py | 21 +- examples/08_standard_classes.py | 7 +- examples/09_subparsers.py | 10 +- examples/10_generics.py | 6 +- examples/11_dictionaries.py | 15 +- examples/12_named_tuples.py | 8 +- examples/13_base_configs.py | 77 +++ setup.py | 1 + tests/test_dict_namedtuple.py | 8 +- tests/test_errors.py | 12 +- tests/test_generics_and_serialization.py | 22 +- tests/test_helptext.py | 4 +- tests/test_missing.py | 73 +++ tests/test_nested.py | 63 +++ tests/test_unsupported_but_should_work.py | 8 +- 30 files changed, 980 insertions(+), 339 deletions(-) create mode 100644 examples/13_base_configs.py create mode 100644 tests/test_missing.py diff --git a/README.md b/README.md index e140e2137..d51dd9661 100644 --- a/README.md +++ b/README.md @@ -5,15 +5,11 @@ ![lint](https://github.com/brentyi/dcargs/workflows/lint/badge.svg) [![codecov](https://codecov.io/gh/brentyi/dcargs/branch/master/graph/badge.svg)](https://codecov.io/gh/brentyi/dcargs) - - - [Overview](#overview) - [Examples](#examples) - [Serialization](#serialization) - [Alternative tools](#alternative-tools) - - ## Overview ```bash @@ -25,8 +21,6 @@ pip install dcargs Our core interface generates an argument parser from a type-annotated callable _`f`_, which may be a function, class, or dataclass: ---- - ```python dcargs.cli( f: Callable[..., T], @@ -127,18 +121,19 @@ Ultimately, we aim to enable configuration interfaces that are: ## Examples -
+ +
1. Functions -
+
-[examples/01_functions.py](examples/01_functions.py) +In the simplest case, `dcargs.cli()` can be used to run a function with arguments +populated from the CLI. -```python -"""CLI generation example from a simple annotated function. `dcargs.cli()` will call -`main()`, with arguments populated from the CLI.""" +**Code ([link](examples/01_functions.py)):** +```python import dcargs @@ -161,10 +156,12 @@ if __name__ == "__main__": dcargs.cli(main) ``` ---- +
+ +**Example usage:** 
-$ python examples/01_functions.py --help
+$ python ./01_functions.py --help
 usage: 01_functions.py [-h] --field1 STR [--field2 INT] [--flag]
 
 Function, whose arguments will be populated from a CLI interface.
@@ -178,19 +175,29 @@ optional arguments:
   --flag        A boolean flag.
-
+
+$ python ./01_functions.py --field1 hello
+hello 3 False
+
+ +
+$ python ./01_functions.py --field1 hello --flag
+hello 3 True
+
+
+
2. Dataclasses -
+
-[examples/02_dataclasses.py](examples/02_dataclasses.py) +Common pattern: use `dcargs.cli()` to instantiate a dataclass. -```python -"""Example using dcargs.cli() to instantiate a dataclass.""" +**Code ([link](examples/02_dataclasses.py)):** +```python import dataclasses import dcargs @@ -209,14 +216,14 @@ class Args: if __name__ == "__main__": args = dcargs.cli(Args) print(args) - print() - print(dcargs.to_yaml(args)) ``` ---- +
+ +**Example usage:** 
-$ python examples/02_dataclasses.py --help
+$ python ./02_dataclasses.py --help
 usage: 02_dataclasses.py [-h] --field1 STR [--field2 INT] [--flag]
 
 Description.
@@ -231,22 +238,31 @@ optional arguments:
   --flag        A boolean flag.
-
+
+$ python ./02_dataclasses.py --field1 hello
+Args(field1='hello', field2=3, flag=False)
+
+ +
+$ python ./02_dataclasses.py --field1 hello --flag
+Args(field1='hello', field2=3, flag=True)
+
+
+
3. Enums And Containers -
+
-[examples/03_enums_and_containers.py](examples/03_enums_and_containers.py) - -```python -"""Examples of more advanced type annotations: enums and containers types. +We can generate argument parsers from more advanced type annotations, like enums and +tuple types. For collections, we only showcase `Tuple` here, but `List`, `Sequence`, +`Set`, `Dict`, etc are all supported as well. -For collections, we only showcase Tuple here, but List, Sequence, Set, etc are all -supported as well.""" +**Code ([link](examples/03_enums_and_containers.py)):** +```python import dataclasses import enum import pathlib @@ -267,15 +283,15 @@ class TrainConfig: """Paths to load training data from. This can be multiple!""" # Fixed-length tuples are also okay: - image_dimensions: Tuple[int, int] + image_dimensions: Tuple[int, int] = (32, 32) """Height and width of some image data.""" # Enums are handled seamlessly. - optimizer_type: OptimizerType + optimizer_type: OptimizerType = OptimizerType.ADAM """Gradient-based optimizer to use.""" # We can also explicitly mark arguments as optional. - checkpoint_interval: Optional[int] + checkpoint_interval: Optional[int] = None """Interval to save checkpoints at.""" @@ -284,41 +300,55 @@ if __name__ == "__main__": print(config) ``` ---- +
+ +**Example usage:** 
-$ python examples/03_enums_and_containers.py --help
+$ python ./03_enums_and_containers.py --help
 usage: 03_enums_and_containers.py [-h] --dataset-sources PATH [PATH ...]
-                                  --image-dimensions INT INT --optimizer-type
-                                  {ADAM,SGD} [--checkpoint-interval INT]
+                                  [--image-dimensions INT INT]
+                                  [--optimizer-type {ADAM,SGD}]
+                                  [--checkpoint-interval INT]
 
 required arguments:
   --dataset-sources PATH [PATH ...]
                         Paths to load training data from. This can be multiple!
-  --image-dimensions INT INT
-                        Height and width of some image data.
-  --optimizer-type {ADAM,SGD}
-                        Gradient-based optimizer to use.
 
 optional arguments:
   -h, --help            show this help message and exit
+  --image-dimensions INT INT
+                        Height and width of some image data. (default: 32 32)
+  --optimizer-type {ADAM,SGD}
+                        Gradient-based optimizer to use. (default: ADAM)
   --checkpoint-interval INT
                         Interval to save checkpoints at. (default: None)
-
+
+$ python ./03_enums_and_containers.py --dataset-sources ./data --image-dimensions 16 16
+TrainConfig(dataset_sources=(PosixPath('data'),), image_dimensions=(16, 16), optimizer_type=<OptimizerType.ADAM: 1>, checkpoint_interval=None)
+
+ +
+$ python ./03_enums_and_containers.py --dataset-sources ./data --optimizer-type SGD
+TrainConfig(dataset_sources=(PosixPath('data'),), image_dimensions=(32, 32), optimizer_type=<OptimizerType.SGD: 2>, checkpoint_interval=None)
+
+
+
4. Flags -
+
-[examples/04_flags.py](examples/04_flags.py) +Booleans can either be expected to be explicitly passed in, or, if given a default +value, automatically converted to flags. -```python -"""Example of how booleans are handled and automatically converted to flags.""" +**Code ([link](examples/04_flags.py)):** +```python import dataclasses from typing import Optional @@ -343,43 +373,41 @@ class Args: if __name__ == "__main__": args = dcargs.cli(Args) print(args) - print() - print(dcargs.to_yaml(args)) ``` ---- +
+ +**Example usage:** 
-$ python examples/04_flags.py --help
-usage: 04_flags.py [-h] --boolean {True,False}
-                   [--optional-boolean {True,False}] [--flag-a] [--no-flag-b]
+$ python ./04_flags.py --boolean True
+Args(boolean=True, optional_boolean=None, flag_a=False, flag_b=True)
+
-required arguments: - --boolean {True,False} - Boolean. This expects an explicit "True" or "False". +
+$ python ./04_flags.py --boolean False --flag-a
+Args(boolean=False, optional_boolean=None, flag_a=True, flag_b=True)
+
-optional arguments: - -h, --help show this help message and exit - --optional-boolean {True,False} - Optional boolean. Same as above, but can be omitted. (default: None) - --flag-a Pass --flag-a in to set this value to True. - --no-flag-b Pass --no-flag-b in to set this value to False. +
+$ python ./04_flags.py --boolean False --no-flag-b
+Args(boolean=False, optional_boolean=None, flag_a=False, flag_b=False)
-
+
5. Hierarchical Configs -
+
-[examples/05_hierarchical_configs.py](examples/05_hierarchical_configs.py) +Parsing of nested types (in this case nested dataclasses) enables hierarchical +configuration objects that are both modular and highly expressive. -```python -"""An example of how we can create hierarchical configuration interfaces by nesting -dataclasses.""" +**Code ([link](examples/05_hierarchical_configs.py)):** +```python import dataclasses import enum import pathlib @@ -435,22 +463,21 @@ def train( restore_checkpoint: Set to restore an existing checkpoint. checkpoint_interval: Training steps between each checkpoint save. """ - print(out_dir) - print("---") + print(f"{out_dir=}, {restore_checkpoint=}, {checkpoint_interval=}") + print(f"{config=}") print(dcargs.to_yaml(config)) - print("---") - print(restore_checkpoint) - print(checkpoint_interval) if __name__ == "__main__": dcargs.cli(train) ``` ---- +
+ +**Example usage:** 
-$ python examples/05_hierarchical_configs.py --help
+$ python ./05_hierarchical_configs.py --help
 usage: 05_hierarchical_configs.py [-h]
                                   [--config.optimizer.algorithm {ADAM,SGD}]
                                   [--config.optimizer.learning-rate FLOAT]
@@ -493,19 +520,49 @@ optional config arguments:
                         reproducible! (default: 0)
-
+
+$ python ./05_hierarchical_configs.py . --config.optimizer.algorithm SGD
+out_dir=PosixPath('.'), restore_checkpoint=False, checkpoint_interval=1000
+config=ExperimentConfig(optimizer=OptimizerConfig(algorithm=<OptimizerType.SGD: 2>, learning_rate=0.0003, weight_decay=0.01), batch_size=32, train_steps=100000, seed=0)
+# dcargs YAML.
+!dataclass:ExperimentConfig
+batch_size: 32
+optimizer: !dataclass:OptimizerConfig
+  algorithm: !enum:OptimizerType 'SGD'
+  learning_rate: 0.0003
+  weight_decay: 0.01
+seed: 0
+train_steps: 100000
+
+ +
+$ python ./05_hierarchical_configs.py . --restore-checkpoint
+out_dir=PosixPath('.'), restore_checkpoint=True, checkpoint_interval=1000
+config=ExperimentConfig(optimizer=OptimizerConfig(algorithm=<OptimizerType.ADAM: 1>, learning_rate=0.0003, weight_decay=0.01), batch_size=32, train_steps=100000, seed=0)
+# dcargs YAML.
+!dataclass:ExperimentConfig
+batch_size: 32
+optimizer: !dataclass:OptimizerConfig
+  algorithm: !enum:OptimizerType 'ADAM'
+  learning_rate: 0.0003
+  weight_decay: 0.01
+seed: 0
+train_steps: 100000
+
+
+
6. Literals -
+
-[examples/06_literals.py](examples/06_literals.py) +`typing.Literal[]` can be used to restrict inputs to a fixed set of choices. -```python -"""typing.Literal[] can be used to specify accepted input choices.""" +**Code ([link](examples/06_literals.py)):** +```python import dataclasses import enum from typing import Literal @@ -535,14 +592,14 @@ class Args: if __name__ == "__main__": args = dcargs.cli(Args) print(args) - print() - print(dcargs.to_yaml(args)) ``` ---- +
+ +**Example usage:** 
-$ python examples/06_literals.py --help
+$ python ./06_literals.py --help
 usage: 06_literals.py [-h] --enum {RED,GREEN,BLUE} --restricted-enum
                       {RED,GREEN} --integer {0,1,2,3} --string {red,green}
                       [--restricted-enum-with-default {RED,GREEN}]
@@ -565,19 +622,24 @@ optional arguments:
                         (default: red)
-
+
+$ python ./06_literals.py --enum RED --restricted-enum GREEN --integer 3 --string green
+Args(enum=<Color.RED: 1>, restricted_enum=<Color.GREEN: 2>, integer=3, string='green', restricted_enum_with_default=<Color.GREEN: 2>, integer_with_default=3, string_with_Default='red')
+
+
+
7. Positional Args -
+
-[examples/07_positional_args.py](examples/07_positional_args.py) +Positional-only arguments in functions are converted to positional CLI arguments. -```python -"""Positional-only arguments in functions are converted to positional CLI arguments.""" +**Code ([link](examples/07_positional_args.py)):** +```python from __future__ import annotations import dataclasses @@ -608,19 +670,7 @@ def main( verbose: Explain what is being done. background_rgb: Background color. Red by default. """ - print( - f"{source.absolute()=}" - "\n" - f"{dest.absolute()=}" - "\n" - f"{optimizer=}" - "\n" - f"{force=}" - "\n" - f"{verbose=}" - "\n" - f"{background_rgb=}" - ) + print(f"{source=}\n{dest=}\n{optimizer=}\n{force=}\n{verbose=}\n{background_rgb=}") class OptimizerType(enum.Enum): @@ -644,10 +694,12 @@ if __name__ == "__main__": dcargs.cli(main) ``` ---- +
+ +**Example usage:** 
-$ python examples/07_positional_args.py --help
+$ python ./07_positional_args.py --help
 usage: 07_positional_args.py [-h] [--optimizer.algorithm {ADAM,SGD}]
                              [--optimizer.learning-rate FLOAT]
                              [--optimizer.weight-decay FLOAT] [--force]
@@ -679,20 +731,30 @@ optional optimizer arguments:
                         Coefficient for L2 regularization. (default: 0.01)
-
+
+$ python ./07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5
+source=PosixPath('a')
+dest=PosixPath('b')
+optimizer=OptimizerConfig(algorithm=<OptimizerType.ADAM: 1>, learning_rate=1e-05, weight_decay=0.01)
+force=False
+verbose=False
+background_rgb=(1.0, 0.0, 0.0)
+
+
+
8. Standard Classes -
+
-[examples/08_standard_classes.py](examples/08_standard_classes.py) +In addition to functions and dataclasses, we can also generate CLIs from (the +constructors of) standard Python classes. -```python -"""In addition to functions and dataclasses, we can also generate CLIs from (the -constructors of) standard Python classes.""" +**Code ([link](examples/08_standard_classes.py)):** +```python import dcargs @@ -718,10 +780,12 @@ if __name__ == "__main__": print(args.data) ``` ---- +
+ +**Example usage:** 
-$ python examples/08_standard_classes.py --help
+$ python ./08_standard_classes.py --help
 usage: 08_standard_classes.py [-h] --field1 STR --field2 INT [--flag]
 
 Arguments.
@@ -735,19 +799,24 @@ optional arguments:
   --flag        A boolean flag.
-
+
+$ python ./08_standard_classes.py --field1 hello --field2 7
+['hello', 7, False]
+
+
+
9. Subparsers -
+
-[examples/09_subparsers.py](examples/09_subparsers.py) +Unions over nested types (classes or dataclasses) are populated using subparsers. -```python -"""Unions over nested types (classes or dataclasses) will result in subparsers.""" +**Code ([link](examples/09_subparsers.py)):** +```python from __future__ import annotations import dataclasses @@ -779,10 +848,12 @@ if __name__ == "__main__": dcargs.cli(main) ``` ---- +
+ +**Example usage:** 
-$ python examples/09_subparsers.py --help
+$ python ./09_subparsers.py --help
 usage: 09_subparsers.py [-h] {checkout,commit} ...
 
 optional arguments:
@@ -792,19 +863,56 @@ subcommands:
   {checkout,commit}
-
+
+$ python ./09_subparsers.py commit --help
+usage: 09_subparsers.py commit [-h] --message STR [--all]
+
+Commit changes.
+
+required arguments:
+  --message STR
+
+optional arguments:
+  -h, --help     show this help message and exit
+  --all
+
+ +
+$ python ./09_subparsers.py commit --message hello --all
+Commit(message='hello', all=True)
+
+ +
+$ python ./09_subparsers.py checkout --help
+usage: 09_subparsers.py checkout [-h] --branch STR
+
+Checkout a branch.
+
+required arguments:
+  --branch STR
+
+optional arguments:
+  -h, --help    show this help message and exit
+
+ +
+$ python ./09_subparsers.py checkout --branch main
+Checkout(branch='main')
+
+
+
10. Generics -
+
-[examples/10_generics.py](examples/10_generics.py) +Example of parsing for generic dataclasses. -```python -"""Example of parsing for generic (~templated) dataclasses.""" +**Code ([link](examples/10_generics.py)):** +```python import dataclasses from typing import Generic, TypeVar @@ -841,10 +949,12 @@ if __name__ == "__main__": print(args) ``` ---- +
+ +**Example usage:** 
-$ python examples/10_generics.py --help
+$ python ./10_generics.py --help
 usage: 10_generics.py [-h] --point-continuous.x FLOAT --point-continuous.y
                       FLOAT --point-continuous.z FLOAT
                       --point-continuous.frame-id STR --point-discrete.x INT
@@ -895,23 +1005,23 @@ required shape.c arguments:
   --shape.c.frame-id STR
-
+
11. Dictionaries -
+
-[examples/11_dictionaries.py](examples/11_dictionaries.py) +Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation, +or a `TypedDict` type. -```python -"""Dictionary inputs can be specified using either a standard Dict[T1, T2] annotation, -or a TypedDict type. +Note that setting `total=False` for `TypedDict` is currently not (but reasonably could be) +supported. -Note that setting total=False for TypedDicts is currently not (but reasonably could be) -supported.""" +**Code ([link](examples/11_dictionaries.py)):** +```python from typing import Dict, TypedDict import dcargs @@ -924,7 +1034,7 @@ class DictionarySchema(TypedDict): def main( - standard_dict: Dict[int, bool], + standard_dict: Dict[str, bool], typed_dict: DictionarySchema = { "field1": "hey", "field2": 3, @@ -941,16 +1051,18 @@ if __name__ == "__main__": dcargs.cli(main) ``` ---- +
+ +**Example usage:** 
-$ python examples/11_dictionaries.py --help
-usage: 11_dictionaries.py [-h] --standard-dict INT {True,False}
-                          [INT {True,False} ...] [--typed-dict.field1 STR]
+$ python ./11_dictionaries.py --help
+usage: 11_dictionaries.py [-h] --standard-dict STR {True,False}
+                          [STR {True,False} ...] [--typed-dict.field1 STR]
                           [--typed-dict.field2 INT] [--typed-dict.field3]
 
 required arguments:
-  --standard-dict INT {True,False} [INT {True,False} ...]
+  --standard-dict STR {True,False} [STR {True,False} ...]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -964,19 +1076,25 @@ optional typed_dict arguments:
   --typed-dict.field3   A boolean field.
-
+
+$ python ./11_dictionaries.py --standard-dict key1 True key2 False
+Standard dict: {'key1': True, 'key2': False}
+Typed dict: {'field1': 'hey', 'field2': 3, 'field3': False}
+
+
+
12. Named Tuples -
+
-[examples/12_named_tuples.py](examples/12_named_tuples.py) +Example using `dcargs.cli()` to instantiate a named tuple. -```python -"""Example using dcargs.cli() to instantiate a named tuple.""" +**Code ([link](examples/12_named_tuples.py)):** +```python from typing import NamedTuple import dcargs @@ -992,18 +1110,17 @@ class TupleType(NamedTuple): if __name__ == "__main__": - print(TupleType.__doc__) x = dcargs.cli(TupleType) assert isinstance(x, tuple) print(x) ``` ---- +
+ +**Example usage:** 
-$ python examples/12_named_tuples.py --help
-Description.
-    This should show up in the helptext!
+$ python ./12_named_tuples.py --help
 usage: 12_named_tuples.py [-h] --field1 STR [--field2 INT] [--flag]
 
 Description.
@@ -1018,7 +1135,163 @@ optional arguments:
   --flag        A boolean flag.
-
+
+$ python ./12_named_tuples.py --field1 hello
+TupleType(field1='hello', field2=3, flag=False)
+
+ +
+ +
+ +13. Base Configs + +
+ +Example of a common configuration pattern: selecting one of multiple possible base +configurations, and then using the CLI to either override existing values or fill in +missing ones. + +`BASE_CONFIG=small python ./13_base_configs.py --help` +`BASE_CONFIG=small python ./13_base_configs.py --seed 94720` +`BASE_CONFIG=big python ./13_base_configs.py --help` +`BASE_CONFIG=big python ./13_base_configs.py --seed 94720` + +**Code ([link](examples/13_base_configs.py)):** + +```python +import dataclasses +import os +import sys +from typing import Literal + +import dcargs + + +@dataclasses.dataclass(frozen=True) +class ExperimentConfig: + # Dataset to run experiment on. + dataset: Literal["mnist", "imagenet-50"] + + # Model size. + num_layers: int + units: int + + # Batch size. + batch_size: int + + # Total number of training steps. + train_steps: int + + # Random seed. This is helpful for making sure that our experiments are all + # reproducible! + seed: int + + +# Note that we could also define this library using separate YAML files (a la +# `config_path`/`config_name` in Hydra), but staying in Python enables seamless type +# checking + IDE support. +base_config_library = { + "small": ExperimentConfig( + dataset="mnist", + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + # The dcargs.MISSING sentinel allows us to specify that the seed should have no + # default, and needs to be populated from the CLI. + seed=dcargs.MISSING, + ), + "big": ExperimentConfig( + dataset="imagenet-50", + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=dcargs.MISSING, + ), +} + +if __name__ == "__main__": + # Get base configuration name from environment. + # base_config_name = os.environ.get("BASE_CONFIG") + # if base_config_name is None or base_config_name not in base_config_library: + # raise SystemExit( + # f"BASE_CONFIG should be set to one of {tuple(base_config_library.keys())}" + # ) + if ( + len(sys.argv) < 2 + or (base_config_name := sys.argv[1]) not in base_config_library + ): + raise SystemExit( + f"BASE_CONFIG should be set to one of {tuple(base_config_library.keys())}" + ) + + # Get base configuration from our library, and use it for default CLI parameters. + base_config = base_config_library[base_config_name] + config = dcargs.cli( + ExperimentConfig, + default_instance=base_config, + args=sys.argv[2:], + ) + + print(config) +``` + +
+ +**Example usage:** + +
+$ python ./13_base_configs.py small --help
+usage: 13_base_configs.py [-h] [--dataset {mnist,imagenet-50}]
+                          [--num-layers INT] [--units INT] [--batch-size INT]
+                          [--train-steps INT] --seed INT
+
+required arguments:
+  --seed INT            Random seed. This is helpful for making sure that our experiments are all
+                        reproducible!
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --dataset {mnist,imagenet-50}
+                        Dataset to run experiment on. (default: mnist)
+  --num-layers INT      Model size. (default: 4)
+  --units INT           Model size. (default: 64)
+  --batch-size INT      Batch size. (default: 2048)
+  --train-steps INT     Total number of training steps. (default: 30000)
+
+ +
+$ python ./13_base_configs.py small --seed 94720
+ExperimentConfig(dataset='mnist', num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720)
+
+ +
+$ python ./13_base_configs.py big --help
+usage: 13_base_configs.py [-h] [--dataset {mnist,imagenet-50}]
+                          [--num-layers INT] [--units INT] [--batch-size INT]
+                          [--train-steps INT] --seed INT
+
+required arguments:
+  --seed INT            Random seed. This is helpful for making sure that our experiments are all
+                        reproducible!
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --dataset {mnist,imagenet-50}
+                        Dataset to run experiment on. (default: imagenet-50)
+  --num-layers INT      Model size. (default: 8)
+  --units INT           Model size. (default: 256)
+  --batch-size INT      Batch size. (default: 32)
+  --train-steps INT     Total number of training steps. (default: 100000)
+
+ +
+$ python ./13_base_configs.py big --seed 94720
+ExperimentConfig(dataset='imagenet-50', num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720)
+
+
## Serialization diff --git a/_update_readme.py b/_update_readme.py index f37639336..1fef17e0f 100644 --- a/_update_readme.py +++ b/_update_readme.py @@ -1,11 +1,13 @@ """Helper script for updating the auto-generated parts of the README. (docstring, examples list)""" - import concurrent.futures import dataclasses +import html import inspect +import os import pathlib import re +import shlex import subprocess import dcargs @@ -44,36 +46,75 @@ def format_script_for_readme(path: pathlib.Path) -> str: title = title.replace("_", " ").title() source = path.read_text().strip() - helptext = subprocess.run( - args=["python", str(path), "--help"], stdout=subprocess.PIPE, encoding="utf8" - ).stdout - helptext = re.sub( # Strip colorcodes. - r"\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))", "", helptext - ).strip() - - return f""" -
- -{index}. {title} - -
- -[{path}]({path}) - -```python -{source} -``` - ---- - -
-$ python {path} --help
-{helptext}
-
- -
-
- """.strip() + example_output_lines = [] + + docstring = source.split('"""')[1].strip() + assert "Usage:" in docstring + description_text, _, usage_text = docstring.partition("Usage:") + example_usages = map( + lambda x: x[1:-1], + filter( + lambda line: line.startswith("`") and line.endswith("`"), + usage_text.split("\n"), + ), + ) + for usage in example_usages: + # Example usage: `ENV=something python ./path --help` + args = shlex.split(usage) + python_index = args.index("python") + + env_vars = {} + if python_index > 0: + env_vars = { + k: v for (k, v) in map(lambda x: x.split("="), args[:python_index]) + } + output = subprocess.run( + args=["python", str(path)] + args[python_index + 2 :], + stdout=subprocess.PIPE, + encoding="utf8", + env=dict(os.environ, **env_vars), + ).stdout + output = re.sub( # Strip colorcodes. + r"\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))", "", output + ).strip() + example_output_lines.extend( + [ + "", + "
",
+                f"$ {usage}",
+                f"{html.escape(output)}",
+                "
", + ] + ) + + return "\n".join( + [ + "", + "
", + "", + f"{index}. {title}", + "", + "
", + "", + description_text, + "", + f"**Code ([link]({path})):**", + "", + "```python", + source[3:].partition('"""')[2].strip(), + "```", + "", + "
", + "", + "**Example usage:**", + "", + ] + + example_output_lines + + [ + "", + "
", + ] + ) def get_examples_str(examples_dir: pathlib.Path, constants: Constants) -> str: diff --git a/dcargs/__init__.py b/dcargs/__init__.py index a4e1a4656..c2ba5e4c4 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,8 +1,10 @@ from ._cli import cli, parse +from ._fields import MISSING from ._instantiators import UnsupportedTypeAnnotationError from ._serialization import from_yaml, to_yaml __all__ = [ + "MISSING", "cli", # Deprecated. # "parse", diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 3a3120350..12ccd5bb0 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -92,8 +92,8 @@ def _rule_handle_defaults( """Set `required=True` if a default value is set.""" # Mark lowered as required if a default is set. - if arg.field.default is None: - return dataclasses.replace(lowered, required=True) + if arg.field.default is None or arg.field.default is _fields.MISSING: + return dataclasses.replace(lowered, default=None, required=True) return dataclasses.replace(lowered, default=arg.field.default) diff --git a/dcargs/_fields.py b/dcargs/_fields.py index caa9e3100..d7c1d180e 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -20,6 +20,23 @@ class Field: positional: bool +class _MISSING_TYPE: + pass + + +MISSING: Any = _MISSING_TYPE() +"""Sentinel value to mark fields as missing. Should generally only be used to mark +fields passed in as a `default_instance` for `dcargs.cli()` as required.""" + +_missing_types = [dataclasses.MISSING, MISSING, inspect.Parameter.empty] +try: + # Undocumented feature: support omegaconf dataclasses out of the box. + import omegaconf + + _missing_types.append(omegaconf.MISSING) +except ImportError: + pass + T = TypeVar("T") @@ -46,15 +63,18 @@ def field_list_from_callable( if cls is not None and is_typeddict(cls): # Handle typed dictionaries. field_list = [] - assert default_instance is None or isinstance(default_instance, dict) + no_default_instance = ( + default_instance is None or default_instance in _missing_types + ) + assert no_default_instance or isinstance(default_instance, dict) for name, typ in get_type_hints(cls).items(): field_list.append( Field( name=name, typ=typ, - default=None - if default_instance is None - else default_instance.get(name, None), + default=MISSING + if no_default_instance + else default_instance.get(name, MISSING), # type: ignore helptext=_docstrings.get_field_docstring(cls, name), positional=False, ) @@ -73,8 +93,10 @@ def field_list_from_callable( for name, typ in _resolver.get_type_hints(cls).items(): # Get default, with priority for `default_instance`. default = field_defaults.get(name) - if default_instance is not None and hasattr(default_instance, name): + if hasattr(default_instance, name): default = getattr(default_instance, name) + if default in _missing_types or default_instance in _missing_types: + default = MISSING field_list.append( Field( @@ -92,11 +114,12 @@ def field_list_from_callable( for dc_field in filter( lambda field: field.init, _resolver.resolved_fields(cls) ): + default = _get_dataclass_field_default(dc_field, default_instance) field_list.append( Field( name=dc_field.name, typ=dc_field.type, - default=_get_dataclass_field_default(dc_field, default_instance), + default=default, helptext=_docstrings.get_field_docstring(cls, dc_field.name), positional=False, ) @@ -128,7 +151,7 @@ def field_list_from_callable( # Get default value. default = param.default - if default is inspect.Parameter.empty: + if default in _missing_types: default = None # Get helptext from docstring. @@ -164,29 +187,37 @@ def _ensure_dataclass_instance_used_as_default_is_frozen( ) -_missing_types = [dataclasses.MISSING] -try: - import omegaconf - - _missing_types.append(omegaconf.MISSING) -except ImportError: - pass - - def _get_dataclass_field_default( field: dataclasses.Field, parent_default_instance: Any ) -> Optional[Any]: """Helper for getting the default instance for a field.""" - field_default_instance = None - if field.default not in _missing_types: - # Populate default from usual default value, or - # `dataclasses.field(default=...)`. - field_default_instance = field.default - if dataclasses.is_dataclass(field_default_instance): - _ensure_dataclass_instance_used_as_default_is_frozen( - field, field_default_instance + # If the dataclass's parent is explicitly marked MISSING, mark this field as missing + # as well. + if parent_default_instance in _missing_types: + return MISSING + + # Try grabbing default from parent instance. + if parent_default_instance is not None: + # Populate default from some parent, eg `default_instance` in `dcargs.cli()`. + if hasattr(parent_default_instance, field.name): + return getattr(parent_default_instance, field.name) + else: + warnings.warn( + f"Could not find field {field.name} in default instance" + f" {parent_default_instance}, which has" + f" type {type(parent_default_instance)},", + stacklevel=2, ) - elif field.default_factory not in _missing_types and not ( + + # Try grabbing default from dataclass field. + if field.default not in _missing_types: + default = field.default + if dataclasses.is_dataclass(default): + _ensure_dataclass_instance_used_as_default_is_frozen(field, default) + return default + + # Populate default from `dataclasses.field(default_factory=...)`. + if field.default_factory is not dataclasses.MISSING and not ( # Special case to ignore default_factory if we write: # `field: Dataclass = dataclasses.field(default_factory=Dataclass)`. # @@ -199,19 +230,8 @@ def _get_dataclass_field_default( dataclasses.is_dataclass(field.type) and field.default_factory is field.type ): - # Populate default from `dataclasses.field(default_factory=...)`. - assert callable(field.default_factory) - field_default_instance = field.default_factory() + return field.default_factory() - if parent_default_instance is not None: - # Populate default from some parent, eg `default_instance` in `dcargs.cli()`. - if hasattr(parent_default_instance, field.name): - field_default_instance = getattr(parent_default_instance, field.name) - else: - warnings.warn( - f"Could not find field {field.name} in default instance" - f" {parent_default_instance}, which has" - f" type {type(parent_default_instance)},", - stacklevel=2, - ) - return field_default_instance + # Otherwise, no default. This is different from MISSING, because MISSING propagates + # to children. We could revisit this design to make it clearer. + return None diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index 496cf0040..ee47ed7c4 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -33,12 +33,11 @@ ``` """ -import collections +import collections.abc import dataclasses import enum import inspect import warnings -from collections.abc import Mapping from typing import ( Any, Callable, @@ -50,22 +49,24 @@ Type, TypeVar, Union, + cast, + overload, ) from typing_extensions import Annotated, Final, Literal, get_args, get_origin from . import _strings -Instantiator = Union[ - # Most standard fields: these are converted from strings from the CLI. - Callable[[str], Any], - # Sequence fields! This should be used whenever argparse's `nargs` field is set. - Callable[[List[str]], Any], - # Special case: the only time that argparse doesn't give us a string is when the - # argument action is set to `store_true` or `store_false`. In this case, we get - # a bool directly, and the field action can be a no-op. - Callable[[bool], bool], -] +# Most standard fields: these are converted from strings from the CLI. +_StandardInstantiator = Callable[[str], Any] +# Sequence fields! This should be used whenever argparse's `nargs` field is set. +_SequenceInstantiator = Callable[[List[str]], Any] +# Special case: the only time that argparse doesn't give us a string is when the +# argument action is set to `store_true` or `store_false`. In this case, we get +# a bool directly, and the field action can be a no-op. +_FlagInstantiator = Callable[[bool], bool] + +Instantiator = Union[_StandardInstantiator, _SequenceInstantiator, _FlagInstantiator] @dataclasses.dataclass @@ -241,17 +242,18 @@ def _instantiator_from_container_type( ) else: - instantiators, metas = zip( - *map( - lambda t: _instantiator_from_type_inner( - t, - type_from_typevar, - allow_sequences=False, - allow_optional=False, - ), - types, + instantiators = [] + metas = [] + for t in types: + a, b = _instantiator_from_type_inner( + t, + type_from_typevar, + allow_sequences=False, + allow_optional=False, ) - ) + instantiators.append(a) + metas.append(b) + if len(set(m.choices for m in metas)) > 1: raise UnsupportedTypeAnnotationError( "Due to constraints in argparse, all choices in fixed-length tuples" @@ -262,7 +264,7 @@ def _instantiator_from_container_type( make(x) for make, x in zip(instantiators, strings) ), InstantiatorMetadata( nargs=len(types), - metavar=tuple(m.metavar for m in metas), + metavar=tuple(cast(str, m.metavar) for m in metas), choices=metas[0].choices, is_optional=False, ) @@ -313,7 +315,7 @@ def _instantiator_from_container_type( ) # Dictionaries. - if type_origin in (dict, Mapping): + if type_origin in (dict, collections.abc.Mapping): key_type, val_type = get_args(typ) key_instantiator, key_metadata = _instantiator_from_type_inner( key_type, @@ -358,6 +360,26 @@ def dict_instantiator(strings: List[str]) -> Any: ) +@overload +def _instantiator_from_type_inner( + typ: Type, + type_from_typevar: Dict[TypeVar, Type], + allow_sequences: Literal[False], + allow_optional: bool, +) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: + ... + + +@overload +def _instantiator_from_type_inner( + typ: Type, + type_from_typevar: Dict[TypeVar, Type], + allow_sequences: Literal[True], + allow_optional: bool, +) -> Tuple[Instantiator, InstantiatorMetadata]: + ... + + def _instantiator_from_type_inner( typ: Type, type_from_typevar: Dict[TypeVar, Type], diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 72a303894..35df346ce 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -50,9 +50,7 @@ def _is_possibly_nested_type(typ: Any) -> bool: Examples of when we return False: int, str, List[int], List[str], pathlib.Path, etc. """ - origin = get_origin(typ) - if origin is not None: - typ = origin + typ = _resolver.unwrap_origin(typ) # Nested types should be callable. if not callable(typ): @@ -330,7 +328,9 @@ def from_field( description=None, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, - default_instance=None, + default_instance=field.default + if isinstance(field.default, _resolver.unwrap_origin(option)) + else None, ) return SubparsersSpecification( diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index e68cfe6a7..568b93cde 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -6,14 +6,21 @@ from typing_extensions import get_args, get_origin, get_type_hints +TypeOrCallable = TypeVar("TypeOrCallable", Type, Callable) -def is_dataclass(cls: Union[Type, Callable]) -> bool: - """Same as `dataclasses.is_dataclass`, but also handles generic aliases.""" - origin_cls = get_origin(cls) - return dataclasses.is_dataclass(cls if origin_cls is None else origin_cls) +def unwrap_origin(tp: TypeOrCallable) -> TypeOrCallable: + """Returns the origin of tp if it exists. Otherwise, returns tp.""" + origin = get_origin(tp) + if origin is None: + return tp + else: + return origin -TypeOrCallable = TypeVar("TypeOrCallable", Type, Callable) + +def is_dataclass(cls: Union[Type, Callable]) -> bool: + """Same as `dataclasses.is_dataclass`, but also handles generic aliases.""" + return dataclasses.is_dataclass(unwrap_origin(cls)) def resolve_generic_types( diff --git a/dcargs/_serialization.py b/dcargs/_serialization.py index 1559d94b4..860ac0557 100644 --- a/dcargs/_serialization.py +++ b/dcargs/_serialization.py @@ -1,17 +1,17 @@ """Type-safe, human-readable serialization helpers.""" import dataclasses -import datetime import enum from typing import IO, Any, Optional, Set, Type, TypeVar, Union import yaml from typing_extensions import get_origin -from . import _resolver +from . import _fields, _resolver ENUM_YAML_TAG_PREFIX = "!enum:" DATACLASS_YAML_TAG_PREFIX = "!dataclass:" +MISSING_YAML_TAG_PREFIX = "!missing" DataclassType = TypeVar("DataclassType") @@ -114,12 +114,18 @@ def make_enum_constructor(typ: Type): else: assert False + DataclassLoader.add_constructor( + tag=MISSING_YAML_TAG_PREFIX, + constructor=lambda *_unused: _fields.MISSING, + ) + return DataclassLoader def _make_dumper(instance: Any) -> Type[yaml.Dumper]: class DataclassDumper(yaml.Dumper): - pass + def ignore_aliases(self, data): + return super().ignore_aliases(data) or data is _fields.MISSING contained_types = list(_get_contained_special_types_from_instance(instance)) contained_type_names = list(map(lambda cls: cls.__name__, contained_types)) @@ -135,7 +141,7 @@ class DataclassDumper(yaml.Dumper): field: dataclasses.Field def make_representer(name: str): - def representer(dumper, data): + def representer(dumper: DataclassDumper, data: Any) -> yaml.Node: if dataclasses.is_dataclass(data): return dumper.represent_mapping( tag=DATACLASS_YAML_TAG_PREFIX + name, @@ -149,11 +155,19 @@ def representer(dumper, data): return dumper.represent_scalar( tag=ENUM_YAML_TAG_PREFIX + name, value=data.name ) + assert False return representer for typ, name in zip(contained_types, contained_type_names): DataclassDumper.add_representer(typ, make_representer(name)) + + DataclassDumper.add_representer( + type(_fields.MISSING), + lambda dumper, data: dumper.represent_scalar( + tag=MISSING_YAML_TAG_PREFIX, value="" + ), + ) return DataclassDumper @@ -169,14 +183,7 @@ def from_yaml( return out -def _timestamp() -> str: - """Get a current timestamp as a string. Example format: `2021-11-05-15:46:32`.""" - return datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - - def to_yaml(instance: Any) -> str: """Serialize a dataclass; returns a yaml-compatible string that can be deserialized via `dcargs.from_yaml()`.""" - return f"# YAML generated via dcargs, at {_timestamp()}.\n" + yaml.dump( - instance, Dumper=_make_dumper(instance) - ) + return "# dcargs YAML.\n" + yaml.dump(instance, Dumper=_make_dumper(instance)) diff --git a/examples/01_functions.py b/examples/01_functions.py index 6aa357b8e..819aedeba 100644 --- a/examples/01_functions.py +++ b/examples/01_functions.py @@ -1,5 +1,11 @@ -"""CLI generation example from a simple annotated function. `dcargs.cli()` will call -`main()`, with arguments populated from the CLI.""" +"""In the simplest case, `dcargs.cli()` can be used to run a function with arguments +populated from the CLI. + +Usage: +`python ./01_functions.py --help` +`python ./01_functions.py --field1 hello` +`python ./01_functions.py --field1 hello --flag` +""" import dcargs diff --git a/examples/02_dataclasses.py b/examples/02_dataclasses.py index 52477948a..64fc65e48 100644 --- a/examples/02_dataclasses.py +++ b/examples/02_dataclasses.py @@ -1,4 +1,10 @@ -"""Example using dcargs.cli() to instantiate a dataclass.""" +"""Common pattern: use `dcargs.cli()` to instantiate a dataclass. + +Usage: +`python ./02_dataclasses.py --help` +`python ./02_dataclasses.py --field1 hello` +`python ./02_dataclasses.py --field1 hello --flag` +""" import dataclasses @@ -18,5 +24,3 @@ class Args: if __name__ == "__main__": args = dcargs.cli(Args) print(args) - print() - print(dcargs.to_yaml(args)) diff --git a/examples/03_enums_and_containers.py b/examples/03_enums_and_containers.py index 3fb86abee..1bde39d09 100644 --- a/examples/03_enums_and_containers.py +++ b/examples/03_enums_and_containers.py @@ -1,7 +1,12 @@ -"""Examples of more advanced type annotations: enums and containers types. +"""We can generate argument parsers from more advanced type annotations, like enums and +tuple types. For collections, we only showcase `Tuple` here, but `List`, `Sequence`, +`Set`, `Dict`, etc are all supported as well. -For collections, we only showcase Tuple here, but List, Sequence, Set, etc are all -supported as well.""" +Usage: +`python ./03_enums_and_containers.py --help` +`python ./03_enums_and_containers.py --dataset-sources ./data --image-dimensions 16 16` +`python ./03_enums_and_containers.py --dataset-sources ./data --optimizer-type SGD` +""" import dataclasses import enum @@ -23,15 +28,15 @@ class TrainConfig: """Paths to load training data from. This can be multiple!""" # Fixed-length tuples are also okay: - image_dimensions: Tuple[int, int] + image_dimensions: Tuple[int, int] = (32, 32) """Height and width of some image data.""" # Enums are handled seamlessly. - optimizer_type: OptimizerType + optimizer_type: OptimizerType = OptimizerType.ADAM """Gradient-based optimizer to use.""" # We can also explicitly mark arguments as optional. - checkpoint_interval: Optional[int] + checkpoint_interval: Optional[int] = None """Interval to save checkpoints at.""" diff --git a/examples/04_flags.py b/examples/04_flags.py index d5fcd7834..40ff68cd6 100644 --- a/examples/04_flags.py +++ b/examples/04_flags.py @@ -1,4 +1,11 @@ -"""Example of how booleans are handled and automatically converted to flags.""" +"""Booleans can either be expected to be explicitly passed in, or, if given a default +value, automatically converted to flags. + +Usage: +`python ./04_flags.py --boolean True` +`python ./04_flags.py --boolean False --flag-a` +`python ./04_flags.py --boolean False --no-flag-b` +""" import dataclasses from typing import Optional @@ -24,5 +31,3 @@ class Args: if __name__ == "__main__": args = dcargs.cli(Args) print(args) - print() - print(dcargs.to_yaml(args)) diff --git a/examples/05_hierarchical_configs.py b/examples/05_hierarchical_configs.py index 4324cb79a..4aaf469d1 100644 --- a/examples/05_hierarchical_configs.py +++ b/examples/05_hierarchical_configs.py @@ -1,5 +1,11 @@ -"""An example of how we can create hierarchical configuration interfaces by nesting -dataclasses.""" +"""Parsing of nested types (in this case nested dataclasses) enables hierarchical +configuration objects that are both modular and highly expressive. + +Usage: +`python ./05_hierarchical_configs.py --help` +`python ./05_hierarchical_configs.py . --config.optimizer.algorithm SGD` +`python ./05_hierarchical_configs.py . --restore-checkpoint` +""" import dataclasses import enum @@ -56,12 +62,9 @@ def train( restore_checkpoint: Set to restore an existing checkpoint. checkpoint_interval: Training steps between each checkpoint save. """ - print(out_dir) - print("---") + print(f"{out_dir=}, {restore_checkpoint=}, {checkpoint_interval=}") + print(f"{config=}") print(dcargs.to_yaml(config)) - print("---") - print(restore_checkpoint) - print(checkpoint_interval) if __name__ == "__main__": diff --git a/examples/06_literals.py b/examples/06_literals.py index e72ac0605..c44a5a8c9 100644 --- a/examples/06_literals.py +++ b/examples/06_literals.py @@ -1,4 +1,9 @@ -"""typing.Literal[] can be used to specify accepted input choices.""" +"""`typing.Literal[]` can be used to restrict inputs to a fixed set of choices. + +Usage: +`python ./06_literals.py --help` +`python ./06_literals.py --enum RED --restricted-enum GREEN --integer 3 --string green` +""" import dataclasses import enum @@ -29,5 +34,3 @@ class Args: if __name__ == "__main__": args = dcargs.cli(Args) print(args) - print() - print(dcargs.to_yaml(args)) diff --git a/examples/07_positional_args.py b/examples/07_positional_args.py index e2a543f91..317eccaa1 100644 --- a/examples/07_positional_args.py +++ b/examples/07_positional_args.py @@ -1,4 +1,9 @@ -"""Positional-only arguments in functions are converted to positional CLI arguments.""" +"""Positional-only arguments in functions are converted to positional CLI arguments. + +Usage: +`python ./07_positional_args.py --help` +`python ./07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5` +""" from __future__ import annotations @@ -30,19 +35,7 @@ def main( verbose: Explain what is being done. background_rgb: Background color. Red by default. """ - print( - f"{source.absolute()=}" - "\n" - f"{dest.absolute()=}" - "\n" - f"{optimizer=}" - "\n" - f"{force=}" - "\n" - f"{verbose=}" - "\n" - f"{background_rgb=}" - ) + print(f"{source=}\n{dest=}\n{optimizer=}\n{force=}\n{verbose=}\n{background_rgb=}") class OptimizerType(enum.Enum): diff --git a/examples/08_standard_classes.py b/examples/08_standard_classes.py index f2ecd5c3f..0c4ce4e6b 100644 --- a/examples/08_standard_classes.py +++ b/examples/08_standard_classes.py @@ -1,5 +1,10 @@ """In addition to functions and dataclasses, we can also generate CLIs from (the -constructors of) standard Python classes.""" +constructors of) standard Python classes. + +Usage: +`python ./08_standard_classes.py --help` +`python ./08_standard_classes.py --field1 hello --field2 7` +""" import dcargs diff --git a/examples/09_subparsers.py b/examples/09_subparsers.py index ed71fb5c8..af300b33d 100644 --- a/examples/09_subparsers.py +++ b/examples/09_subparsers.py @@ -1,4 +1,12 @@ -"""Unions over nested types (classes or dataclasses) will result in subparsers.""" +"""Unions over nested types (classes or dataclasses) are populated using subparsers. + +Usage: +`python ./09_subparsers.py --help` +`python ./09_subparsers.py commit --help` +`python ./09_subparsers.py commit --message hello --all` +`python ./09_subparsers.py checkout --help` +`python ./09_subparsers.py checkout --branch main` +""" from __future__ import annotations diff --git a/examples/10_generics.py b/examples/10_generics.py index 31168fb9e..2175ad629 100644 --- a/examples/10_generics.py +++ b/examples/10_generics.py @@ -1,4 +1,8 @@ -"""Example of parsing for generic (~templated) dataclasses.""" +"""Example of parsing for generic dataclasses. + +Usage: +`python ./10_generics.py --help` +""" import dataclasses from typing import Generic, TypeVar diff --git a/examples/11_dictionaries.py b/examples/11_dictionaries.py index b20d478f8..529b785c7 100644 --- a/examples/11_dictionaries.py +++ b/examples/11_dictionaries.py @@ -1,8 +1,13 @@ -"""Dictionary inputs can be specified using either a standard Dict[T1, T2] annotation, -or a TypedDict type. +"""Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation, +or a `TypedDict` type. -Note that setting total=False for TypedDicts is currently not (but reasonably could be) -supported.""" +Note that setting `total=False` for `TypedDict` is currently not (but reasonably could be) +supported. + +Usage: +`python ./11_dictionaries.py --help` +`python ./11_dictionaries.py --standard-dict key1 True key2 False` +""" from typing import Dict, TypedDict @@ -16,7 +21,7 @@ class DictionarySchema(TypedDict): def main( - standard_dict: Dict[int, bool], + standard_dict: Dict[str, bool], typed_dict: DictionarySchema = { "field1": "hey", "field2": 3, diff --git a/examples/12_named_tuples.py b/examples/12_named_tuples.py index bc11bbff4..6e2f2c3dc 100644 --- a/examples/12_named_tuples.py +++ b/examples/12_named_tuples.py @@ -1,4 +1,9 @@ -"""Example using dcargs.cli() to instantiate a named tuple.""" +"""Example using `dcargs.cli()` to instantiate a named tuple. + +Usage: +`python ./12_named_tuples.py --help` +`python ./12_named_tuples.py --field1 hello` +""" from typing import NamedTuple @@ -15,7 +20,6 @@ class TupleType(NamedTuple): if __name__ == "__main__": - print(TupleType.__doc__) x = dcargs.cli(TupleType) assert isinstance(x, tuple) print(x) diff --git a/examples/13_base_configs.py b/examples/13_base_configs.py new file mode 100644 index 000000000..9a04520b8 --- /dev/null +++ b/examples/13_base_configs.py @@ -0,0 +1,77 @@ +"""Example of a common configuration pattern: selecting one of multiple possible base +configurations, and then using the CLI to either override existing values or fill in +missing ones. + +Usage: +`BASE_CONFIG=small python ./13_base_configs.py --help` +`BASE_CONFIG=small python ./13_base_configs.py --seed 94720` +`BASE_CONFIG=big python ./13_base_configs.py --help` +`BASE_CONFIG=big python ./13_base_configs.py --seed 94720` +""" + +import dataclasses +import os +from typing import Literal + +import dcargs + + +@dataclasses.dataclass(frozen=True) +class ExperimentConfig: + # Dataset to run experiment on. + dataset: Literal["mnist", "imagenet-50"] + + # Model size. + num_layers: int + units: int + + # Batch size. + batch_size: int + + # Total number of training steps. + train_steps: int + + # Random seed. This is helpful for making sure that our experiments are all + # reproducible! + seed: int + + +# Note that we could also define this library using separate YAML files (similar to +# `config_path`/`config_name` in Hydra), but staying in Python enables seamless type +# checking + IDE support. +base_config_library = { + "small": ExperimentConfig( + dataset="mnist", + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + # The dcargs.MISSING sentinel allows us to specify that the seed should have no + # default, and needs to be populated from the CLI. + seed=dcargs.MISSING, + ), + "big": ExperimentConfig( + dataset="imagenet-50", + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=dcargs.MISSING, + ), +} + +if __name__ == "__main__": + # Get base configuration name from environment. + base_config_name = os.environ.get("BASE_CONFIG") + if base_config_name is None or base_config_name not in base_config_library: + raise SystemExit( + f"BASE_CONFIG should be set to one of {tuple(base_config_library.keys())}" + ) + + # Get base configuration from our library, and use it for default CLI parameters. + base_config = base_config_library[base_config_name] + config = dcargs.cli( + ExperimentConfig, + default_instance=base_config, + ) + print(config) diff --git a/setup.py b/setup.py index 6017fdab3..898375dbe 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ ], "type-checking": [ "mypy", + "pyright", ], }, classifiers=[ diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index 3eda0d86d..e87c421f7 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -1,7 +1,7 @@ import contextlib import io import pathlib -from typing import Any, Dict, Mapping, NamedTuple +from typing import Any, Dict, Mapping, NamedTuple, cast import pytest from typing_extensions import Literal, TypedDict @@ -102,7 +102,7 @@ class HelptextTypedDict(TypedDict): with contextlib.redirect_stdout(f): dcargs.cli(HelptextTypedDict, default_instance={"z": 3}, args=["--help"]) helptext = f.getvalue() - assert HelptextTypedDict.__doc__ in helptext + assert cast(str, HelptextTypedDict.__doc__) in helptext assert ":\n --x INT Documentation 1\n" in helptext assert "--y INT Documentation 2\n" in helptext assert "--z INT Documentation 3 (default: 3)\n" in helptext @@ -162,7 +162,7 @@ class HelptextNamedTupleDefault(NamedTuple): with contextlib.redirect_stdout(f): dcargs.cli(HelptextNamedTupleDefault, args=["--help"]) helptext = f.getvalue() - assert HelptextNamedTupleDefault.__doc__ in helptext + assert cast(str, HelptextNamedTupleDefault.__doc__) in helptext assert ":\n --x INT Documentation 1\n" in helptext assert "--y INT Documentation 2\n" in helptext assert "--z INT Documentation 3 (default: 3)\n" in helptext @@ -194,7 +194,7 @@ class HelptextNamedTuple(NamedTuple): args=["--help"], ) helptext = f.getvalue() - assert HelptextNamedTuple.__doc__ in helptext + assert cast(str, HelptextNamedTuple.__doc__) in helptext assert ":\n --x INT Documentation 1\n" in helptext assert "--y INT Documentation 2\n" in helptext assert "--z INT Documentation 3 (default: 3)\n" in helptext diff --git a/tests/test_errors.py b/tests/test_errors.py index 4a7a7ada4..2cf5344d9 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -12,14 +12,14 @@ def test_choices_in_tuples(): future, we might avoid this by implementing choice restrictions manually.""" # OK @dataclasses.dataclass - class A: + class A: # type: ignore x: Tuple[bool, bool] assert dcargs.cli(A, args=["--x", "True", "False"]) == A((True, False)) # OK @dataclasses.dataclass - class A: + class A: # type: ignore x: Tuple[bool, Literal["True", "False"]] assert dcargs.cli(A, args=["--x", "True", "False"]) == A((True, "False")) @@ -120,7 +120,7 @@ def test_nested_annotation(): class OneIntArg: x: int - def main(arg: List[OneIntArg]) -> List[OneIntArg]: + def main(arg: List[OneIntArg]) -> List[OneIntArg]: # type: ignore return arg with pytest.raises(dcargs.UnsupportedTypeAnnotationError): @@ -130,7 +130,7 @@ def main(arg: List[OneIntArg]) -> List[OneIntArg]: class OneStringArg: x: str - def main(arg: List[OneStringArg]) -> List[OneStringArg]: + def main(arg: List[OneStringArg]) -> List[OneStringArg]: # type: ignore return arg assert dcargs.cli(main, args=["--arg", "0", "1", "2"]) == [ @@ -168,7 +168,7 @@ class ActualParentClass(Generic[T]): # Documentation 2 y: T - z: T = 3 + z: T = 3 # type: ignore """Documentation 3""" @dataclasses.dataclass @@ -176,4 +176,4 @@ class ChildClass(UnrelatedParentClass, ActualParentClass[int]): pass with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(ChildClass, args=["--x", 1, "--y", 2, "--z", 3]) + dcargs.cli(ChildClass, args=["--x", "1", "--y", "2", "--z", "3"]) diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 3d76829da..72fbc9586 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -305,10 +305,6 @@ class Subparser(Generic[T1, T2]): ) -# Not implemented. It's unclear whether this is feasible, because generics are lost in -# the mro: https://github.com/python/typing/issues/777 - - def test_generic_inherited(): class UnrelatedParentClass: pass @@ -322,7 +318,7 @@ class ActualParentClass(Generic[T]): # Documentation 2 y: T - z: T = 3 + z: T = 3 # type: ignore """Documentation 3""" @dataclasses.dataclass @@ -331,5 +327,19 @@ class ChildClass(UnrelatedParentClass, ActualParentClass[int]): with pytest.raises(dcargs.UnsupportedTypeAnnotationError): assert dcargs.cli( - ChildClass, args=["--x", 1, "--y", 2, "--z", 3] + ChildClass, args=["--x", "1", "--y", "2", "--z", "3"] ) == ChildClass(1, 2, 3) + + +def test_serialize_missing(): + @dataclasses.dataclass + class TupleGenericVariableMissing(Generic[ScalarType]): + xyz: Tuple[ScalarType, ...] + + x = TupleGenericVariableMissing[int](xyz=(dcargs.MISSING, dcargs.MISSING)) + assert dcargs.to_yaml(x).count("!missing") == 2 + _check_serialization_identity(TupleGenericVariableMissing[int], x) + assert ( + dcargs.from_yaml(TupleGenericVariableMissing[int], dcargs.to_yaml(x)).xyz[0] + is dcargs.MISSING + ) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 8355700d3..574afbd17 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -3,7 +3,7 @@ import enum import io import pathlib -from typing import Generic, List, Optional, Tuple, TypeVar +from typing import Generic, List, Optional, Tuple, TypeVar, cast import pytest from typing_extensions import Literal @@ -29,7 +29,7 @@ class Helptext: with contextlib.redirect_stdout(f): dcargs.cli(Helptext, args=["--help"]) helptext = f.getvalue() - assert Helptext.__doc__ in helptext + assert cast(str, Helptext.__doc__) in helptext assert ":\n --x INT Documentation 1\n" in helptext assert "--y INT Documentation 2\n" in helptext assert "--z INT Documentation 3 (default: 3)\n" in helptext diff --git a/tests/test_missing.py b/tests/test_missing.py new file mode 100644 index 000000000..74b0eb27c --- /dev/null +++ b/tests/test_missing.py @@ -0,0 +1,73 @@ +"""Tests for dcargs.MISSING.""" + +import dataclasses +from typing import Tuple + +import pytest + +import dcargs + + +def test_missing(): + def main(a: int = 5, b: int = dcargs.MISSING, c: int = 3) -> Tuple[int, int, int]: + return a, b, c + + with pytest.raises(SystemExit): + dcargs.cli(main, args=[]) + assert dcargs.cli(main, args=["--b", "7"]) == (5, 7, 3) + + +def test_missing_dataclass(): + @dataclasses.dataclass + class Args2: + a: int = 5 + b: int = dcargs.MISSING + c: int = 3 + + with pytest.raises(SystemExit): + dcargs.cli(Args2, args=[]) + assert dcargs.cli(Args2, args=["--b", "7"]) == Args2(5, 7, 3) + + +def test_missing_default_instance(): + @dataclasses.dataclass + class Args2: + a: int + b: int + c: int + + with pytest.raises(SystemExit): + dcargs.cli( + Args2, + args=[], + default_instance=Args2(5, dcargs.MISSING, 3), + ) + assert dcargs.cli( + Args2, + args=["--b", "7"], + default_instance=Args2(5, dcargs.MISSING, 3), + ) == Args2(5, 7, 3) + + +def test_missing_nested_default_instance(): + @dataclasses.dataclass + class Child: + a: int = 5 + b: int = 3 + c: int = 7 + + @dataclasses.dataclass + class Parent: + child: Child + + with pytest.raises(SystemExit): + dcargs.cli( + Parent, + args=[], + default_instance=Parent(child=dcargs.MISSING), + ) + assert dcargs.cli( + Parent, + args=["--child.a", "5", "--child.b", "7", "--child.c", "3"], + default_instance=Parent(child=dcargs.MISSING), + ) == Parent(Child(5, 7, 3)) diff --git a/tests/test_nested.py b/tests/test_nested.py index 2b526b4b4..9203433f4 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -190,6 +190,69 @@ class DefaultSubparser: dcargs.cli(DefaultSubparser, args=["--x", "1", "c", "--y", "3"]) +def test_subparser_with_default_instance(): + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + + assert ( + dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "default-instance-http-server", "--y", "5"], + ) + == dcargs.cli( + DefaultInstanceSubparser, + args=[], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=5) + ), + ) + == dcargs.cli( + DefaultInstanceSubparser, + args=["default-instance-http-server"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=5) + ), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) + ) + assert dcargs.cli( + DefaultInstanceSubparser, + args=["default-instance-smtp-server", "--z", "3"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=5) + ), + ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) + assert ( + dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "default-instance-http-server", "--y", "8"], + ) + == dcargs.cli( + DefaultInstanceSubparser, + args=[], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=8) + ), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + ) + + with pytest.raises(SystemExit): + dcargs.cli(DefaultInstanceSubparser, args=["--x", "1", "b", "--z", "3"]) + with pytest.raises(SystemExit): + dcargs.cli(DefaultInstanceSubparser, args=["--x", "1", "c", "--y", "3"]) + + def test_optional_subparser(): @dataclasses.dataclass class OptionalHTTPServer: diff --git a/tests/test_unsupported_but_should_work.py b/tests/test_unsupported_but_should_work.py index 05e4a6465..572cac0c1 100644 --- a/tests/test_unsupported_but_should_work.py +++ b/tests/test_unsupported_but_should_work.py @@ -7,7 +7,7 @@ import contextlib import io import pathlib -from typing import Tuple +from typing import Tuple, cast import attr import omegaconf @@ -16,8 +16,8 @@ import dcargs -def test_missing(): - """Passing in a 'missing' default; this will mark an argument as required.""" +def test_omegaconf_missing(): + """Passing in a omegaconf.MISSING default; this will mark an argument as required.""" def main( required_a: int, @@ -123,7 +123,7 @@ class Helptext: with contextlib.redirect_stdout(f): dcargs.cli(Helptext, args=["--help"]) helptext = f.getvalue() - assert Helptext.__doc__ in helptext + assert cast(str, Helptext.__doc__) in helptext assert ":\n --x INT Documentation 1\n" in helptext assert "--y INT Documentation 2\n" in helptext assert "--z INT Documentation 3 (default: 3)\n" in helptext From ba8edf61d51211935c459345f567ba5760239d89 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 10 Jul 2022 20:22:18 -0700 Subject: [PATCH 050/491] Optional nested classes, multiple unions/subparsers --- README.md | 645 ++++++++++-------- _update_readme.py | 4 +- dcargs/_calling.py | 28 +- dcargs/_cli.py | 4 +- dcargs/_parsers.py | 150 ++-- dcargs/_serialization.py | 2 +- ...{13_base_configs.py => 06_base_configs.py} | 14 +- examples/{06_literals.py => 07_literals.py} | 4 +- ...sitional_args.py => 08_positional_args.py} | 4 +- examples/09_subparsers.py | 12 +- examples/10_multiple_subparsers.py | 66 ++ ...dard_classes.py => 13_standard_classes.py} | 4 +- examples/{10_generics.py => 14_generics.py} | 2 +- examples/_rename_example.py | 15 + setup.py | 2 +- tests/test_dict_namedtuple.py | 4 +- tests/test_errors.py | 20 - tests/test_forward_ref.py | 16 +- tests/test_generics_and_serialization.py | 10 +- tests/test_helptext.py | 49 +- tests/test_nested.py | 297 ++++++-- 21 files changed, 934 insertions(+), 418 deletions(-) rename examples/{13_base_configs.py => 06_base_configs.py} (80%) rename examples/{06_literals.py => 07_literals.py} (88%) rename examples/{07_positional_args.py => 08_positional_args.py} (93%) create mode 100644 examples/10_multiple_subparsers.py rename examples/{08_standard_classes.py => 13_standard_classes.py} (84%) rename examples/{10_generics.py => 14_generics.py} (95%) create mode 100644 examples/_rename_example.py diff --git a/README.md b/README.md index d51dd9661..bb47a994e 100644 --- a/README.md +++ b/README.md @@ -554,13 +554,150 @@ train_steps: 100000
-6. Literals +6. Base Configs + +
+ +We can integrate `dcargs.cli()` into common configuration patterns: here, we select +one of multiple possible base configurations, and then use the CLI to either override +(existing) or fill in (missing) values. + +**Code ([link](examples/06_base_configs.py)):** + +```python +import dataclasses +import os +from typing import Literal + +import dcargs + + +@dataclasses.dataclass(frozen=True) +class ExperimentConfig: + # Dataset to run experiment on. + dataset: Literal["mnist", "imagenet-50"] + + # Model size. + num_layers: int + units: int + + # Batch size. + batch_size: int + + # Total number of training steps. + train_steps: int + + # Random seed. This is helpful for making sure that our experiments are all + # reproducible! + seed: int + + +# Note that we could also define this library using separate YAML files (similar to +# `config_path`/`config_name` in Hydra), but staying in Python enables seamless type +# checking + IDE support. +base_config_library = { + "small": ExperimentConfig( + dataset="mnist", + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + # The dcargs.MISSING sentinel allows us to specify that the seed should have no + # default, and needs to be populated from the CLI. + seed=dcargs.MISSING, + ), + "big": ExperimentConfig( + dataset="imagenet-50", + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=dcargs.MISSING, + ), +} + +if __name__ == "__main__": + # Get base configuration name from environment. + base_config_name = os.environ.get("BASE_CONFIG") + if base_config_name is None or base_config_name not in base_config_library: + raise SystemExit( + f"BASE_CONFIG should be set to one of {tuple(base_config_library.keys())}" + ) + + # Get base configuration from our library, and use it for default CLI parameters. + base_config = base_config_library[base_config_name] + config = dcargs.cli( + ExperimentConfig, + default_instance=base_config, + ) + print(config) +``` + +
+ +**Example usage:** + +
+$ BASE_CONFIG=small python ./06_base_configs.py --help
+usage: 06_base_configs.py [-h] [--dataset {mnist,imagenet-50}]
+                          [--num-layers INT] [--units INT] [--batch-size INT]
+                          [--train-steps INT] --seed INT
+
+required arguments:
+  --seed INT            Random seed. This is helpful for making sure that our experiments are all
+                        reproducible!
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --dataset {mnist,imagenet-50}
+                        Dataset to run experiment on. (default: mnist)
+  --num-layers INT      Model size. (default: 4)
+  --units INT           Model size. (default: 64)
+  --batch-size INT      Batch size. (default: 2048)
+  --train-steps INT     Total number of training steps. (default: 30000)
+
+ +
+$ BASE_CONFIG=small python ./06_base_configs.py --seed 94720
+ExperimentConfig(dataset='mnist', num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720)
+
+ +
+$ BASE_CONFIG=big python ./06_base_configs.py --help
+usage: 06_base_configs.py [-h] [--dataset {mnist,imagenet-50}]
+                          [--num-layers INT] [--units INT] [--batch-size INT]
+                          [--train-steps INT] --seed INT
+
+required arguments:
+  --seed INT            Random seed. This is helpful for making sure that our experiments are all
+                        reproducible!
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --dataset {mnist,imagenet-50}
+                        Dataset to run experiment on. (default: imagenet-50)
+  --num-layers INT      Model size. (default: 8)
+  --units INT           Model size. (default: 256)
+  --batch-size INT      Batch size. (default: 32)
+  --train-steps INT     Total number of training steps. (default: 100000)
+
+ +
+$ BASE_CONFIG=big python ./06_base_configs.py --seed 94720
+ExperimentConfig(dataset='imagenet-50', num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720)
+
+ +
+ +
+ +7. Literals
`typing.Literal[]` can be used to restrict inputs to a fixed set of choices. -**Code ([link](examples/06_literals.py)):** +**Code ([link](examples/07_literals.py)):** ```python import dataclasses @@ -599,8 +736,8 @@ if __name__ == "__main__": **Example usage:** 
-$ python ./06_literals.py --help
-usage: 06_literals.py [-h] --enum {RED,GREEN,BLUE} --restricted-enum
+$ python ./07_literals.py --help
+usage: 07_literals.py [-h] --enum {RED,GREEN,BLUE} --restricted-enum
                       {RED,GREEN} --integer {0,1,2,3} --string {red,green}
                       [--restricted-enum-with-default {RED,GREEN}]
                       [--integer-with-default {0,1,2,3}]
@@ -623,7 +760,7 @@ optional arguments:
 
-$ python ./06_literals.py --enum RED --restricted-enum GREEN --integer 3 --string green
+$ python ./07_literals.py --enum RED --restricted-enum GREEN --integer 3 --string green
 Args(enum=<Color.RED: 1>, restricted_enum=<Color.GREEN: 2>, integer=3, string='green', restricted_enum_with_default=<Color.GREEN: 2>, integer_with_default=3, string_with_Default='red')
@@ -631,13 +768,13 @@ Args(enum=<Color.RED: 1>, restricted_enum=<Color.GREEN: 2>, integer=
-7. Positional Args +8. Positional Args
Positional-only arguments in functions are converted to positional CLI arguments. -**Code ([link](examples/07_positional_args.py)):** +**Code ([link](examples/08_positional_args.py)):** ```python from __future__ import annotations @@ -699,8 +836,8 @@ if __name__ == "__main__": **Example usage:** 
-$ python ./07_positional_args.py --help
-usage: 07_positional_args.py [-h] [--optimizer.algorithm {ADAM,SGD}]
+$ python ./08_positional_args.py --help
+usage: 08_positional_args.py [-h] [--optimizer.algorithm {ADAM,SGD}]
                              [--optimizer.learning-rate FLOAT]
                              [--optimizer.weight-decay FLOAT] [--force]
                              [--verbose] [--background-rgb FLOAT FLOAT FLOAT]
@@ -732,7 +869,7 @@ optional optimizer arguments:
 
-$ python ./07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5
+$ python ./08_positional_args.py ./a ./b --optimizer.learning-rate 1e-5
 source=PosixPath('a')
 dest=PosixPath('b')
 optimizer=OptimizerConfig(algorithm=<OptimizerType.ADAM: 1>, learning_rate=1e-05, weight_decay=0.01)
@@ -743,69 +880,6 @@ background_rgb=(1.0, 0.0, 0.0)
-
- -8. Standard Classes - -
- -In addition to functions and dataclasses, we can also generate CLIs from (the -constructors of) standard Python classes. - -**Code ([link](examples/08_standard_classes.py)):** - -```python -import dcargs - - -class Args: - def __init__( - self, - field1: str, - field2: int, - flag: bool = False, - ): - """Arguments. - - Args: - field1: A string field. - field2: A numeric field. - flag: A boolean flag. - """ - self.data = [field1, field2, flag] - - -if __name__ == "__main__": - args = dcargs.cli(Args) - print(args.data) -``` - -
- -**Example usage:** - -
-$ python ./08_standard_classes.py --help
-usage: 08_standard_classes.py [-h] --field1 STR --field2 INT [--flag]
-
-Arguments.
-
-required arguments:
-  --field1 STR  A string field.
-  --field2 INT  A numeric field.
-
-optional arguments:
-  -h, --help    show this help message and exit
-  --flag        A boolean flag.
-
- -
-$ python ./08_standard_classes.py --field1 hello --field2 7
-['hello', 7, False]
-
- -
-
9. Subparsers @@ -825,10 +899,6 @@ from typing import Union import dcargs -def main(command: Union[Checkout, Commit]) -> None: - print(command) - - @dataclasses.dataclass(frozen=True) class Checkout: """Checkout a branch.""" @@ -844,6 +914,10 @@ class Commit: all: bool = False +def main(cmd: Union[Checkout, Commit] = Checkout("main")) -> None: + print(cmd) + + if __name__ == "__main__": dcargs.cli(main) ``` @@ -854,49 +928,53 @@ if __name__ == "__main__": 
 $ python ./09_subparsers.py --help
-usage: 09_subparsers.py [-h] {checkout,commit} ...
+usage: 09_subparsers.py [-h] [{checkout,commit}] ...
 
 optional arguments:
-  -h, --help         show this help message and exit
+  -h, --help           show this help message and exit
+
+optional subcommands:
+   (default: checkout)
 
-subcommands:
-  {checkout,commit}
+  [{checkout,commit}]
 
 $ python ./09_subparsers.py commit --help
-usage: 09_subparsers.py commit [-h] --message STR [--all]
+usage: 09_subparsers.py commit [-h] --cmd.message STR [--cmd.all]
 
 Commit changes.
 
-required arguments:
-  --message STR
-
 optional arguments:
-  -h, --help     show this help message and exit
-  --all
+  -h, --help         show this help message and exit
+
+required cmd arguments:
+  --cmd.message STR
+
+optional cmd arguments:
+  --cmd.all
 
-$ python ./09_subparsers.py commit --message hello --all
+$ python ./09_subparsers.py commit --cmd.message hello --cmd.all
 Commit(message='hello', all=True)
 
 $ python ./09_subparsers.py checkout --help
-usage: 09_subparsers.py checkout [-h] --branch STR
+usage: 09_subparsers.py checkout [-h] [--cmd.branch STR]
 
 Checkout a branch.
 
-required arguments:
-  --branch STR
-
 optional arguments:
-  -h, --help    show this help message and exit
+  -h, --help        show this help message and exit
+
+optional cmd arguments:
+  --cmd.branch STR  (default: main)
 
-$ python ./09_subparsers.py checkout --branch main
+$ python ./09_subparsers.py checkout --cmd.branch main
 Checkout(branch='main')
@@ -904,49 +982,73 @@ Checkout(branch='main')
-10. Generics +10. Multiple Subparsers
-Example of parsing for generic dataclasses. +Multiple unions over nested types are populated using a series of subparsers. -**Code ([link](examples/10_generics.py)):** +**Code ([link](examples/10_multiple_subparsers.py)):** ```python +from __future__ import annotations + import dataclasses -from typing import Generic, TypeVar +from typing import Literal, Tuple, Union import dcargs -ScalarType = TypeVar("ScalarType") -ShapeType = TypeVar("ShapeType") +# Possible dataset configurations. -@dataclasses.dataclass(frozen=True) -class Point3(Generic[ScalarType]): - x: ScalarType - y: ScalarType - z: ScalarType - frame_id: str +@dataclasses.dataclass +class MnistDataset: + binary: bool = False + """Set to load binary version of MNIST dataset.""" -@dataclasses.dataclass(frozen=True) -class Triangle: - a: Point3[float] - b: Point3[float] - c: Point3[float] +@dataclasses.dataclass +class ImageNetDataset: + subset: Literal[50, 100, 1000] + """Choose between ImageNet-50, ImageNet-100, ImageNet-1000, etc.""" -@dataclasses.dataclass(frozen=True) -class Args(Generic[ShapeType]): - point_continuous: Point3[float] - point_discrete: Point3[int] - shape: ShapeType +# Possible optimizer configurations. + + +@dataclasses.dataclass +class AdamOptimizer: + learning_rate: float = 1e-3 + betas: Tuple[float, float] = (0.9, 0.999) + + +@dataclasses.dataclass +class SgdOptimizer: + learning_rate: float = 3e-4 + + +# Train script. + + +def train( + dataset: Union[MnistDataset, ImageNetDataset] = MnistDataset(), + optimizer: Union[AdamOptimizer, SgdOptimizer] = AdamOptimizer(), +) -> None: + """Example training script. + + Args: + dataset: Dataset to train on. + optimizer: Optimizer to train with. + + Returns: + None: + """ + print(dataset) + print(optimizer) if __name__ == "__main__": - args = dcargs.cli(Args[Triangle]) - print(args) + dcargs.cli(train) ```
@@ -954,55 +1056,48 @@ if __name__ == "__main__": **Example usage:** 
-$ python ./10_generics.py --help
-usage: 10_generics.py [-h] --point-continuous.x FLOAT --point-continuous.y
-                      FLOAT --point-continuous.z FLOAT
-                      --point-continuous.frame-id STR --point-discrete.x INT
-                      --point-discrete.y INT --point-discrete.z INT
-                      --point-discrete.frame-id STR --shape.a.x FLOAT
-                      --shape.a.y FLOAT --shape.a.z FLOAT --shape.a.frame-id
-                      STR --shape.b.x FLOAT --shape.b.y FLOAT --shape.b.z
-                      FLOAT --shape.b.frame-id STR --shape.c.x FLOAT
-                      --shape.c.y FLOAT --shape.c.z FLOAT --shape.c.frame-id
-                      STR
+$ python ./10_multiple_subparsers.py
+MnistDataset(binary=False)
+AdamOptimizer(learning_rate=0.001, betas=(0.9, 0.999))
+
+ +
+$ python ./10_multiple_subparsers.py --help
+usage: 10_multiple_subparsers.py [-h] [{mnist-dataset,image-net-dataset}] ...
+
+Example training script.
 
 optional arguments:
   -h, --help            show this help message and exit
 
-required point_continuous arguments:
-
-  --point-continuous.x FLOAT
-  --point-continuous.y FLOAT
-  --point-continuous.z FLOAT
-  --point-continuous.frame-id STR
+optional subcommands:
+  Dataset to train on.  (default: mnist-dataset)
 
-required point_discrete arguments:
+  [{mnist-dataset,image-net-dataset}]
+
- --point-discrete.x INT - --point-discrete.y INT - --point-discrete.z INT - --point-discrete.frame-id STR +
+$ python ./10_multiple_subparsers.py mnist-dataset --help
+usage: 10_multiple_subparsers.py mnist-dataset [-h] [--dataset.binary]
+                                               [{adam-optimizer,sgd-optimizer}]
+                                               ...
 
-required shape.a arguments:
+optional arguments:
+  -h, --help            show this help message and exit
 
-  --shape.a.x FLOAT
-  --shape.a.y FLOAT
-  --shape.a.z FLOAT
-  --shape.a.frame-id STR
+optional dataset arguments:
+  --dataset.binary      Set to load binary version of MNIST dataset.
 
-required shape.b arguments:
+optional subcommands:
+  Optimizer to train with. (default: adam-optimizer)
 
-  --shape.b.x FLOAT
-  --shape.b.y FLOAT
-  --shape.b.z FLOAT
-  --shape.b.frame-id STR
-
-required shape.c arguments:
+  [{adam-optimizer,sgd-optimizer}]
+
- --shape.c.x FLOAT - --shape.c.y FLOAT - --shape.c.z FLOAT - --shape.c.frame-id STR +
+$ python ./10_multiple_subparsers.py mnist-dataset adam-optimizer --optimizer.learning-rate 3e-4
+MnistDataset(binary=False)
+AdamOptimizer(learning_rate=0.0003, betas=(0.9, 0.999))
@@ -1144,98 +1239,112 @@ TupleType(field1='hello', field2=3, flag=False)
-13. Base Configs +13. Standard Classes
-Example of a common configuration pattern: selecting one of multiple possible base -configurations, and then using the CLI to either override existing values or fill in -missing ones. +In addition to functions and dataclasses, we can also generate CLIs from (the +constructors of) standard Python classes. + +**Code ([link](examples/13_standard_classes.py)):** + +```python +import dcargs + + +class Args: + def __init__( + self, + field1: str, + field2: int, + flag: bool = False, + ): + """Arguments. + + Args: + field1: A string field. + field2: A numeric field. + flag: A boolean flag. + """ + self.data = [field1, field2, flag] + + +if __name__ == "__main__": + args = dcargs.cli(Args) + print(args.data) +``` + +
+ +**Example usage:** -`BASE_CONFIG=small python ./13_base_configs.py --help` -`BASE_CONFIG=small python ./13_base_configs.py --seed 94720` -`BASE_CONFIG=big python ./13_base_configs.py --help` -`BASE_CONFIG=big python ./13_base_configs.py --seed 94720` +
+$ python ./13_standard_classes.py --help
+usage: 13_standard_classes.py [-h] --field1 STR --field2 INT [--flag]
 
-**Code ([link](examples/13_base_configs.py)):**
+Arguments.
+
+required arguments:
+  --field1 STR  A string field.
+  --field2 INT  A numeric field.
+
+optional arguments:
+  -h, --help    show this help message and exit
+  --flag        A boolean flag.
+
+ +
+$ python ./13_standard_classes.py --field1 hello --field2 7
+['hello', 7, False]
+
+ +
+ +
+ +14. Generics + +
+ +Example of parsing for generic dataclasses. + +**Code ([link](examples/14_generics.py)):** ```python import dataclasses -import os -import sys -from typing import Literal +from typing import Generic, TypeVar import dcargs +ScalarType = TypeVar("ScalarType") +ShapeType = TypeVar("ShapeType") -@dataclasses.dataclass(frozen=True) -class ExperimentConfig: - # Dataset to run experiment on. - dataset: Literal["mnist", "imagenet-50"] - # Model size. - num_layers: int - units: int +@dataclasses.dataclass(frozen=True) +class Point3(Generic[ScalarType]): + x: ScalarType + y: ScalarType + z: ScalarType + frame_id: str - # Batch size. - batch_size: int - # Total number of training steps. - train_steps: int +@dataclasses.dataclass(frozen=True) +class Triangle: + a: Point3[float] + b: Point3[float] + c: Point3[float] - # Random seed. This is helpful for making sure that our experiments are all - # reproducible! - seed: int +@dataclasses.dataclass(frozen=True) +class Args(Generic[ShapeType]): + point_continuous: Point3[float] + point_discrete: Point3[int] + shape: ShapeType -# Note that we could also define this library using separate YAML files (a la -# `config_path`/`config_name` in Hydra), but staying in Python enables seamless type -# checking + IDE support. -base_config_library = { - "small": ExperimentConfig( - dataset="mnist", - batch_size=2048, - num_layers=4, - units=64, - train_steps=30_000, - # The dcargs.MISSING sentinel allows us to specify that the seed should have no - # default, and needs to be populated from the CLI. - seed=dcargs.MISSING, - ), - "big": ExperimentConfig( - dataset="imagenet-50", - batch_size=32, - num_layers=8, - units=256, - train_steps=100_000, - seed=dcargs.MISSING, - ), -} if __name__ == "__main__": - # Get base configuration name from environment. - # base_config_name = os.environ.get("BASE_CONFIG") - # if base_config_name is None or base_config_name not in base_config_library: - # raise SystemExit( - # f"BASE_CONFIG should be set to one of {tuple(base_config_library.keys())}" - # ) - if ( - len(sys.argv) < 2 - or (base_config_name := sys.argv[1]) not in base_config_library - ): - raise SystemExit( - f"BASE_CONFIG should be set to one of {tuple(base_config_library.keys())}" - ) - - # Get base configuration from our library, and use it for default CLI parameters. - base_config = base_config_library[base_config_name] - config = dcargs.cli( - ExperimentConfig, - default_instance=base_config, - args=sys.argv[2:], - ) - - print(config) + args = dcargs.cli(Args[Triangle]) + print(args) ```
@@ -1243,53 +1352,55 @@ if __name__ == "__main__": **Example usage:** 
-$ python ./13_base_configs.py small --help
-usage: 13_base_configs.py [-h] [--dataset {mnist,imagenet-50}]
-                          [--num-layers INT] [--units INT] [--batch-size INT]
-                          [--train-steps INT] --seed INT
-
-required arguments:
-  --seed INT            Random seed. This is helpful for making sure that our experiments are all
-                        reproducible!
+$ python ./14_generics.py --help
+usage: 14_generics.py [-h] --point-continuous.x FLOAT --point-continuous.y
+                      FLOAT --point-continuous.z FLOAT
+                      --point-continuous.frame-id STR --point-discrete.x INT
+                      --point-discrete.y INT --point-discrete.z INT
+                      --point-discrete.frame-id STR --shape.a.x FLOAT
+                      --shape.a.y FLOAT --shape.a.z FLOAT --shape.a.frame-id
+                      STR --shape.b.x FLOAT --shape.b.y FLOAT --shape.b.z
+                      FLOAT --shape.b.frame-id STR --shape.c.x FLOAT
+                      --shape.c.y FLOAT --shape.c.z FLOAT --shape.c.frame-id
+                      STR
 
 optional arguments:
   -h, --help            show this help message and exit
-  --dataset {mnist,imagenet-50}
-                        Dataset to run experiment on. (default: mnist)
-  --num-layers INT      Model size. (default: 4)
-  --units INT           Model size. (default: 64)
-  --batch-size INT      Batch size. (default: 2048)
-  --train-steps INT     Total number of training steps. (default: 30000)
-
-
-$ python ./13_base_configs.py small --seed 94720
-ExperimentConfig(dataset='mnist', num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720)
-
+required point_continuous arguments: -
-$ python ./13_base_configs.py big --help
-usage: 13_base_configs.py [-h] [--dataset {mnist,imagenet-50}]
-                          [--num-layers INT] [--units INT] [--batch-size INT]
-                          [--train-steps INT] --seed INT
+  --point-continuous.x FLOAT
+  --point-continuous.y FLOAT
+  --point-continuous.z FLOAT
+  --point-continuous.frame-id STR
 
-required arguments:
-  --seed INT            Random seed. This is helpful for making sure that our experiments are all
-                        reproducible!
+required point_discrete arguments:
 
-optional arguments:
-  -h, --help            show this help message and exit
-  --dataset {mnist,imagenet-50}
-                        Dataset to run experiment on. (default: imagenet-50)
-  --num-layers INT      Model size. (default: 8)
-  --units INT           Model size. (default: 256)
-  --batch-size INT      Batch size. (default: 32)
-  --train-steps INT     Total number of training steps. (default: 100000)
-
+ --point-discrete.x INT + --point-discrete.y INT + --point-discrete.z INT + --point-discrete.frame-id STR -
-$ python ./13_base_configs.py big --seed 94720
-ExperimentConfig(dataset='imagenet-50', num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720)
+required shape.a arguments:
+
+  --shape.a.x FLOAT
+  --shape.a.y FLOAT
+  --shape.a.z FLOAT
+  --shape.a.frame-id STR
+
+required shape.b arguments:
+
+  --shape.b.x FLOAT
+  --shape.b.y FLOAT
+  --shape.b.z FLOAT
+  --shape.b.frame-id STR
+
+required shape.c arguments:
+
+  --shape.c.x FLOAT
+  --shape.c.y FLOAT
+  --shape.c.z FLOAT
+  --shape.c.frame-id STR
diff --git a/_update_readme.py b/_update_readme.py index 1fef17e0f..096113b60 100644 --- a/_update_readme.py +++ b/_update_readme.py @@ -118,7 +118,9 @@ def format_script_for_readme(path: pathlib.Path) -> str: def get_examples_str(examples_dir: pathlib.Path, constants: Constants) -> str: - script_paths = sorted(examples_dir.glob("*.py")) + script_paths = filter( + lambda p: not p.name.startswith("_"), sorted(examples_dir.glob("*.py")) + ) with concurrent.futures.ThreadPoolExecutor() as executor: return "\n".join( executor.map( diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 70a98d072..c1aa66c2e 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -83,22 +83,34 @@ def get_value_from_arg(arg: str) -> Any: consumed_keywords |= consumed_keywords_child else: # Unions over dataclasses (subparsers). This is the only other option. - assert parser_definition.subparsers is not None - assert field.name == parser_definition.subparsers.name + assert len(parser_definition.subparsers_from_name) > 0 + assert field.name in parser_definition.subparsers_from_name subparser_dest = _strings.SUBPARSER_DEST_FMT.format( name=prefixed_field_name ) - subparser_name = get_value_from_arg(subparser_dest) + if subparser_dest in value_from_arg: + subparser_name = get_value_from_arg(subparser_dest) + else: + default_instance = parser_definition.subparsers_from_name[ + field.name + ].default_instance + assert default_instance is not None + subparser_name = None if subparser_name is None: # No subparser selected -- this should only happen when we do either # Optional[Union[A, B, ...]] or Union[A, B, None], or have a # default/default_factory set. assert ( type(None) in get_args(field_type) - or parser_definition.subparsers.default_instance is not None + or parser_definition.subparsers_from_name[ + field.name + ].default_instance + is not None ) - value = parser_definition.subparsers.default_instance + value = parser_definition.subparsers_from_name[ + field.name + ].default_instance else: options = map( lambda x: x if x not in type_from_typevar else type_from_typevar[x], @@ -112,8 +124,12 @@ def get_value_from_arg(arg: str) -> Any: assert chosen_f is not None value, consumed_keywords_child = call_from_args( chosen_f, - parser_definition.subparsers.parser_from_name[subparser_name], + parser_definition.subparsers_from_name[field.name].parser_from_name[ + subparser_name + ], value_from_arg, + field_name_prefix=prefixed_field_name + + _strings.NESTED_FIELD_DELIMETER, ) consumed_keywords |= consumed_keywords_child diff --git a/dcargs/_cli.py b/dcargs/_cli.py index fd95c7530..eea3d58c4 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -111,5 +111,7 @@ def cli( print(e.args[0]) raise SystemExit() - assert consumed_keywords == value_from_arg.keys() + # print(consumed_keywords) + # print(value_from_arg.keys()) + # assert consumed_keywords == value_from_arg.keys() return out diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 35df346ce..ea454a143 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -86,7 +86,8 @@ class ParserSpecification: description: str args: List[_arguments.ArgumentDefinition] helptext_from_nested_class_field_name: Dict[str, Optional[str]] - subparsers: Optional[SubparsersSpecification] + subparsers_from_name: Dict[str, SubparsersSpecification] + prefix: str @staticmethod def from_callable( @@ -95,6 +96,7 @@ def from_callable( parent_classes: Set[Type], parent_type_from_typevar: Optional[Dict[TypeVar, Type]], default_instance: Optional[T], + prefix: str = "", ) -> ParserSpecification: """Create a parser definition from a callable.""" @@ -121,7 +123,7 @@ def from_callable( args = [] helptext_from_nested_class_field_name = {} - subparsers = None + subparsers_from_name = {} for field in _fields.field_list_from_callable( f=f, default_instance=default_instance @@ -147,13 +149,10 @@ def from_callable( field, type_from_typevar=type_from_typevar, parent_classes=parent_classes, + prefix=prefix + field.name + _strings.NESTED_FIELD_DELIMETER, ) if subparsers_attempt is not None: - if subparsers is not None: - raise _instantiators.UnsupportedTypeAnnotationError( - "Only one set of subparsers is allowed." - ) - subparsers = subparsers_attempt + subparsers_from_name[subparsers_attempt.name] = subparsers_attempt continue # (2) Handle nested callables. @@ -164,16 +163,10 @@ def from_callable( parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, default_instance=field.default, + prefix=prefix + field.name + _strings.NESTED_FIELD_DELIMETER, ) - for arg in nested_parser.args: - args.append( - dataclasses.replace( - arg, - prefix=field.name - + _strings.NESTED_FIELD_DELIMETER - + arg.prefix, - ) - ) + args.extend(nested_parser.args) + for k, v in nested_parser.helptext_from_nested_class_field_name.items(): helptext_from_nested_class_field_name[ field.name + _strings.NESTED_FIELD_DELIMETER + k @@ -192,17 +185,27 @@ def from_callable( # (3) Handle primitive types. These produce a single argument! args.append( _arguments.ArgumentDefinition( - prefix="", field=field, type_from_typevar=type_from_typevar + prefix=prefix, field=field, type_from_typevar=type_from_typevar ) ) + # If a later subparser is required, all previous ones should be as well. + subparsers_required = False + for name, subparsers in list(subparsers_from_name.items())[::-1]: + if subparsers.required: + subparsers_required = True + subparsers_from_name[name] = dataclasses.replace( + subparsers, required=subparsers_required + ) + return ParserSpecification( description=description if description is not None else _docstrings.get_callable_description(f), args=args, helptext_from_nested_class_field_name=helptext_from_nested_class_field_name, - subparsers=subparsers, + subparsers_from_name=subparsers_from_name, + prefix=prefix, ) def apply(self, parser: argparse.ArgumentParser) -> None: @@ -264,32 +267,60 @@ def format_group_name(nested_field_name: str, required: bool) -> str: target_groups[arg.prefix] = parser.add_argument_group( format_group_name(nested_field_name, required=arg.lowered.required), # Add a description, but only to the first group for a field. - description=self.helptext_from_nested_class_field_name[ + description=self.helptext_from_nested_class_field_name.get( nested_field_name - ] + ) if arg.prefix not in other_groups else None, ) arg.add_argument(target_groups[arg.prefix]) - # Add subparsers. - if self.subparsers is not None: - title = "subcommands" - metavar = "{" + ",".join(self.subparsers.parser_from_name.keys()) + "}" - if not self.subparsers.required: - title = "optional " + title - metavar = f"[{metavar}]" - - argparse_subparsers = parser.add_subparsers( - dest=_strings.SUBPARSER_DEST_FMT.format(name=self.subparsers.name), - description=self.subparsers.description, - required=self.subparsers.required, - title=title, - metavar=metavar, - ) - for name, subparser_def in self.subparsers.parser_from_name.items(): - subparser = argparse_subparsers.add_parser(name) - subparser_def.apply(subparser) + # Create subparser tree. + if len(self.subparsers_from_name) > 0: + prev_subparser_tree_nodes = [parser] # Root node. + for subparsers in self.subparsers_from_name.values(): + title = "subcommands" + metavar = "{" + ",".join(subparsers.parser_from_name.keys()) + "}" + if not subparsers.required: + title = "optional " + title + metavar = f"[{metavar}]" + + subparser_tree_nodes = [] + for p in prev_subparser_tree_nodes: + # Add subparsers to every node in previous level of the tree. + argparse_subparsers = p.add_subparsers( + dest=self.prefix + + _strings.SUBPARSER_DEST_FMT.format(name=subparsers.name), + description=subparsers.description, + required=subparsers.required, + title=title, + metavar=metavar, + ) + for name, subparser_def in subparsers.parser_from_name.items(): + subparser = argparse_subparsers.add_parser(name) + subparser_def.apply(subparser) + + def _get_leaf_subparsers( + node: argparse.ArgumentParser, + ) -> List[argparse.ArgumentParser]: + if node._subparsers is None: + return [node] + else: + # Magic! + return list( + itertools.chain( + *map( + _get_leaf_subparsers, + node._subparsers._actions[ + -1 + ]._name_parser_map.values(), # type: ignore + ) + ) + ) + + subparser_tree_nodes.extend(_get_leaf_subparsers(subparser)) + + prev_subparser_tree_nodes = subparser_tree_nodes @dataclasses.dataclass(frozen=True) @@ -307,6 +338,7 @@ def from_field( field: _fields.Field, type_from_typevar: Dict[TypeVar, Type], parent_classes: Set[Type], + prefix: str, ) -> Optional[SubparsersSpecification]: # Union of classes should create subparsers. if get_origin(field.typ) is not Union: @@ -315,9 +347,7 @@ def from_field( # We don't use sets here to retain order of subcommands. options = [type_from_typevar.get(typ, typ) for typ in get_args(field.typ)] options_no_none = [o for o in options if o != type(None)] # noqa - if len(options_no_none) < 2 or not all( - map(_is_possibly_nested_type, options_no_none) - ): + if not all(map(_is_possibly_nested_type, options_no_none)): return None parser_from_name: Dict[str, ParserSpecification] = {} @@ -331,17 +361,47 @@ def from_field( default_instance=field.default if isinstance(field.default, _resolver.unwrap_origin(option)) else None, + prefix=prefix, ) + # Optional if: type hint is Optional[], or a default instance is provided. + required = True + if options != options_no_none: + # Optional[] type. + required = False + elif field.default is not None: + required = False + + # If there are any required arguments in the default subparser, we should mark + # the subparser group as a whole as required. + if field.default is not None: + default_parser = parser_from_name[ + _strings.subparser_name_from_type(type(field.default)) + ] + if any(map(lambda arg: arg.lowered.required, default_parser.args)): + required = True + if any( + map( + lambda subparsers: subparsers.required, + default_parser.subparsers_from_name.values(), + ) + ): + required = True + + description_parts = [] + if field.helptext is not None: + description_parts.append(field.helptext) + if not required and field.default is not None: + default = _strings.subparser_name_from_type(type(field.default)) + description_parts.append(f" (default: {default})") + return SubparsersSpecification( name=field.name, # If we wanted, we could add information about the default instance # automatically, as is done for normal fields. But for now we just rely on # the user to include it in the docstring. - description=field.helptext, + description=" ".join(description_parts), parser_from_name=parser_from_name, - # Required if: type hint is not Optional[], or a default instance is - # provided. - required=(options == options_no_none) and field.default is None, + required=required, default_instance=field.default, ) diff --git a/dcargs/_serialization.py b/dcargs/_serialization.py index 860ac0557..7279efae8 100644 --- a/dcargs/_serialization.py +++ b/dcargs/_serialization.py @@ -116,7 +116,7 @@ def make_enum_constructor(typ: Type): DataclassLoader.add_constructor( tag=MISSING_YAML_TAG_PREFIX, - constructor=lambda *_unused: _fields.MISSING, + constructor=lambda *_unused: _fields.MISSING, # type: ignore ) return DataclassLoader diff --git a/examples/13_base_configs.py b/examples/06_base_configs.py similarity index 80% rename from examples/13_base_configs.py rename to examples/06_base_configs.py index 9a04520b8..608dc97a2 100644 --- a/examples/13_base_configs.py +++ b/examples/06_base_configs.py @@ -1,12 +1,12 @@ -"""Example of a common configuration pattern: selecting one of multiple possible base -configurations, and then using the CLI to either override existing values or fill in -missing ones. +"""We can integrate `dcargs.cli()` into common configuration patterns: here, we select +one of multiple possible base configurations, and then use the CLI to either override +(existing) or fill in (missing) values. Usage: -`BASE_CONFIG=small python ./13_base_configs.py --help` -`BASE_CONFIG=small python ./13_base_configs.py --seed 94720` -`BASE_CONFIG=big python ./13_base_configs.py --help` -`BASE_CONFIG=big python ./13_base_configs.py --seed 94720` +`BASE_CONFIG=small python ./06_base_configs.py --help` +`BASE_CONFIG=small python ./06_base_configs.py --seed 94720` +`BASE_CONFIG=big python ./06_base_configs.py --help` +`BASE_CONFIG=big python ./06_base_configs.py --seed 94720` """ import dataclasses diff --git a/examples/06_literals.py b/examples/07_literals.py similarity index 88% rename from examples/06_literals.py rename to examples/07_literals.py index c44a5a8c9..b804bbfd5 100644 --- a/examples/06_literals.py +++ b/examples/07_literals.py @@ -1,8 +1,8 @@ """`typing.Literal[]` can be used to restrict inputs to a fixed set of choices. Usage: -`python ./06_literals.py --help` -`python ./06_literals.py --enum RED --restricted-enum GREEN --integer 3 --string green` +`python ./07_literals.py --help` +`python ./07_literals.py --enum RED --restricted-enum GREEN --integer 3 --string green` """ import dataclasses diff --git a/examples/07_positional_args.py b/examples/08_positional_args.py similarity index 93% rename from examples/07_positional_args.py rename to examples/08_positional_args.py index 317eccaa1..713d2aab0 100644 --- a/examples/07_positional_args.py +++ b/examples/08_positional_args.py @@ -1,8 +1,8 @@ """Positional-only arguments in functions are converted to positional CLI arguments. Usage: -`python ./07_positional_args.py --help` -`python ./07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5` +`python ./08_positional_args.py --help` +`python ./08_positional_args.py ./a ./b --optimizer.learning-rate 1e-5` """ from __future__ import annotations diff --git a/examples/09_subparsers.py b/examples/09_subparsers.py index af300b33d..911b9eb2e 100644 --- a/examples/09_subparsers.py +++ b/examples/09_subparsers.py @@ -3,9 +3,9 @@ Usage: `python ./09_subparsers.py --help` `python ./09_subparsers.py commit --help` -`python ./09_subparsers.py commit --message hello --all` +`python ./09_subparsers.py commit --cmd.message hello --cmd.all` `python ./09_subparsers.py checkout --help` -`python ./09_subparsers.py checkout --branch main` +`python ./09_subparsers.py checkout --cmd.branch main` """ from __future__ import annotations @@ -16,10 +16,6 @@ import dcargs -def main(command: Union[Checkout, Commit]) -> None: - print(command) - - @dataclasses.dataclass(frozen=True) class Checkout: """Checkout a branch.""" @@ -35,5 +31,9 @@ class Commit: all: bool = False +def main(cmd: Union[Checkout, Commit] = Checkout("main")) -> None: + print(cmd) + + if __name__ == "__main__": dcargs.cli(main) diff --git a/examples/10_multiple_subparsers.py b/examples/10_multiple_subparsers.py new file mode 100644 index 000000000..e1ed25244 --- /dev/null +++ b/examples/10_multiple_subparsers.py @@ -0,0 +1,66 @@ +"""Multiple unions over nested types are populated using a series of subparsers. + +Usage: +`python ./10_multiple_subparsers.py` +`python ./10_multiple_subparsers.py --help` +`python ./10_multiple_subparsers.py mnist-dataset --help` +`python ./10_multiple_subparsers.py mnist-dataset adam-optimizer --optimizer.learning-rate 3e-4` +""" +from __future__ import annotations + +import dataclasses +from typing import Literal, Tuple, Union + +import dcargs + +# Possible dataset configurations. + + +@dataclasses.dataclass +class MnistDataset: + binary: bool = False + """Set to load binary version of MNIST dataset.""" + + +@dataclasses.dataclass +class ImageNetDataset: + subset: Literal[50, 100, 1000] + """Choose between ImageNet-50, ImageNet-100, ImageNet-1000, etc.""" + + +# Possible optimizer configurations. + + +@dataclasses.dataclass +class AdamOptimizer: + learning_rate: float = 1e-3 + betas: Tuple[float, float] = (0.9, 0.999) + + +@dataclasses.dataclass +class SgdOptimizer: + learning_rate: float = 3e-4 + + +# Train script. + + +def train( + dataset: Union[MnistDataset, ImageNetDataset] = MnistDataset(), + optimizer: Union[AdamOptimizer, SgdOptimizer] = AdamOptimizer(), +) -> None: + """Example training script. + + Args: + dataset: Dataset to train on. + optimizer: Optimizer to train with. + + Returns: + None: + """ + print(dataset) + print(optimizer) + + +if __name__ == "__main__": + dcargs.cli(train) diff --git a/examples/08_standard_classes.py b/examples/13_standard_classes.py similarity index 84% rename from examples/08_standard_classes.py rename to examples/13_standard_classes.py index 0c4ce4e6b..c4d942e45 100644 --- a/examples/08_standard_classes.py +++ b/examples/13_standard_classes.py @@ -2,8 +2,8 @@ constructors of) standard Python classes. Usage: -`python ./08_standard_classes.py --help` -`python ./08_standard_classes.py --field1 hello --field2 7` +`python ./13_standard_classes.py --help` +`python ./13_standard_classes.py --field1 hello --field2 7` """ import dcargs diff --git a/examples/10_generics.py b/examples/14_generics.py similarity index 95% rename from examples/10_generics.py rename to examples/14_generics.py index 2175ad629..c377e06ef 100644 --- a/examples/10_generics.py +++ b/examples/14_generics.py @@ -1,7 +1,7 @@ """Example of parsing for generic dataclasses. Usage: -`python ./10_generics.py --help` +`python ./14_generics.py --help` """ import dataclasses diff --git a/examples/_rename_example.py b/examples/_rename_example.py new file mode 100644 index 000000000..324558e88 --- /dev/null +++ b/examples/_rename_example.py @@ -0,0 +1,15 @@ +import pathlib + +import dcargs + + +def main(source: pathlib.Path, dest: pathlib.Path, /) -> None: + """Rename an example, while also replacing any occurrences of its old name within + its contents.""" + assert not dest.exists() + source.write_text(source.read_text().replace(str(source), str(dest))) + source.rename(dest) + + +if __name__ == "__main__": + dcargs.cli(main) diff --git a/setup.py b/setup.py index 898375dbe..d3044ec94 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.1.5", + version="0.1.6", description="Strongly typed, zero-effort CLIs", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index e87c421f7..81857910c 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -187,8 +187,8 @@ class HelptextNamedTuple(NamedTuple): HelptextNamedTuple, default_instance=HelptextNamedTuple( # Sketchy, unsupported behavior... - x=None, # type: ignore - y=None, # type: ignore + x=dcargs.MISSING, + y=dcargs.MISSING, z=3, ), args=["--help"], diff --git a/tests/test_errors.py b/tests/test_errors.py index 2cf5344d9..1276d3644 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -76,26 +76,6 @@ def main(x: Literal[0, "5"]) -> None: dcargs.cli(main, args=[]) -def test_multiple_subparsers(): - """argparse doesn't support multiple subparsers.""" - - @dataclasses.dataclass - class Subcommmand1: - pass - - @dataclasses.dataclass - class Subcommand2: - pass - - @dataclasses.dataclass - class MultipleSubparsers: - x: Union[Subcommmand1, Subcommand2] - y: Union[Subcommmand1, Subcommand2] - - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(MultipleSubparsers, args=[]) - - # Must be global. @dataclasses.dataclass class _CycleDataclass: diff --git a/tests/test_forward_ref.py b/tests/test_forward_ref.py index 5915e2478..fb96e0b08 100644 --- a/tests/test_forward_ref.py +++ b/tests/test_forward_ref.py @@ -29,20 +29,20 @@ class C: def test_forward_ref_1(): - assert dcargs.cli(A1, args=["--x", "1", "b", "--y", "3"]) == A1(x=1, bc=B(y=3)) - assert dcargs.cli(A1, args=["--x", "1", "c", "--z", "3"]) == A1(x=1, bc=C(z=3)) + assert dcargs.cli(A1, args=["--x", "1", "b", "--bc.y", "3"]) == A1(x=1, bc=B(y=3)) + assert dcargs.cli(A1, args=["--x", "1", "c", "--bc.z", "3"]) == A1(x=1, bc=C(z=3)) with pytest.raises(SystemExit): - dcargs.cli(A1, args=["--x", "1", "b", "--z", "3"]) + dcargs.cli(A1, args=["--x", "1", "b", "--bc.z", "3"]) with pytest.raises(SystemExit): - dcargs.cli(A1, args=["--x", "1", "c", "--y", "3"]) + dcargs.cli(A1, args=["--x", "1", "c", "--bc.y", "3"]) def test_forward_ref_2(): - assert dcargs.cli(A2, args=["--x", "1", "b", "--y", "3"]) == A2(x=1, bc=B(y=3)) - assert dcargs.cli(A2, args=["--x", "1", "c", "--z", "3"]) == A2(x=1, bc=C(z=3)) + assert dcargs.cli(A2, args=["--x", "1", "b", "--bc.y", "3"]) == A2(x=1, bc=B(y=3)) + assert dcargs.cli(A2, args=["--x", "1", "c", "--bc.z", "3"]) == A2(x=1, bc=C(z=3)) with pytest.raises(SystemExit): - dcargs.cli(A2, args=["--x", "1", "b", "--z", "3"]) + dcargs.cli(A2, args=["--x", "1", "b", "--bc.z", "3"]) with pytest.raises(SystemExit): - dcargs.cli(A2, args=["--x", "1", "c", "--y", "3"]) + dcargs.cli(A2, args=["--x", "1", "c", "--bc.y", "3"]) diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 72fbc9586..e5119804d 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -258,13 +258,13 @@ class Subparser(Generic[T1, T2]): command: Union[T1, T2] parsed_instance = dcargs.cli( - Subparser[CommandOne, CommandTwo], args="command-one --a 5".split(" ") + Subparser[CommandOne, CommandTwo], args="command-one --command.a 5".split(" ") ) assert parsed_instance == Subparser(CommandOne(5)) _check_serialization_identity(Subparser[CommandOne, CommandTwo], parsed_instance) parsed_instance = dcargs.cli( - Subparser[CommandOne, CommandTwo], args="command-two --b 7".split(" ") + Subparser[CommandOne, CommandTwo], args="command-two --command.b 7".split(" ") ) assert parsed_instance == Subparser(CommandTwo(7)) _check_serialization_identity(Subparser[CommandOne, CommandTwo], parsed_instance) @@ -285,7 +285,8 @@ class Subparser(Generic[T1, T2]): command: Union[T1, T2] parsed_instance = dcargs.cli( - Subparser[Command[int], Command[float]], args="command-int --a 5 3".split(" ") + Subparser[Command[int], Command[float]], + args="command-int --command.a 5 3".split(" "), ) assert parsed_instance == Subparser(Command([5, 3])) and isinstance( parsed_instance.command.a[0], int @@ -295,7 +296,8 @@ class Subparser(Generic[T1, T2]): ) parsed_instance = dcargs.cli( - Subparser[Command[int], Command[float]], args="command-float --a 7 2".split(" ") + Subparser[Command[int], Command[float]], + args="command-float --command.a 7 2".split(" "), ) assert parsed_instance == Subparser(Command([7.0, 2.0])) and isinstance( parsed_instance.command.a[0], float diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 574afbd17..138459ad0 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -3,7 +3,7 @@ import enum import io import pathlib -from typing import Generic, List, Optional, Tuple, TypeVar, cast +from typing import Generic, List, Optional, Tuple, TypeVar, Union, cast import pytest from typing_extensions import Literal @@ -371,3 +371,50 @@ class OptionalLiteralHelptext: dcargs.cli(OptionalLiteralHelptext, args=["--help"]) helptext = f.getvalue() assert "--x {1,2,3} A number. (default: None)\n" in helptext + + +def test_multiple_subparsers_helptext(): + @dataclasses.dataclass + class Subcommand1: + x: int = 0 + + @dataclasses.dataclass + class Subcommand2: + y: int = 1 + + @dataclasses.dataclass + class Subcommand3: + z: int = 2 + + @dataclasses.dataclass + class MultipleSubparsers: + # Field a description. + a: Union[Subcommand1, Subcommand2, Subcommand3] + # Field b description. + b: Union[Subcommand1, Subcommand2, Subcommand3] + # Field c description. + c: Union[Subcommand1, Subcommand2, Subcommand3] = dataclasses.field( + default_factory=Subcommand3 + ) + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli(MultipleSubparsers, args=["--help"]) + helptext = f.getvalue() + + assert "Field a description." in helptext + assert "Field b description." not in helptext + assert "Field c description." not in helptext + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli( + MultipleSubparsers, args=["subcommand1", "subcommand1", "--help"] + ) + helptext = f.getvalue() + + assert "Field a description." not in helptext + assert "Field b description." not in helptext + assert "Field c description. (default: subcommand3)" in helptext diff --git a/tests/test_nested.py b/tests/test_nested.py index 9203433f4..0871cbdec 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -89,29 +89,31 @@ class Nested: assert dcargs.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) -# TODO: implement this! -# def test_optional_nested(): -# @dataclasses.dataclass -# class OptionalNestedChild: -# y: int -# z: int -# -# @dataclasses.dataclass -# class OptionalNested: -# x: int -# b: Optional[OptionalNestedChild] -# -# assert dcargs.cli(OptionalNested, args=["--x", "1"]) == OptionalNested( -# x=1, b=None -# ) -# with pytest.raises(SystemExit): -# dcargs.cli(OptionalNested, args=["--x", "1", "--b.y", "3"]) -# with pytest.raises(SystemExit): -# dcargs.cli(OptionalNested, args=["--x", "1", "--b.z", "3"]) -# -# assert dcargs.cli( -# OptionalNested, args=["--x", "1", "--b.y", "2", "--b.z", "3"] -# ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) +def test_optional_nested(): + @dataclasses.dataclass + class OptionalNestedChild: + y: int + z: int + + @dataclasses.dataclass + class OptionalNested: + x: int + b: Optional[OptionalNestedChild] + + assert dcargs.cli(OptionalNested, args=["--x", "1"]) == OptionalNested(x=1, b=None) + with pytest.raises(SystemExit): + dcargs.cli( + OptionalNested, args=["--x", "1", "optional-nested-child", "--b.y", "3"] + ) + with pytest.raises(SystemExit): + dcargs.cli( + OptionalNested, args=["--x", "1", "optional-nested-child", "--b.z", "3"] + ) + + assert dcargs.cli( + OptionalNested, + args=["--x", "1", "optional-nested-child", "--b.y", "2", "--b.z", "3"], + ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) def test_subparser(): @@ -129,10 +131,10 @@ class Subparser: bc: Union[HTTPServer, SMTPServer] assert dcargs.cli( - Subparser, args=["--x", "1", "http-server", "--y", "3"] + Subparser, args=["--x", "1", "http-server", "--bc.y", "3"] ) == Subparser(x=1, bc=HTTPServer(y=3)) assert dcargs.cli( - Subparser, args=["--x", "1", "smtp-server", "--z", "3"] + Subparser, args=["--x", "1", "smtp-server", "--bc.z", "3"] ) == Subparser(x=1, bc=SMTPServer(z=3)) with pytest.raises(SystemExit): @@ -140,10 +142,10 @@ class Subparser: dcargs.cli(Subparser, args=["--x", "1"]) with pytest.raises(SystemExit): # Wrong field. - dcargs.cli(Subparser, args=["--x", "1", "http-server", "--z", "3"]) + dcargs.cli(Subparser, args=["--x", "1", "http-server", "--bc.z", "3"]) with pytest.raises(SystemExit): # Wrong field. - dcargs.cli(Subparser, args=["--x", "1", "smtp-server", "--y", "3"]) + dcargs.cli(Subparser, args=["--x", "1", "smtp-server", "--bc.y", "3"]) def test_subparser_with_default(): @@ -164,17 +166,17 @@ class DefaultSubparser: assert ( dcargs.cli( - DefaultSubparser, args=["--x", "1", "default-http-server", "--y", "5"] + DefaultSubparser, args=["--x", "1", "default-http-server", "--bc.y", "5"] ) == dcargs.cli(DefaultSubparser, args=["--x", "1"]) == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=5)) ) assert dcargs.cli( - DefaultSubparser, args=["--x", "1", "default-smtp-server", "--z", "3"] + DefaultSubparser, args=["--x", "1", "default-smtp-server", "--bc.z", "3"] ) == DefaultSubparser(x=1, bc=DefaultSMTPServer(z=3)) assert ( dcargs.cli( - DefaultSubparser, args=["--x", "1", "default-http-server", "--y", "8"] + DefaultSubparser, args=["--x", "1", "default-http-server", "--bc.y", "8"] ) == dcargs.cli( DefaultSubparser, @@ -185,9 +187,9 @@ class DefaultSubparser: ) with pytest.raises(SystemExit): - dcargs.cli(DefaultSubparser, args=["--x", "1", "b", "--z", "3"]) + dcargs.cli(DefaultSubparser, args=["--x", "1", "b", "--bc.z", "3"]) with pytest.raises(SystemExit): - dcargs.cli(DefaultSubparser, args=["--x", "1", "c", "--y", "3"]) + dcargs.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) def test_subparser_with_default_instance(): @@ -207,7 +209,7 @@ class DefaultInstanceSubparser: assert ( dcargs.cli( DefaultInstanceSubparser, - args=["--x", "1", "default-instance-http-server", "--y", "5"], + args=["--x", "1", "default-instance-http-server", "--bc.y", "5"], ) == dcargs.cli( DefaultInstanceSubparser, @@ -227,7 +229,7 @@ class DefaultInstanceSubparser: ) assert dcargs.cli( DefaultInstanceSubparser, - args=["default-instance-smtp-server", "--z", "3"], + args=["default-instance-smtp-server", "--bc.z", "3"], default_instance=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=5) ), @@ -235,7 +237,7 @@ class DefaultInstanceSubparser: assert ( dcargs.cli( DefaultInstanceSubparser, - args=["--x", "1", "default-instance-http-server", "--y", "8"], + args=["--x", "1", "default-instance-http-server", "--bc.y", "8"], ) == dcargs.cli( DefaultInstanceSubparser, @@ -248,9 +250,9 @@ class DefaultInstanceSubparser: ) with pytest.raises(SystemExit): - dcargs.cli(DefaultInstanceSubparser, args=["--x", "1", "b", "--z", "3"]) + dcargs.cli(DefaultInstanceSubparser, args=["--x", "1", "b", "--bc.z", "3"]) with pytest.raises(SystemExit): - dcargs.cli(DefaultInstanceSubparser, args=["--x", "1", "c", "--y", "3"]) + dcargs.cli(DefaultInstanceSubparser, args=["--x", "1", "c", "--bc.y", "3"]) def test_optional_subparser(): @@ -268,10 +270,10 @@ class OptionalSubparser: bc: Optional[Union[OptionalHTTPServer, OptionalSMTPServer]] assert dcargs.cli( - OptionalSubparser, args=["--x", "1", "optional-http-server", "--y", "3"] + OptionalSubparser, args=["--x", "1", "optional-http-server", "--bc.y", "3"] ) == OptionalSubparser(x=1, bc=OptionalHTTPServer(y=3)) assert dcargs.cli( - OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--z", "3"] + OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--bc.z", "3"] ) == OptionalSubparser(x=1, bc=OptionalSMTPServer(z=3)) assert dcargs.cli(OptionalSubparser, args=["--x", "1"]) == OptionalSubparser( x=1, bc=None @@ -280,12 +282,12 @@ class OptionalSubparser: with pytest.raises(SystemExit): # Wrong field. dcargs.cli( - OptionalSubparser, args=["--x", "1", "optional-http-server", "--z", "3"] + OptionalSubparser, args=["--x", "1", "optional-http-server", "--bc.z", "3"] ) with pytest.raises(SystemExit): # Wrong field. dcargs.cli( - OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--y", "3"] + OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--bc.y", "3"] ) @@ -315,3 +317,216 @@ class DefaultFactoryPostInitArgs: == dcargs.cli(DefaultFactoryPostInitArgs, args=["--inner.x", "5"]).inner == DataclassWithDynamicDefault(x=5, y=5) ) + + +def test_multiple_subparsers(): + @dataclasses.dataclass + class Subcommand1: + x: int = 0 + + @dataclasses.dataclass + class Subcommand2: + y: int = 1 + + @dataclasses.dataclass + class Subcommand3: + z: int = 2 + + @dataclasses.dataclass + class MultipleSubparsers: + a: Union[Subcommand1, Subcommand2, Subcommand3] + b: Union[Subcommand1, Subcommand2, Subcommand3] + c: Union[Subcommand1, Subcommand2, Subcommand3] + + with pytest.raises(SystemExit): + dcargs.cli(MultipleSubparsers, args=[]) + + assert dcargs.cli( + MultipleSubparsers, args="subcommand1 subcommand2 subcommand3".split(" ") + ) == MultipleSubparsers(Subcommand1(), Subcommand2(), Subcommand3()) + + assert dcargs.cli( + MultipleSubparsers, + args="subcommand1 --a.x 5 subcommand2 --b.y 7 subcommand3 --c.z 3".split(" "), + ) == MultipleSubparsers(Subcommand1(x=5), Subcommand2(y=7), Subcommand3(z=3)) + + assert dcargs.cli( + MultipleSubparsers, + args="subcommand2 --a.y 5 subcommand1 --b.x 7 subcommand3 --c.z 3".split(" "), + ) == MultipleSubparsers(Subcommand2(y=5), Subcommand1(x=7), Subcommand3(z=3)) + + assert dcargs.cli( + MultipleSubparsers, + args="subcommand3 --a.z 5 subcommand1 --b.x 7 subcommand3 --c.z 3".split(" "), + ) == MultipleSubparsers(Subcommand3(z=5), Subcommand1(x=7), Subcommand3(z=3)) + + +def test_multiple_subparsers_with_default(): + @dataclasses.dataclass(frozen=True) + class Subcommand1: + x: int = 0 + + @dataclasses.dataclass(frozen=True) + class Subcommand2: + y: int = 1 + + @dataclasses.dataclass(frozen=True) + class Subcommand3: + z: int = 2 + + @dataclasses.dataclass + class MultipleSubparsers: + a: Union[Subcommand1, Subcommand2, Subcommand3] = Subcommand1(dcargs.MISSING) + b: Union[Subcommand1, Subcommand2, Subcommand3] = Subcommand2(7) + c: Union[Subcommand1, Subcommand2, Subcommand3] = Subcommand3(3) + + with pytest.raises(SystemExit): + dcargs.cli( + MultipleSubparsers, + args=[], + ) + + assert dcargs.cli( + MultipleSubparsers, + args=["subcommand1", "--a.x", "5"], + ) == MultipleSubparsers(Subcommand1(x=5), Subcommand2(y=7), Subcommand3(z=3)) + + assert dcargs.cli( + MultipleSubparsers, + args="subcommand1 --a.x 3".split(" "), + ) == MultipleSubparsers(Subcommand1(x=3), Subcommand2(y=7), Subcommand3(z=3)) + + with pytest.raises(SystemExit): + dcargs.cli( + MultipleSubparsers, + args=[], + default_instance=MultipleSubparsers( + Subcommand1(), + Subcommand2(), + Subcommand3(dcargs.MISSING), + ), + ) + with pytest.raises(SystemExit): + dcargs.cli( + MultipleSubparsers, + args=[ + "subcommand1", + ], + default_instance=MultipleSubparsers( + Subcommand1(), + Subcommand2(), + Subcommand3(dcargs.MISSING), + ), + ) + with pytest.raises(SystemExit): + dcargs.cli( + MultipleSubparsers, + args=["subcommand1", "subcommand2"], + default_instance=MultipleSubparsers( + Subcommand1(), + Subcommand2(), + Subcommand3(dcargs.MISSING), + ), + ) + with pytest.raises(SystemExit): + dcargs.cli( + MultipleSubparsers, + args=["subcommand1", "subcommand2", "subcommand3"], + default_instance=MultipleSubparsers( + Subcommand1(), + Subcommand2(), + Subcommand3(dcargs.MISSING), + ), + ) + assert dcargs.cli( + MultipleSubparsers, + args=["subcommand1", "subcommand2", "subcommand3", "--c.z", "3"], + default_instance=MultipleSubparsers( + Subcommand1(), + Subcommand2(), + Subcommand3(dcargs.MISSING), + ), + ) == MultipleSubparsers(Subcommand1(x=0), Subcommand2(y=1), Subcommand3(z=3)) + assert dcargs.cli( + MultipleSubparsers, + args=["subcommand1", "subcommand2", "subcommand2"], + default_instance=MultipleSubparsers( + Subcommand1(), + Subcommand2(), + Subcommand3(dcargs.MISSING), + ), + ) == MultipleSubparsers(Subcommand1(x=0), Subcommand2(y=1), Subcommand2(y=1)) + + +def test_nested_subparsers_with_default(): + @dataclasses.dataclass(frozen=True) + class Subcommand1: + x: int = 0 + + @dataclasses.dataclass(frozen=True) + class Subcommand3: + z: int = 2 + + @dataclasses.dataclass(frozen=True) + class Subcommand2: + y: Union[Subcommand1, Subcommand3] + + @dataclasses.dataclass(frozen=True) + class MultipleSubparsers: + a: Union[Subcommand1, Subcommand2] = Subcommand2(Subcommand1(dcargs.MISSING)) + + with pytest.raises(SystemExit): + dcargs.cli(MultipleSubparsers, args=[]) + with pytest.raises(SystemExit): + dcargs.cli(MultipleSubparsers, args=["subcommand2"]) + + assert dcargs.cli( + MultipleSubparsers, args="subcommand1 --a.x 3".split(" ") + ) == MultipleSubparsers(Subcommand1(3)) + assert dcargs.cli( + MultipleSubparsers, args="subcommand2 subcommand3 --a.y.z 2".split(" ") + ) == MultipleSubparsers(Subcommand2(Subcommand3())) + assert dcargs.cli( + MultipleSubparsers, args="subcommand2 subcommand3 --a.y.z 7".split(" ") + ) == MultipleSubparsers(Subcommand2(Subcommand3(7))) + assert dcargs.cli( + MultipleSubparsers, args="subcommand2 subcommand1 --a.y.x 7".split(" ") + ) == MultipleSubparsers(Subcommand2(Subcommand1(7))) + + +def test_nested_subparsers_multiple(): + @dataclasses.dataclass(frozen=True) + class Subcommand1: + x: int = 0 + + @dataclasses.dataclass(frozen=True) + class Subcommand3: + z: int = 2 + + @dataclasses.dataclass(frozen=True) + class Subcommand2: + y: Union[Subcommand1, Subcommand3] + + @dataclasses.dataclass(frozen=True) + class MultipleSubparsers: + a: Union[Subcommand1, Subcommand2] + b: Union[Subcommand1, Subcommand2] + + with pytest.raises(SystemExit): + dcargs.cli(MultipleSubparsers, args=[]) + assert dcargs.cli( + MultipleSubparsers, args="subcommand1 subcommand1".split(" ") + ) == MultipleSubparsers(Subcommand1(), Subcommand1()) + assert dcargs.cli( + MultipleSubparsers, args="subcommand1 subcommand2 subcommand1".split(" ") + ) == MultipleSubparsers(Subcommand1(), Subcommand2(Subcommand1())) + assert dcargs.cli( + MultipleSubparsers, + args="subcommand2 subcommand1 subcommand2 subcommand1".split(" "), + ) == MultipleSubparsers(Subcommand2(Subcommand1()), Subcommand2(Subcommand1())) + assert dcargs.cli( + MultipleSubparsers, + args="subcommand2 subcommand1 --a.y.x 3 subcommand2 subcommand1 --b.y.x 7".split( + " " + ), + ) == MultipleSubparsers(Subcommand2(Subcommand1(3)), Subcommand2(Subcommand1(7))) From 2fd96ab6e0893347808cb588e07c4ceb185c3eeb Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 11 Jul 2022 11:46:31 -0700 Subject: [PATCH 051/491] Add `avoid_subparsers` option --- README.md | 68 ++++++++++++++++++++++++++++++------- dcargs/_calling.py | 17 +++++++--- dcargs/_cli.py | 12 ++++++- dcargs/_parsers.py | 28 ++++++++++++--- examples/06_base_configs.py | 26 +++++++++++++- tests/test_nested.py | 59 ++++++++++++++++++++++++++++++-- 6 files changed, 185 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index bb47a994e..ef75f0c13 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,10 @@ _`f`_, which may be a function, class, or dataclass: dcargs.cli( f: Callable[..., T], *, - description: Optional[str]=None, - args: Optional[Sequence[str]]=None, - default_instance: Optional[T]=None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + default_instance: Optional[T] = None, + avoid_subparsers: bool = False, ) -> T ``` @@ -83,6 +84,8 @@ Keyword Args: if `T` is a dataclass, TypedDict, or NamedTuple. Helpful for merging CLI arguments with values loaded from elsewhere. (for example, a config object loaded from a yaml file) + avoid_subparsers: Avoid creating a subparser when defaults are provided for + unions over nested types. Generates cleaner but less expressive CLIs. Returns: The output of `f(...)`. @@ -567,16 +570,34 @@ one of multiple possible base configurations, and then use the CLI to either ove ```python import dataclasses import os -from typing import Literal +from typing import Literal, Tuple, Union import dcargs +@dataclasses.dataclass +class AdamOptimizer: + # Adam learning rate. + learning_rate: float = 1e-3 + + # Moving average parameters. + betas: Tuple[float, float] = (0.9, 0.999) + + +@dataclasses.dataclass +class SgdOptimizer: + # SGD learning rate. + learning_rate: float = 3e-4 + + @dataclasses.dataclass(frozen=True) class ExperimentConfig: # Dataset to run experiment on. dataset: Literal["mnist", "imagenet-50"] + # Optimizer parameters. + optimizer: Union[AdamOptimizer, SgdOptimizer] + # Model size. num_layers: int units: int @@ -598,6 +619,7 @@ class ExperimentConfig: base_config_library = { "small": ExperimentConfig( dataset="mnist", + optimizer=SgdOptimizer(), batch_size=2048, num_layers=4, units=64, @@ -608,6 +630,7 @@ base_config_library = { ), "big": ExperimentConfig( dataset="imagenet-50", + optimizer=AdamOptimizer(), batch_size=32, num_layers=8, units=256, @@ -629,6 +652,10 @@ if __name__ == "__main__": config = dcargs.cli( ExperimentConfig, default_instance=base_config, + # `avoid_subparsers` will avoid making a subparser for unions when a default is + # provided; in this case, it makes our CLI less expressive (cannot switch + # away from the base optimizer types), but easier to use. + avoid_subparsers=True, ) print(config) ``` @@ -640,8 +667,9 @@ if __name__ == "__main__": 
 $ BASE_CONFIG=small python ./06_base_configs.py --help
 usage: 06_base_configs.py [-h] [--dataset {mnist,imagenet-50}]
-                          [--num-layers INT] [--units INT] [--batch-size INT]
-                          [--train-steps INT] --seed INT
+                          [--optimizer.learning-rate FLOAT] [--num-layers INT]
+                          [--units INT] [--batch-size INT] [--train-steps INT]
+                          --seed INT
 
 required arguments:
   --seed INT            Random seed. This is helpful for making sure that our experiments are all
@@ -654,19 +682,27 @@ optional arguments:
   --num-layers INT      Model size. (default: 4)
   --units INT           Model size. (default: 64)
   --batch-size INT      Batch size. (default: 2048)
-  --train-steps INT     Total number of training steps. (default: 30000)
+  --train-steps INT     Total number of training steps. (default: 30000)
+
+optional optimizer arguments:
+  Optimizer parameters.
+
+  --optimizer.learning-rate FLOAT
+                        SGD learning rate. (default: 0.0003)
 
 $ BASE_CONFIG=small python ./06_base_configs.py --seed 94720
-ExperimentConfig(dataset='mnist', num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720)
+ExperimentConfig(dataset='mnist', optimizer=SgdOptimizer(learning_rate=0.0003), num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720)
 
 $ BASE_CONFIG=big python ./06_base_configs.py --help
 usage: 06_base_configs.py [-h] [--dataset {mnist,imagenet-50}]
-                          [--num-layers INT] [--units INT] [--batch-size INT]
-                          [--train-steps INT] --seed INT
+                          [--optimizer.learning-rate FLOAT]
+                          [--optimizer.betas FLOAT FLOAT] [--num-layers INT]
+                          [--units INT] [--batch-size INT] [--train-steps INT]
+                          --seed INT
 
 required arguments:
   --seed INT            Random seed. This is helpful for making sure that our experiments are all
@@ -679,12 +715,20 @@ optional arguments:
   --num-layers INT      Model size. (default: 8)
   --units INT           Model size. (default: 256)
   --batch-size INT      Batch size. (default: 32)
-  --train-steps INT     Total number of training steps. (default: 100000)
+  --train-steps INT     Total number of training steps. (default: 100000)
+
+optional optimizer arguments:
+  Optimizer parameters.
+
+  --optimizer.learning-rate FLOAT
+                        Adam learning rate. (default: 0.001)
+  --optimizer.betas FLOAT FLOAT
+                        Moving average parameters. (default: 0.9 0.999)
 
 $ BASE_CONFIG=big python ./06_base_configs.py --seed 94720
-ExperimentConfig(dataset='imagenet-50', num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720)
+ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learning_rate=0.001, betas=(0.9, 0.999)), num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720)
diff --git a/dcargs/_calling.py b/dcargs/_calling.py index c1aa66c2e..70bed8b04 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -3,9 +3,9 @@ from __future__ import annotations -from typing import Any, Callable, Dict, List, Set, Tuple, TypeVar +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypeVar, Union -from typing_extensions import get_args +from typing_extensions import get_args, get_origin from . import _fields, _parsers, _resolver, _strings @@ -21,8 +21,10 @@ class InstantiationError(Exception): def call_from_args( f: Callable[..., T], parser_definition: _parsers.ParserSpecification, + default_instance: Optional[T], value_from_arg: Dict[str, Any], - field_name_prefix: str = "", + field_name_prefix: str, + avoid_subparsers: bool, ) -> Tuple[T, Set[str]]: """Call `f` with arguments specified by a dictionary of values from argparse. @@ -46,7 +48,7 @@ def get_value_from_arg(arg: str) -> Any: for arg in parser_definition.args: arg_from_prefixed_field_name[arg.prefix + arg.field.name] = arg - for field in _fields.field_list_from_callable(f, default_instance=None): # type: ignore + for field in _fields.field_list_from_callable(f, default_instance=default_instance): # type: ignore value: Any prefixed_field_name = field_name_prefix + field.name @@ -74,11 +76,16 @@ def get_value_from_arg(arg: str) -> Any: in parser_definition.helptext_from_nested_class_field_name ): # Nested callable. + if get_origin(field_type) is Union: + assert avoid_subparsers + field_type = type(field.default) value, consumed_keywords_child = call_from_args( field_type, parser_definition, + field.default, value_from_arg, field_name_prefix=prefixed_field_name + _strings.NESTED_FIELD_DELIMETER, + avoid_subparsers=avoid_subparsers, ) consumed_keywords |= consumed_keywords_child else: @@ -127,9 +134,11 @@ def get_value_from_arg(arg: str) -> Any: parser_definition.subparsers_from_name[field.name].parser_from_name[ subparser_name ], + field.default if type(field.default) is chosen_f else None, value_from_arg, field_name_prefix=prefixed_field_name + _strings.NESTED_FIELD_DELIMETER, + avoid_subparsers=avoid_subparsers, ) consumed_keywords |= consumed_keywords_child diff --git a/dcargs/_cli.py b/dcargs/_cli.py index eea3d58c4..2a9867674 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -31,6 +31,7 @@ def cli( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[T] = None, + avoid_subparsers: bool = False, ) -> T: """Call `f(...)`, with arguments populated from an automatically generated CLI interface. @@ -78,6 +79,8 @@ def cli( if `T` is a dataclass, TypedDict, or NamedTuple. Helpful for merging CLI arguments with values loaded from elsewhere. (for example, a config object loaded from a yaml file) + avoid_subparsers: Avoid creating a subparser when defaults are provided for + unions over nested types. Generates cleaner but less expressive CLIs. Returns: The output of `f(...)`. @@ -90,6 +93,8 @@ def cli( parent_classes=set(), # Used for recursive calls. parent_type_from_typevar=None, # Used for recursive calls. default_instance=default_instance, # Overrides for default values. + prefix="", # Used for recursive calls. + avoid_subparsers=avoid_subparsers, ) # Parse using argparse! @@ -102,7 +107,12 @@ def cli( try: # Attempt to call `f` using whatever was passed in. out, consumed_keywords = _calling.call_from_args( - f, parser_definition, value_from_arg + f, + parser_definition, + default_instance, + value_from_arg, + field_name_prefix="", + avoid_subparsers=avoid_subparsers, ) except _calling.InstantiationError as e: # Emulate argparse's error behavior when invalid arguments are passed in. diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index ea454a143..bbb9e9936 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -96,7 +96,8 @@ def from_callable( parent_classes: Set[Type], parent_type_from_typevar: Optional[Dict[TypeVar, Type]], default_instance: Optional[T], - prefix: str = "", + prefix: str, + avoid_subparsers: bool, ) -> ParserSpecification: """Create a parser definition from a callable.""" @@ -150,10 +151,22 @@ def from_callable( type_from_typevar=type_from_typevar, parent_classes=parent_classes, prefix=prefix + field.name + _strings.NESTED_FIELD_DELIMETER, + avoid_subparsers=avoid_subparsers, ) if subparsers_attempt is not None: - subparsers_from_name[subparsers_attempt.name] = subparsers_attempt - continue + if ( + not avoid_subparsers + # Required subparsers => must create a subparser. + or subparsers_attempt.required + # If the output can be None, the only way to specify whether this is + # or isn't None is with a subparser. + or subparsers_attempt.can_be_none + ): + subparsers_from_name[subparsers_attempt.name] = subparsers_attempt + continue + else: + field = dataclasses.replace(field, typ=type(field.default)) + assert _is_possibly_nested_type(field.typ) # (2) Handle nested callables. if _is_possibly_nested_type(field.typ): @@ -164,6 +177,7 @@ def from_callable( parent_type_from_typevar=type_from_typevar, default_instance=field.default, prefix=prefix + field.name + _strings.NESTED_FIELD_DELIMETER, + avoid_subparsers=avoid_subparsers, ) args.extend(nested_parser.args) @@ -332,6 +346,7 @@ class SubparsersSpecification: parser_from_name: Dict[str, ParserSpecification] required: bool default_instance: Optional[Any] + can_be_none: bool # If underlying type is Optional[Something]. @staticmethod def from_field( @@ -339,6 +354,7 @@ def from_field( type_from_typevar: Dict[TypeVar, Type], parent_classes: Set[Type], prefix: str, + avoid_subparsers: bool, ) -> Optional[SubparsersSpecification]: # Union of classes should create subparsers. if get_origin(field.typ) is not Union: @@ -362,12 +378,13 @@ def from_field( if isinstance(field.default, _resolver.unwrap_origin(option)) else None, prefix=prefix, + avoid_subparsers=avoid_subparsers, ) # Optional if: type hint is Optional[], or a default instance is provided. required = True - if options != options_no_none: - # Optional[] type. + can_be_none = options != options_no_none # Optional[] type. + if can_be_none: required = False elif field.default is not None: required = False @@ -404,4 +421,5 @@ def from_field( parser_from_name=parser_from_name, required=required, default_instance=field.default, + can_be_none=can_be_none, ) diff --git a/examples/06_base_configs.py b/examples/06_base_configs.py index 608dc97a2..8ec9bdf15 100644 --- a/examples/06_base_configs.py +++ b/examples/06_base_configs.py @@ -11,16 +11,34 @@ import dataclasses import os -from typing import Literal +from typing import Literal, Tuple, Union import dcargs +@dataclasses.dataclass +class AdamOptimizer: + # Adam learning rate. + learning_rate: float = 1e-3 + + # Moving average parameters. + betas: Tuple[float, float] = (0.9, 0.999) + + +@dataclasses.dataclass +class SgdOptimizer: + # SGD learning rate. + learning_rate: float = 3e-4 + + @dataclasses.dataclass(frozen=True) class ExperimentConfig: # Dataset to run experiment on. dataset: Literal["mnist", "imagenet-50"] + # Optimizer parameters. + optimizer: Union[AdamOptimizer, SgdOptimizer] + # Model size. num_layers: int units: int @@ -42,6 +60,7 @@ class ExperimentConfig: base_config_library = { "small": ExperimentConfig( dataset="mnist", + optimizer=SgdOptimizer(), batch_size=2048, num_layers=4, units=64, @@ -52,6 +71,7 @@ class ExperimentConfig: ), "big": ExperimentConfig( dataset="imagenet-50", + optimizer=AdamOptimizer(), batch_size=32, num_layers=8, units=256, @@ -73,5 +93,9 @@ class ExperimentConfig: config = dcargs.cli( ExperimentConfig, default_instance=base_config, + # `avoid_subparsers` will avoid making a subparser for unions when a default is + # provided; in this case, it makes our CLI less expressive (cannot switch + # away from the base optimizer types), but easier to use. + avoid_subparsers=True, ) print(config) diff --git a/tests/test_nested.py b/tests/test_nested.py index 0871cbdec..069588473 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -255,6 +255,59 @@ class DefaultInstanceSubparser: dcargs.cli(DefaultInstanceSubparser, args=["--x", "1", "c", "--bc.y", "3"]) +def test_avoid_subparser_with_default_instance(): + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + + assert ( + dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "default-instance-http-server", "--bc.y", "5"], + ) + == dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "--bc.y", "5"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=3) + ), + avoid_subparsers=True, + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) + ) + assert dcargs.cli( + DefaultInstanceSubparser, + args=["default-instance-smtp-server", "--bc.z", "3"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=5) + ), + ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) + assert ( + dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "default-instance-http-server", "--bc.y", "8"], + ) + == dcargs.cli( + DefaultInstanceSubparser, + args=["--bc.y", "8"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=7) + ), + avoid_subparsers=True, + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + ) + + def test_optional_subparser(): @dataclasses.dataclass class OptionalHTTPServer: @@ -526,7 +579,9 @@ class MultipleSubparsers: ) == MultipleSubparsers(Subcommand2(Subcommand1()), Subcommand2(Subcommand1())) assert dcargs.cli( MultipleSubparsers, - args="subcommand2 subcommand1 --a.y.x 3 subcommand2 subcommand1 --b.y.x 7".split( - " " + args=( + "subcommand2 subcommand1 --a.y.x 3 subcommand2 subcommand1 --b.y.x 7".split( + " " + ) ), ) == MultipleSubparsers(Subcommand2(Subcommand1(3)), Subcommand2(Subcommand1(7))) From 5ba6fcfb89504243803b9ce7301518216c923a1a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 12 Jul 2022 03:19:00 -0700 Subject: [PATCH 052/491] Expand support for optionals --- README.md | 8 ++++---- dcargs/_instantiators.py | 24 ++++++++++-------------- examples/06_base_configs.py | 4 ++-- tests/test_collections.py | 13 +++++++++++++ tests/test_errors.py | 18 +----------------- tests/test_helptext.py | 4 ++-- 6 files changed, 32 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index ef75f0c13..8bb1a07a5 100644 --- a/README.md +++ b/README.md @@ -312,7 +312,7 @@ if __name__ == "__main__": usage: 03_enums_and_containers.py [-h] --dataset-sources PATH [PATH ...] [--image-dimensions INT INT] [--optimizer-type {ADAM,SGD}] - [--checkpoint-interval INT] + [--checkpoint-interval (INT | None)] required arguments: --dataset-sources PATH [PATH ...] @@ -324,7 +324,7 @@ optional arguments: Height and width of some image data. (default: 32 32) --optimizer-type {ADAM,SGD} Gradient-based optimizer to use. (default: ADAM) - --checkpoint-interval INT + --checkpoint-interval (INT | None) Interval to save checkpoints at. (default: None) @@ -653,8 +653,8 @@ if __name__ == "__main__": ExperimentConfig, default_instance=base_config, # `avoid_subparsers` will avoid making a subparser for unions when a default is - # provided; in this case, it makes our CLI less expressive (cannot switch - # away from the base optimizer types), but easier to use. + # provided; in this case, it simplifies our CLI but makes it less expressive + # (cannot switch away from the base optimizer types). avoid_subparsers=True, ) print(config) diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index ee47ed7c4..fbf6130e6 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -203,7 +203,6 @@ def _instantiator_from_container_type( contained_type, type_from_typevar, allow_sequences=False, - allow_optional=False, ) return lambda strings: container_type( [make(x) for x in strings] @@ -230,7 +229,6 @@ def _instantiator_from_container_type( contained_type, type_from_typevar, allow_sequences=False, - allow_optional=False, ) return lambda strings: tuple( [make(x) for x in strings] @@ -249,7 +247,6 @@ def _instantiator_from_container_type( t, type_from_typevar, allow_sequences=False, - allow_optional=False, ) instantiators.append(a) metas.append(b) @@ -279,13 +276,19 @@ def _instantiator_from_container_type( " (Union[T, None])" ) (typ,) = options - {type(None)} - instantiator, metadata = _instantiator_from_type_inner( + inner_instantiator, metadata = _instantiator_from_type_inner( typ, type_from_typevar, allow_sequences=True, - allow_optional=False, ) - return instantiator, dataclasses.replace(metadata, is_optional=True) + instantiator = ( + lambda string: None if string == "None" else inner_instantiator(string) + ) + return instantiator, dataclasses.replace( + metadata, + metavar=f"({metadata.metavar} | None)", + is_optional=True, + ) # Literals. if type_origin is Literal: @@ -301,7 +304,6 @@ def _instantiator_from_container_type( contained_type, type_from_typevar, allow_sequences=False, - allow_optional=False, ) assert ( # Choices provided by the contained type @@ -312,6 +314,7 @@ def _instantiator_from_container_type( metadata, choices=tuple(map(str, choices)), metavar="{" + ",".join(map(str, choices)) + "}", + is_optional=False, ) # Dictionaries. @@ -321,13 +324,11 @@ def _instantiator_from_container_type( key_type, type_from_typevar, allow_sequences=False, - allow_optional=False, ) val_instantiator, val_metadata = _instantiator_from_type_inner( val_type, type_from_typevar, allow_sequences=False, - allow_optional=False, ) def dict_instantiator(strings: List[str]) -> Any: @@ -365,7 +366,6 @@ def _instantiator_from_type_inner( typ: Type, type_from_typevar: Dict[TypeVar, Type], allow_sequences: Literal[False], - allow_optional: bool, ) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: ... @@ -375,7 +375,6 @@ def _instantiator_from_type_inner( typ: Type, type_from_typevar: Dict[TypeVar, Type], allow_sequences: Literal[True], - allow_optional: bool, ) -> Tuple[Instantiator, InstantiatorMetadata]: ... @@ -384,13 +383,10 @@ def _instantiator_from_type_inner( typ: Type, type_from_typevar: Dict[TypeVar, Type], allow_sequences: bool, - allow_optional: bool, ) -> Tuple[Instantiator, InstantiatorMetadata]: """Thin wrapper over instantiator_from_type, with some extra asserts for catching errors.""" out = instantiator_from_type(typ, type_from_typevar) if not allow_sequences and out[1].nargs is not None: raise UnsupportedTypeAnnotationError("Nested sequence types are not supported!") - if not allow_optional and out[1].is_optional: - raise UnsupportedTypeAnnotationError("Nested optional types are not supported!") return out diff --git a/examples/06_base_configs.py b/examples/06_base_configs.py index 8ec9bdf15..60da22687 100644 --- a/examples/06_base_configs.py +++ b/examples/06_base_configs.py @@ -94,8 +94,8 @@ class ExperimentConfig: ExperimentConfig, default_instance=base_config, # `avoid_subparsers` will avoid making a subparser for unions when a default is - # provided; in this case, it makes our CLI less expressive (cannot switch - # away from the base optimizer types), but easier to use. + # provided; in this case, it simplifies our CLI but makes it less expressive + # (cannot switch away from the base optimizer types). avoid_subparsers=True, ) print(config) diff --git a/tests/test_collections.py b/tests/test_collections.py index c06fe2760..23af76ea2 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -240,3 +240,16 @@ class A: with pytest.raises(SystemExit): dcargs.cli(A, args=["--x"]) assert dcargs.cli(A, args=[]) == A(x=None) + + +def test_nested_optional_types(): + """We support "None" as a special-case keyword. (note: this is a bit weird because + Optional[str] might interprete "None" as either a string or an actual `None` + value)""" + + @dataclasses.dataclass + class A: + x: Tuple[Optional[int], ...] + + assert dcargs.cli(A, args=["--x", "0", "1"]) == A((0, 1)) + assert dcargs.cli(A, args=["--x", "0", "None", "1"]) == A((0, None, 1)) diff --git a/tests/test_errors.py b/tests/test_errors.py index 1276d3644..40d80eb91 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Generic, List, Optional, Tuple, TypeVar, Union +from typing import Generic, List, Tuple, TypeVar, Union import pytest from typing_extensions import Literal @@ -44,22 +44,6 @@ class A: dcargs.cli(A, args=["--x", "0", "1"]) -def test_nested_optional_types(): - """Unclear how to handle optionals nested in other types, so we don't support - them. - - In the future, we might support "None" as a special-case keyword. But this is a bit - weird because Optional[str] might interprete "None" as either a string or an actual - `None` value.""" - - @dataclasses.dataclass - class A: - x: Tuple[Optional[int], ...] - - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(A, args=["--x", "0", "1"]) - - def test_unsupported_union(): def main(x: Union[int, str]) -> None: return diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 138459ad0..440c0e57d 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -195,7 +195,7 @@ class Config: with contextlib.redirect_stdout(f): dcargs.cli(Config, args=["--help"]) helptext = f.getvalue() - assert " --x INT An optional variable. (default: None)\n" in helptext + assert " --x (INT | None) An optional variable. (default: None)\n" in helptext def test_helptext_hard_bool(): @@ -370,7 +370,7 @@ class OptionalLiteralHelptext: with contextlib.redirect_stdout(f): dcargs.cli(OptionalLiteralHelptext, args=["--help"]) helptext = f.getvalue() - assert "--x {1,2,3} A number. (default: None)\n" in helptext + assert "--x ({1,2,3} | None) A number. (default: None)\n" in helptext def test_multiple_subparsers_helptext(): From 72184f1675c6fb893122cb8587a22c42656c8044 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 12 Jul 2022 13:12:14 -0700 Subject: [PATCH 053/491] Optional[] tests, nits --- README.md | 12 ++++++------ dcargs/_instantiators.py | 2 +- examples/03_enums_and_containers.py | 8 ++++---- tests/test_helptext.py | 28 ++++++++++++++++++++++++++-- 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 8bb1a07a5..4be2ddb12 100644 --- a/README.md +++ b/README.md @@ -260,8 +260,7 @@ Args(field1='hello', field2=3, flag=True)
We can generate argument parsers from more advanced type annotations, like enums and -tuple types. For collections, we only showcase `Tuple` here, but `List`, `Sequence`, -`Set`, `Dict`, etc are all supported as well. +tuple types. **Code ([link](examples/03_enums_and_containers.py)):** @@ -281,11 +280,12 @@ class OptimizerType(enum.Enum): @dataclasses.dataclass(frozen=True) class TrainConfig: - # Example of a variable-length tuple: + # Example of a variable-length tuple. `typing.List`, `typing.Sequence`, + # `typing.Set`, `typing.Dict`, etc are all supported as well. dataset_sources: Tuple[pathlib.Path, ...] """Paths to load training data from. This can be multiple!""" - # Fixed-length tuples are also okay: + # Fixed-length tuples are also okay. image_dimensions: Tuple[int, int] = (32, 32) """Height and width of some image data.""" @@ -312,7 +312,7 @@ if __name__ == "__main__": usage: 03_enums_and_containers.py [-h] --dataset-sources PATH [PATH ...] [--image-dimensions INT INT] [--optimizer-type {ADAM,SGD}] - [--checkpoint-interval (INT | None)] + [--checkpoint-interval (INT|None)] required arguments: --dataset-sources PATH [PATH ...] @@ -324,7 +324,7 @@ optional arguments: Height and width of some image data. (default: 32 32) --optimizer-type {ADAM,SGD} Gradient-based optimizer to use. (default: ADAM) - --checkpoint-interval (INT | None) + --checkpoint-interval (INT|None) Interval to save checkpoints at. (default: None) diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index fbf6130e6..39ca1b4bc 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -286,7 +286,7 @@ def _instantiator_from_container_type( ) return instantiator, dataclasses.replace( metadata, - metavar=f"({metadata.metavar} | None)", + metavar=f"({metadata.metavar}|None)", is_optional=True, ) diff --git a/examples/03_enums_and_containers.py b/examples/03_enums_and_containers.py index 1bde39d09..946ad4664 100644 --- a/examples/03_enums_and_containers.py +++ b/examples/03_enums_and_containers.py @@ -1,6 +1,5 @@ """We can generate argument parsers from more advanced type annotations, like enums and -tuple types. For collections, we only showcase `Tuple` here, but `List`, `Sequence`, -`Set`, `Dict`, etc are all supported as well. +tuple types. Usage: `python ./03_enums_and_containers.py --help` @@ -23,11 +22,12 @@ class OptimizerType(enum.Enum): @dataclasses.dataclass(frozen=True) class TrainConfig: - # Example of a variable-length tuple: + # Example of a variable-length tuple. `typing.List`, `typing.Sequence`, + # `typing.Set`, `typing.Dict`, etc are all supported as well. dataset_sources: Tuple[pathlib.Path, ...] """Paths to load training data from. This can be multiple!""" - # Fixed-length tuples are also okay: + # Fixed-length tuples are also okay. image_dimensions: Tuple[int, int] = (32, 32) """Height and width of some image data.""" diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 440c0e57d..48f74ea6b 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -195,7 +195,7 @@ class Config: with contextlib.redirect_stdout(f): dcargs.cli(Config, args=["--help"]) helptext = f.getvalue() - assert " --x (INT | None) An optional variable. (default: None)\n" in helptext + assert " --x (INT|None) An optional variable. (default: None)\n" in helptext def test_helptext_hard_bool(): @@ -370,7 +370,7 @@ class OptionalLiteralHelptext: with contextlib.redirect_stdout(f): dcargs.cli(OptionalLiteralHelptext, args=["--help"]) helptext = f.getvalue() - assert "--x ({1,2,3} | None) A number. (default: None)\n" in helptext + assert "--x ({1,2,3}|None) A number. (default: None)\n" in helptext def test_multiple_subparsers_helptext(): @@ -418,3 +418,27 @@ class MultipleSubparsers: assert "Field a description." not in helptext assert "Field b description." not in helptext assert "Field c description. (default: subcommand3)" in helptext + + +def test_optional_helptext(): + @dataclasses.dataclass + class OptionalHelptext: + """This docstring should be printed as a description.""" + + x: Optional[int] # Documentation 1 + + # Documentation 2 + y: List[Optional[int]] + + z: Optional[int] = 3 + """Documentation 3""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + dcargs.cli(OptionalHelptext, args=["--help"]) + helptext = f.getvalue() + assert cast(str, OptionalHelptext.__doc__) in helptext + assert "[--x (INT|None)]" in helptext + assert "--y (INT|None) [(INT|None) ...]\n" in helptext + assert "[--z (INT|None)]\n" in helptext From 8de6ca0c10e605024ff26845fc01c6f85ec07e20 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 13 Jul 2022 00:58:40 -0700 Subject: [PATCH 054/491] Support `typing.Union` recursively --- README.md | 100 ++++--- dcargs/_arguments.py | 4 + dcargs/_cli.py | 1 + dcargs/_instantiators.py | 417 ++++++++++++++++----------- dcargs/_parsers.py | 29 +- examples/07_literals.py | 36 --- examples/07_literals_and_unions.py | 40 +++ examples/09_subparsers.py | 2 +- tests/test_collections.py | 18 +- tests/test_dcargs.py | 31 +- tests/test_errors.py | 16 +- tests/test_helptext.py | 10 +- tests/test_positional_ignore_py37.py | 11 +- 13 files changed, 444 insertions(+), 271 deletions(-) delete mode 100644 examples/07_literals.py create mode 100644 examples/07_literals_and_unions.py diff --git a/README.md b/README.md index 4be2ddb12..622b6209a 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ annotations; a broad range of core type annotations are supported... `typing.Tuple[T, ...]`. - `typing.Set[T]`. - `typing.Final[T]` and `typing.Annotated[T]`. + - `typing.Union[T1, T2]`. - Various nested combinations of the above: `Optional[Literal[T]]`, `Final[Optional[Sequence[T]]]`, etc. - Hierarchical structures via nested dataclasses, TypedDict, NamedTuple, @@ -312,7 +313,7 @@ if __name__ == "__main__": usage: 03_enums_and_containers.py [-h] --dataset-sources PATH [PATH ...] [--image-dimensions INT INT] [--optimizer-type {ADAM,SGD}] - [--checkpoint-interval (INT|None)] + [--checkpoint-interval (None|INT)] required arguments: --dataset-sources PATH [PATH ...] @@ -324,7 +325,7 @@ optional arguments: Height and width of some image data. (default: 32 32) --optimizer-type {ADAM,SGD} Gradient-based optimizer to use. (default: ADAM) - --checkpoint-interval (INT|None) + --checkpoint-interval (None|INT) Interval to save checkpoints at. (default: None) @@ -735,18 +736,19 @@ ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learni
-7. Literals +7. Literals And Unions
-`typing.Literal[]` can be used to restrict inputs to a fixed set of choices. +`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices; +`typing.Union[]` can be used to restrict inputs to a fixed set of types. -**Code ([link](examples/07_literals.py)):** +**Code ([link](examples/07_literals_and_unions.py)):** ```python import dataclasses import enum -from typing import Literal +from typing import Literal, Tuple, Union import dcargs @@ -759,15 +761,15 @@ class Color(enum.Enum): @dataclasses.dataclass(frozen=True) class Args: - enum: Color - restricted_enum: Literal[Color.RED, Color.GREEN] - - integer: Literal[0, 1, 2, 3] - string: Literal["red", "green"] - - restricted_enum_with_default: Literal[Color.RED, Color.GREEN] = Color.GREEN - integer_with_default: Literal[0, 1, 2, 3] = 3 - string_with_Default: Literal["red", "green"] = "red" + enum: Color = Color.RED + restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED + integer: Literal[0, 1, 2, 3] = 0 + string: Literal["red", "green"] = "red" + string_or_enum: Union[Literal["red", "green"], Color] = "red" + tuple_of_string_or_enum: Tuple[Union[Literal["red", "green"], Color], ...] = ( + "red", + Color.RED, + ) if __name__ == "__main__": @@ -780,32 +782,45 @@ if __name__ == "__main__": **Example usage:** 
-$ python ./07_literals.py --help
-usage: 07_literals.py [-h] --enum {RED,GREEN,BLUE} --restricted-enum
-                      {RED,GREEN} --integer {0,1,2,3} --string {red,green}
-                      [--restricted-enum-with-default {RED,GREEN}]
-                      [--integer-with-default {0,1,2,3}]
-                      [--string-with-Default {red,green}]
+$ python ./07_literals_and_unions.py --help
+usage: 07_literals_and_unions.py [-h] [--enum {RED,GREEN,BLUE}]
+                                 [--restricted-enum {RED,GREEN}]
+                                 [--integer {0,1,2,3}] [--string {red,green}]
+                                 [--string-or-enum ({red,green}|{RED,GREEN,BLUE})]
+                                 [--tuple-of-string-or-enum ({red,green}|{RED,GREEN,BLUE}) [({red,green}|{RED,GREEN,BLUE}) ...]]
 
-required arguments:
+optional arguments:
+  -h, --help            show this help message and exit
   --enum {RED,GREEN,BLUE}
+                        (default: RED)
   --restricted-enum {RED,GREEN}
-  --integer {0,1,2,3}
-  --string {red,green}
+                        (default: RED)
+  --integer {0,1,2,3}   (default: 0)
+  --string {red,green}  (default: red)
+  --string-or-enum ({red,green}|{RED,GREEN,BLUE})
+                        (default: red)
+  --tuple-of-string-or-enum ({red,green}|{RED,GREEN,BLUE}) [({red,green}|{RED,GREEN,BLUE}) ...]
+                        (default: red RED)
+
-optional arguments: - -h, --help show this help message and exit - --restricted-enum-with-default {RED,GREEN} - (default: GREEN) - --integer-with-default {0,1,2,3} - (default: 3) - --string-with-Default {red,green} - (default: red) +
+$ python ./07_literals_and_unions.py --enum RED --restricted-enum GREEN --integer 3 --string green
+Args(enum=<Color.RED: 1>, restricted_enum=<Color.GREEN: 2>, integer=3, string='green', string_or_enum='red', tuple_of_string_or_enum=('red', <Color.RED: 1>))
 
-$ python ./07_literals.py --enum RED --restricted-enum GREEN --integer 3 --string green
-Args(enum=<Color.RED: 1>, restricted_enum=<Color.GREEN: 2>, integer=3, string='green', restricted_enum_with_default=<Color.GREEN: 2>, integer_with_default=3, string_with_Default='red')
+$ python ./07_literals_and_unions.py --string-or-enum green
+Args(enum=<Color.RED: 1>, restricted_enum=<Color.RED: 1>, integer=0, string='red', string_or_enum='green', tuple_of_string_or_enum=('red', <Color.RED: 1>))
+
+ +
+$ python ./07_literals_and_unions.py --string-or-enum RED
+Args(enum=<Color.RED: 1>, restricted_enum=<Color.RED: 1>, integer=0, string='red', string_or_enum=<Color.RED: 1>, tuple_of_string_or_enum=('red', <Color.RED: 1>))
+
+ +
+$ python ./07_literals_and_unions.py --tuple-of-string-or-enum RED green BLUE
+Args(enum=<Color.RED: 1>, restricted_enum=<Color.RED: 1>, integer=0, string='red', string_or_enum='red', tuple_of_string_or_enum=(<Color.RED: 1>, 'green', <Color.BLUE: 3>))
@@ -958,7 +973,7 @@ class Commit: all: bool = False -def main(cmd: Union[Checkout, Commit] = Checkout("main")) -> None: +def main(cmd: Union[Checkout, Commit]) -> None: print(cmd) @@ -972,15 +987,14 @@ if __name__ == "__main__": 
 $ python ./09_subparsers.py --help
-usage: 09_subparsers.py [-h] [{checkout,commit}] ...
+usage: 09_subparsers.py [-h] {checkout,commit} ...
 
 optional arguments:
-  -h, --help           show this help message and exit
+  -h, --help         show this help message and exit
 
-optional subcommands:
-   (default: checkout)
+subcommands:
 
-  [{checkout,commit}]
+  {checkout,commit}
 
@@ -1006,15 +1020,15 @@ Commit(message='hello', all=True)
 
 
 $ python ./09_subparsers.py checkout --help
-usage: 09_subparsers.py checkout [-h] [--cmd.branch STR]
+usage: 09_subparsers.py checkout [-h] --cmd.branch STR
 
 Checkout a branch.
 
 optional arguments:
   -h, --help        show this help message and exit
 
-optional cmd arguments:
-  --cmd.branch STR  (default: main)
+required cmd arguments:
+  --cmd.branch STR
 

 
 
diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py
index 12ccd5bb0..b027aad1a 100644
--- a/dcargs/_arguments.py
+++ b/dcargs/_arguments.py
@@ -246,6 +246,10 @@ def _rule_positional_special_handling(
     if not arg.field.positional:
         return lowered
 
+    if lowered.nargs is not None and not lowered.required:
+        raise _instantiators.UnsupportedTypeAnnotationError(
+            "Optional sequences are not supported for positional arguments!"
+        )
     return dataclasses.replace(
         lowered,
         dest=None,
diff --git a/dcargs/_cli.py b/dcargs/_cli.py
index 2a9867674..eccdee353 100644
--- a/dcargs/_cli.py
+++ b/dcargs/_cli.py
@@ -57,6 +57,7 @@ def cli(
             `typing.Tuple[T, ...]`.
           - `typing.Set[T]`.
           - `typing.Final[T]` and `typing.Annotated[T]`.
+          - `typing.Union[T1, T2]`.
           - Various nested combinations of the above: `Optional[Literal[T]]`,
             `Final[Optional[Sequence[T]]]`, etc.
         - Hierarchical structures via nested dataclasses, TypedDict, NamedTuple,
diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py
index 39ca1b4bc..4180f3a91 100644
--- a/dcargs/_instantiators.py
+++ b/dcargs/_instantiators.py
@@ -68,6 +68,8 @@
 
 Instantiator = Union[_StandardInstantiator, _SequenceInstantiator, _FlagInstantiator]
 
+NoneType = type(None)
+
 
 @dataclasses.dataclass
 class InstantiatorMetadata:
@@ -116,6 +118,18 @@ def instantiator_from_type(
             is_optional=False,
         )
 
+    if typ is NoneType:
+
+        def instantiator(_unused_string: str) -> None:
+            return None
+
+        return instantiator, InstantiatorMetadata(
+            nargs=None,
+            metavar="None",
+            choices=("None",),
+            is_optional=False,
+        )
+
     # Address container types. If a matching container is found, this will recursively
     # call instantiator_from_type().
     container_out = _instantiator_from_container_type(typ, type_from_typevar)
@@ -173,6 +187,41 @@ def instantiator_from_type(
     )
 
 
+@overload
+def _instantiator_from_type_inner(
+    typ: Type,
+    type_from_typevar: Dict[TypeVar, Type],
+    allow_sequences: Literal[False],
+) -> Tuple[_StandardInstantiator, InstantiatorMetadata]:
+    ...
+
+
+@overload
+def _instantiator_from_type_inner(
+    typ: Type,
+    type_from_typevar: Dict[TypeVar, Type],
+    allow_sequences: Literal[True],
+) -> Tuple[Instantiator, InstantiatorMetadata]:
+    ...
+
+
+def _instantiator_from_type_inner(
+    typ: Type,
+    type_from_typevar: Dict[TypeVar, Type],
+    allow_sequences: bool,
+) -> Tuple[Instantiator, InstantiatorMetadata]:
+    """Thin wrapper over instantiator_from_type, with some extra asserts for catching
+    errors."""
+    out = instantiator_from_type(typ, type_from_typevar)
+    if (
+        not allow_sequences
+        and out[1].nargs is not None
+        and get_origin(typ) is not Union
+    ):
+        raise UnsupportedTypeAnnotationError("Nested sequence types are not supported!")
+    return out
+
+
 def _instantiator_from_container_type(
     typ: Type, type_from_typevar: Dict[TypeVar, Type]
 ) -> Optional[Tuple[Instantiator, InstantiatorMetadata]]:
@@ -188,205 +237,243 @@ def _instantiator_from_container_type(
         contained_type = get_args(typ)[0]
         return instantiator_from_type(contained_type, type_from_typevar)
 
-    # List, tuples, and sequences.
-    if type_origin in (
-        collections.abc.Sequence,  # different from typing.Sequence!
-        list,  # different from typing.List!
-        set,  # different from typing.Set!
-    ):
-        (contained_type,) = get_args(typ)
-        container_type = type_origin
-        if container_type is collections.abc.Sequence:
-            container_type = list
+    for make, matched_origins in {
+        _instantiator_from_list_sequence_or_set: (collections.abc.Sequence, list, set),
+        _instantiator_from_tuple: (tuple,),
+        _instantiator_from_dict: (dict, collections.abc.Mapping),
+        _instantiator_from_union: (Union,),
+        _instantiator_from_literal: (Literal,),
+    }.items():
+        if type_origin in matched_origins:
+            return make(typ, type_from_typevar)
+
+    raise UnsupportedTypeAnnotationError(  # pragma: no cover
+        f"Unsupported type {typ} with origin {type_origin}"
+    )
+
+
+def _instantiator_from_tuple(
+    typ: Type, type_from_typevar: Dict[TypeVar, Type]
+) -> Tuple[Instantiator, InstantiatorMetadata]:
+    types = get_args(typ)
+    typeset = set(types)  # Note that sets are unordered.
+    typeset_no_ellipsis = typeset - {Ellipsis}  # type: ignore
+
+    if typeset_no_ellipsis != typeset:
+        # Ellipsis: variable argument counts. When an ellipsis is used, tuples must
+        # contain only one type.
+        assert len(typeset_no_ellipsis) == 1
+        (contained_type,) = typeset_no_ellipsis
 
         make, inner_meta = _instantiator_from_type_inner(
             contained_type,
             type_from_typevar,
             allow_sequences=False,
         )
-        return lambda strings: container_type(
-            [make(x) for x in strings]
-        ), InstantiatorMetadata(
+        return lambda strings: tuple([make(x) for x in strings]), InstantiatorMetadata(
             nargs="+",
             metavar=inner_meta.metavar,
             choices=inner_meta.choices,
             is_optional=False,
         )
 
-    # Tuples.
-    if type_origin is tuple:
-        types = get_args(typ)
-        typeset = set(types)  # Note that sets are unordered.
-        typeset_no_ellipsis = typeset - {Ellipsis}  # type: ignore
-
-        if typeset_no_ellipsis != typeset:
-            # Ellipsis: variable argument counts. When an ellipsis is used, tuples must
-            # contain only one type.
-            assert len(typeset_no_ellipsis) == 1
-            (contained_type,) = typeset_no_ellipsis
-
-            make, inner_meta = _instantiator_from_type_inner(
-                contained_type,
+    else:
+        instantiators = []
+        metas = []
+        for t in types:
+            a, b = _instantiator_from_type_inner(
+                t,
                 type_from_typevar,
                 allow_sequences=False,
             )
-            return lambda strings: tuple(
-                [make(x) for x in strings]
-            ), InstantiatorMetadata(
-                nargs="+",
-                metavar=inner_meta.metavar,
-                choices=inner_meta.choices,
-                is_optional=False,
-            )
-
-        else:
-            instantiators = []
-            metas = []
-            for t in types:
-                a, b = _instantiator_from_type_inner(
-                    t,
-                    type_from_typevar,
-                    allow_sequences=False,
-                )
-                instantiators.append(a)
-                metas.append(b)
-
-            if len(set(m.choices for m in metas)) > 1:
-                raise UnsupportedTypeAnnotationError(
-                    "Due to constraints in argparse, all choices in fixed-length tuples"
-                    " must match. This restricts mixing enums & literals with other"
-                    " types."
-                )
-            return lambda strings: tuple(
-                make(x) for make, x in zip(instantiators, strings)
-            ), InstantiatorMetadata(
-                nargs=len(types),
-                metavar=tuple(cast(str, m.metavar) for m in metas),
-                choices=metas[0].choices,
-                is_optional=False,
-            )
+            instantiators.append(a)
+            metas.append(b)
 
-    # Optionals.
-    if type_origin is Union:
-        options = set(get_args(typ))
-        if len(options) != 2 or type(None) not in options:
-            # Note that the subparsers logic happens much earlier.
+        if len(set(m.choices for m in metas)) > 1:
             raise UnsupportedTypeAnnotationError(
-                "Union must be either over dataclasses (for subparsers) or Optional"
-                " (Union[T, None])"
+                "Due to constraints in argparse, all choices in fixed-length tuples"
+                " must match. This restricts mixing enums & literals with other"
+                " types."
             )
-        (typ,) = options - {type(None)}
-        inner_instantiator, metadata = _instantiator_from_type_inner(
-            typ,
+        return lambda strings: tuple(
+            make(x) for make, x in zip(instantiators, strings)
+        ), InstantiatorMetadata(
+            nargs=len(types),
+            metavar=tuple(cast(str, m.metavar) for m in metas),
+            choices=metas[0].choices,
+            is_optional=False,
+        )
+
+
+def _instantiator_from_union(
+    typ: Type, type_from_typevar: Dict[TypeVar, Type]
+) -> Tuple[Instantiator, InstantiatorMetadata]:
+    options = list(get_args(typ))
+    if NoneType in options:
+        # Move `None` types to the beginning.
+        # If we have `Optional[str]`, we want this to be parsed as
+        # `Union[NoneType, str]`.
+        options.remove(NoneType)
+        options.insert(0, NoneType)
+
+    # General unions, eg Union[int, bool]. We'll try to convert these from left to
+    # right.
+    instantiators = []
+    metas = []
+    nargs = None
+    first = True
+    for t in options:
+        a, b = _instantiator_from_type_inner(
+            t,
             type_from_typevar,
             allow_sequences=True,
         )
-        instantiator = (
-            lambda string: None if string == "None" else inner_instantiator(string)
-        )
-        return instantiator, dataclasses.replace(
-            metadata,
-            metavar=f"({metadata.metavar}|None)",
-            is_optional=True,
-        )
+        instantiators.append(a)
+        metas.append(b)
+
+        if t is not NoneType:
+            # Enforce that `nargs` is the same for all child types, except for
+            # NoneType.
+            if first:
+                nargs = b.nargs
+                first = False
+            elif nargs != b.nargs:
+                raise UnsupportedTypeAnnotationError(
+                    f"All options in {typ} must expect the same number of CLI"
+                    " arguments. (this is a limitation from argparse's nargs"
+                    " option)"
+                )
 
-    # Literals.
-    if type_origin is Literal:
-        choices = get_args(typ)
-        if not len(set(map(type, choices))) == 1:
-            raise UnsupportedTypeAnnotationError(
-                "All choices in literal must have the same type!"
+    # Set metavar.
+    if nargs is not None and options[0] is NoneType:
+        # For optional types + sequences, the only way to set a field to None will
+        # be to omit its corresponding argument.
+        instantiators.pop(0)
+        metas.pop(0)
+
+    metavar: Union[str, Tuple[str, ...]]
+    if isinstance(metas[0].metavar, str):
+        metavar = "(" + "|".join(map(lambda x: cast(str, x.metavar), metas)) + ")"
+    else:
+        # Do our best to create a reasonable looking metavar.
+        # This is imperfect!
+        assert isinstance(metas[0].metavar, tuple)
+        metavar = tuple(
+            map(
+                lambda metavars: "(" + "|".join(metavars) + ")",
+                zip(*map(lambda m: m.metavar, metas)),
             )
-        contained_type = type(next(iter(choices)))
-        if issubclass(contained_type, enum.Enum):
-            choices = tuple(map(lambda x: x.name, choices))
-        instantiator, metadata = _instantiator_from_type_inner(
-            contained_type,
-            type_from_typevar,
-            allow_sequences=False,
-        )
-        assert (
-            # Choices provided by the contained type
-            metadata.choices is None
-            or len(set(choices) - set(metadata.choices)) == 0
-        )
-        return instantiator, dataclasses.replace(
-            metadata,
-            choices=tuple(map(str, choices)),
-            metavar="{" + ",".join(map(str, choices)) + "}",
-            is_optional=False,
         )
 
-    # Dictionaries.
-    if type_origin in (dict, collections.abc.Mapping):
-        key_type, val_type = get_args(typ)
-        key_instantiator, key_metadata = _instantiator_from_type_inner(
-            key_type,
-            type_from_typevar,
-            allow_sequences=False,
-        )
-        val_instantiator, val_metadata = _instantiator_from_type_inner(
-            val_type,
-            type_from_typevar,
-            allow_sequences=False,
+    def union_instantiator(string_or_strings: Union[str, List[str]]) -> Any:
+        for instantiator, metadata in zip(instantiators, metas):
+            if metadata.choices is None or string_or_strings in metadata.choices:
+                try:
+                    return instantiator(string_or_strings)  # type: ignore
+                except ValueError:
+                    # Failed, try next instantiator.
+                    pass
+        raise ValueError(
+            f"No type in {options} could be instantiated from {string_or_strings}."
         )
 
-        def dict_instantiator(strings: List[str]) -> Any:
-            out = {}
-            if len(strings) % 2 != 0:
-                raise ValueError("incomplete set of key value pairs!")
-            for i in range(len(strings) // 2):
-                k = strings[i * 2]
-                v = strings[i * 2 + 1]
-                if key_metadata.choices is not None and k not in key_metadata.choices:
-                    raise ValueError(
-                        f"invalid choice: {k} (choose from {key_metadata.choices}))"
-                    )
-                if val_metadata.choices is not None and v not in val_metadata.choices:
-                    raise ValueError(
-                        f"invalid choice: {v} (choose from {val_metadata.choices}))"
-                    )
-                out[key_instantiator(k)] = val_instantiator(v)  # type: ignore
-            return out
-
-        return dict_instantiator, InstantiatorMetadata(
-            nargs="+",
-            metavar=f"{key_metadata.metavar} {val_metadata.metavar}",
-            choices=None,
-            is_optional=False,
-        )
+    return union_instantiator, InstantiatorMetadata(
+        nargs=nargs,
+        metavar=metavar,
+        choices=None,
+        is_optional=NoneType in options,
+    )
 
-    raise UnsupportedTypeAnnotationError(  # pragma: no cover
-        f"Unsupported type {typ} with origin {type_origin}"
+
+def _instantiator_from_dict(
+    typ: Type, type_from_typevar: Dict[TypeVar, Type]
+) -> Tuple[Instantiator, InstantiatorMetadata]:
+    key_type, val_type = get_args(typ)
+    key_instantiator, key_metadata = _instantiator_from_type_inner(
+        key_type,
+        type_from_typevar,
+        allow_sequences=False,
+    )
+    val_instantiator, val_metadata = _instantiator_from_type_inner(
+        val_type,
+        type_from_typevar,
+        allow_sequences=False,
     )
 
+    def dict_instantiator(strings: List[str]) -> Any:
+        out = {}
+        if len(strings) % 2 != 0:
+            raise ValueError("incomplete set of key value pairs!")
+        for i in range(len(strings) // 2):
+            k = strings[i * 2]
+            v = strings[i * 2 + 1]
+            if key_metadata.choices is not None and k not in key_metadata.choices:
+                raise ValueError(
+                    f"invalid choice: {k} (choose from {key_metadata.choices}))"
+                )
+            if val_metadata.choices is not None and v not in val_metadata.choices:
+                raise ValueError(
+                    f"invalid choice: {v} (choose from {val_metadata.choices}))"
+                )
+            out[key_instantiator(k)] = val_instantiator(v)  # type: ignore
+        return out
 
-@overload
-def _instantiator_from_type_inner(
-    typ: Type,
-    type_from_typevar: Dict[TypeVar, Type],
-    allow_sequences: Literal[False],
-) -> Tuple[_StandardInstantiator, InstantiatorMetadata]:
-    ...
+    return dict_instantiator, InstantiatorMetadata(
+        nargs="+",
+        metavar=f"{key_metadata.metavar} {val_metadata.metavar}",
+        choices=None,
+        is_optional=False,
+    )
 
 
-@overload
-def _instantiator_from_type_inner(
-    typ: Type,
-    type_from_typevar: Dict[TypeVar, Type],
-    allow_sequences: Literal[True],
+def _instantiator_from_list_sequence_or_set(
+    typ: Type, type_from_typevar: Dict[TypeVar, Type]
 ) -> Tuple[Instantiator, InstantiatorMetadata]:
-    ...
+    (contained_type,) = get_args(typ)
+    container_type = get_origin(typ)
+    assert container_type is not None
+    if container_type is collections.abc.Sequence:
+        container_type = list
+
+    make, inner_meta = _instantiator_from_type_inner(
+        contained_type,
+        type_from_typevar,
+        allow_sequences=False,
+    )
+    return lambda strings: container_type(
+        [make(x) for x in strings]
+    ), InstantiatorMetadata(
+        nargs="+",
+        metavar=inner_meta.metavar,
+        choices=inner_meta.choices,
+        is_optional=False,
+    )
 
 
-def _instantiator_from_type_inner(
-    typ: Type,
-    type_from_typevar: Dict[TypeVar, Type],
-    allow_sequences: bool,
+def _instantiator_from_literal(
+    typ: Type, type_from_typevar: Dict[TypeVar, Type]
 ) -> Tuple[Instantiator, InstantiatorMetadata]:
-    """Thin wrapper over instantiator_from_type, with some extra asserts for catching
-    errors."""
-    out = instantiator_from_type(typ, type_from_typevar)
-    if not allow_sequences and out[1].nargs is not None:
-        raise UnsupportedTypeAnnotationError("Nested sequence types are not supported!")
-    return out
+    choices = get_args(typ)
+    if not len(set(map(type, choices))) == 1:
+        raise UnsupportedTypeAnnotationError(
+            "All choices in literal must have the same type!"
+        )
+    contained_type = type(next(iter(choices)))
+    if issubclass(contained_type, enum.Enum):
+        choices = tuple(map(lambda x: x.name, choices))
+    instantiator, metadata = _instantiator_from_type_inner(
+        contained_type,
+        type_from_typevar,
+        allow_sequences=False,
+    )
+    assert (
+        # Choices provided by the contained type
+        metadata.choices is None
+        or len(set(choices) - set(metadata.choices)) == 0
+    )
+    return instantiator, dataclasses.replace(
+        metadata,
+        choices=tuple(map(str, choices)),
+        metavar="{" + ",".join(map(str, choices)) + "}",
+        is_optional=False,
+    )
diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py
index bbb9e9936..11c699184 100644
--- a/dcargs/_parsers.py
+++ b/dcargs/_parsers.py
@@ -134,16 +134,25 @@ def from_callable(
                 typ=type_from_typevar.get(field.typ, field.typ),  # type: ignore
             )
             if isinstance(field.typ, TypeVar):
-                # Found an unbound TypeVar. This could be because inheriting from
-                # generics is currently not implemented. It's unclear whether this is
-                # feasible, because generics are lost in the mro:
-                # https://github.com/python/typing/issues/777
-                raise _instantiators.UnsupportedTypeAnnotationError(
-                    f"Field {field.name} has an unbound TypeVar: {field.typ}. Note that"
-                    " inheriting from generics is currently not implemented. It's"
-                    " unclear whether this is feasible, because generics are lost in"
-                    " the mro: https://github.com/python/typing/issues/777"
-                )
+                if field.typ.__bound__ is not None:
+                    # Try to infer type from TypeVar bound.
+                    field = dataclasses.replace(field, typ=field.typ.__bound__)
+                elif len(field.typ.__constraints__) > 0:
+                    # Try to infer type from TypeVar constraints.
+                    field = dataclasses.replace(
+                        field, typ=Union.__getitem__(field.typ.__constraints__)  # type: ignore
+                    )
+                else:
+                    # Found an unbound TypeVar. This could be because inheriting from
+                    # generics is currently not implemented. It's unclear whether this is
+                    # feasible, because generics are lost in the mro:
+                    # https://github.com/python/typing/issues/777
+                    raise _instantiators.UnsupportedTypeAnnotationError(
+                        f"Field {field.name} has an unbound TypeVar: {field.typ}. Note"
+                        " that inheriting from generics is currently not implemented."
+                        " It's unclear whether this is feasible, because generics are"
+                        " lost in the mro: https://github.com/python/typing/issues/777"
+                    )
 
             # (1) Handle Unions over callables; these result in subparsers.
             subparsers_attempt = SubparsersSpecification.from_field(
diff --git a/examples/07_literals.py b/examples/07_literals.py
deleted file mode 100644
index b804bbfd5..000000000
--- a/examples/07_literals.py
+++ /dev/null
@@ -1,36 +0,0 @@
-"""`typing.Literal[]` can be used to restrict inputs to a fixed set of choices.
-
-Usage:
-`python ./07_literals.py --help`
-`python ./07_literals.py --enum RED --restricted-enum GREEN --integer 3 --string green`
-"""
-
-import dataclasses
-import enum
-from typing import Literal
-
-import dcargs
-
-
-class Color(enum.Enum):
-    RED = enum.auto()
-    GREEN = enum.auto()
-    BLUE = enum.auto()
-
-
-@dataclasses.dataclass(frozen=True)
-class Args:
-    enum: Color
-    restricted_enum: Literal[Color.RED, Color.GREEN]
-
-    integer: Literal[0, 1, 2, 3]
-    string: Literal["red", "green"]
-
-    restricted_enum_with_default: Literal[Color.RED, Color.GREEN] = Color.GREEN
-    integer_with_default: Literal[0, 1, 2, 3] = 3
-    string_with_Default: Literal["red", "green"] = "red"
-
-
-if __name__ == "__main__":
-    args = dcargs.cli(Args)
-    print(args)
diff --git a/examples/07_literals_and_unions.py b/examples/07_literals_and_unions.py
new file mode 100644
index 000000000..9438ccd95
--- /dev/null
+++ b/examples/07_literals_and_unions.py
@@ -0,0 +1,40 @@
+"""`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices;
+`typing.Union[]` can be used to restrict inputs to a fixed set of types.
+
+Usage:
+`python ./07_literals_and_unions.py --help`
+`python ./07_literals_and_unions.py --enum RED --restricted-enum GREEN --integer 3 --string green`
+`python ./07_literals_and_unions.py --string-or-enum green`
+`python ./07_literals_and_unions.py --string-or-enum RED`
+`python ./07_literals_and_unions.py --tuple-of-string-or-enum RED green BLUE`
+"""
+
+import dataclasses
+import enum
+from typing import Literal, Tuple, Union
+
+import dcargs
+
+
+class Color(enum.Enum):
+    RED = enum.auto()
+    GREEN = enum.auto()
+    BLUE = enum.auto()
+
+
+@dataclasses.dataclass(frozen=True)
+class Args:
+    enum: Color = Color.RED
+    restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED
+    integer: Literal[0, 1, 2, 3] = 0
+    string: Literal["red", "green"] = "red"
+    string_or_enum: Union[Literal["red", "green"], Color] = "red"
+    tuple_of_string_or_enum: Tuple[Union[Literal["red", "green"], Color], ...] = (
+        "red",
+        Color.RED,
+    )
+
+
+if __name__ == "__main__":
+    args = dcargs.cli(Args)
+    print(args)
diff --git a/examples/09_subparsers.py b/examples/09_subparsers.py
index 911b9eb2e..34808114c 100644
--- a/examples/09_subparsers.py
+++ b/examples/09_subparsers.py
@@ -31,7 +31,7 @@ class Commit:
     all: bool = False
 
 
-def main(cmd: Union[Checkout, Commit] = Checkout("main")) -> None:
+def main(cmd: Union[Checkout, Commit]) -> None:
     print(cmd)
 
 
diff --git a/tests/test_collections.py b/tests/test_collections.py
index 23af76ea2..2c6e31ce1 100644
--- a/tests/test_collections.py
+++ b/tests/test_collections.py
@@ -1,6 +1,6 @@
 import dataclasses
 import enum
-from typing import List, Optional, Sequence, Set, Tuple
+from typing import Any, List, Optional, Sequence, Set, Tuple, Union
 
 import pytest
 from typing_extensions import Literal
@@ -253,3 +253,19 @@ class A:
 
     assert dcargs.cli(A, args=["--x", "0", "1"]) == A((0, 1))
     assert dcargs.cli(A, args=["--x", "0", "None", "1"]) == A((0, None, 1))
+
+
+def test_union_over_collections():
+    def main(a: Union[Tuple[float, ...], Tuple[int, ...]]) -> Any:
+        return a
+
+    assert dcargs.cli(main, args="--a 3.3 3.3 7.0".split(" ")) == (3.3, 3.3, 7.0)
+    assert dcargs.cli(main, args="--a 3 3 7".split(" ")) == (3, 3, 7)
+
+
+def test_union_over_collections_2():
+    def main(a: Union[Tuple[str, float, str], Tuple[str, str, float]]) -> Any:
+        return a
+
+    assert dcargs.cli(main, args="--a 3.3 hey 7.0".split(" ")) == ("3.3", "hey", 7.0)
+    assert dcargs.cli(main, args="--a 3 3 hey".split(" ")) == ("3", 3.0, "hey")
diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py
index 94ae28e68..d0adbebc5 100644
--- a/tests/test_dcargs.py
+++ b/tests/test_dcargs.py
@@ -1,7 +1,7 @@
 import dataclasses
 import enum
 import pathlib
-from typing import Any, ClassVar, Optional
+from typing import Any, ClassVar, Optional, TypeVar, Union
 
 import pytest
 from typing_extensions import Annotated, Final, Literal, TypeAlias
@@ -202,6 +202,35 @@ class A:
     assert dcargs.cli(A, args=[]) == A(x=None)
 
 
+def test_union():
+    def main(x: Union[int, str]) -> Union[int, str]:
+        return x
+
+    assert dcargs.cli(main, args=["--x", "5"]) == 5
+    assert dcargs.cli(main, args=["--x", "five"]) == "five"
+
+
+def test_func_typevar():
+    T = TypeVar("T", int, str)
+
+    def main(x: T) -> T:
+        return x
+
+    assert dcargs.cli(main, args=["--x", "5"]) == 5
+    assert dcargs.cli(main, args=["--x", "five"]) == "five"
+
+
+def test_func_typevar_bound():
+    T = TypeVar("T", bound=int)
+
+    def main(x: T) -> T:
+        return x
+
+    assert dcargs.cli(main, args=["--x", "5"]) == 5
+    with pytest.raises(SystemExit):
+        dcargs.cli(main, args=["--x", "five"])
+
+
 def test_enum():
     class Color(enum.Enum):
         RED = enum.auto()
diff --git a/tests/test_errors.py b/tests/test_errors.py
index 40d80eb91..a8fb1e678 100644
--- a/tests/test_errors.py
+++ b/tests/test_errors.py
@@ -44,14 +44,6 @@ class A:
         dcargs.cli(A, args=["--x", "0", "1"])
 
 
-def test_unsupported_union():
-    def main(x: Union[int, str]) -> None:
-        return
-
-    with pytest.raises(dcargs.UnsupportedTypeAnnotationError):
-        dcargs.cli(main, args=[])
-
-
 def test_unsupported_literal():
     def main(x: Literal[0, "5"]) -> None:
         return
@@ -141,3 +133,11 @@ class ChildClass(UnrelatedParentClass, ActualParentClass[int]):
 
     with pytest.raises(dcargs.UnsupportedTypeAnnotationError):
         dcargs.cli(ChildClass, args=["--x", "1", "--y", "2", "--z", "3"])
+
+
+def test_unsupported_union():
+    def main(a: Union[Tuple[int, int], Tuple[int, int, int]]) -> None:
+        pass
+
+    with pytest.raises(dcargs.UnsupportedTypeAnnotationError):
+        dcargs.cli(main, args=["--a", "5", "5"])
diff --git a/tests/test_helptext.py b/tests/test_helptext.py
index 48f74ea6b..182893242 100644
--- a/tests/test_helptext.py
+++ b/tests/test_helptext.py
@@ -195,7 +195,7 @@ class Config:
         with contextlib.redirect_stdout(f):
             dcargs.cli(Config, args=["--help"])
     helptext = f.getvalue()
-    assert "  --x (INT|None)  An optional variable. (default: None)\n" in helptext
+    assert "  --x (None|INT)  An optional variable. (default: None)\n" in helptext
 
 
 def test_helptext_hard_bool():
@@ -370,7 +370,7 @@ class OptionalLiteralHelptext:
         with contextlib.redirect_stdout(f):
             dcargs.cli(OptionalLiteralHelptext, args=["--help"])
     helptext = f.getvalue()
-    assert "--x ({1,2,3}|None)  A number. (default: None)\n" in helptext
+    assert "--x (None|{1,2,3})  A number. (default: None)\n" in helptext
 
 
 def test_multiple_subparsers_helptext():
@@ -439,6 +439,6 @@ class OptionalHelptext:
             dcargs.cli(OptionalHelptext, args=["--help"])
     helptext = f.getvalue()
     assert cast(str, OptionalHelptext.__doc__) in helptext
-    assert "[--x (INT|None)]" in helptext
-    assert "--y (INT|None) [(INT|None) ...]\n" in helptext
-    assert "[--z (INT|None)]\n" in helptext
+    assert "[--x (None|INT)]" in helptext
+    assert "--y (None|INT) [(None|INT) ...]\n" in helptext
+    assert "[--z (None|INT)]\n" in helptext
diff --git a/tests/test_positional_ignore_py37.py b/tests/test_positional_ignore_py37.py
index eb1d9d3c9..dd66f26cf 100644
--- a/tests/test_positional_ignore_py37.py
+++ b/tests/test_positional_ignore_py37.py
@@ -1,6 +1,6 @@
 import contextlib
 import io
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 
 import pytest
 
@@ -99,3 +99,12 @@ def main(
         dcargs.cli(main, args=["true"])
     with pytest.raises(SystemExit):
         dcargs.cli(main, args=["True", "false"])
+
+
+def test_unsupported_positional():
+    # Not super clear how to parse optional positional sequences...
+    def main(a: Optional[List[int]], /) -> None:
+        pass
+
+    with pytest.raises(dcargs.UnsupportedTypeAnnotationError):
+        dcargs.cli(main, args=["--a", "5", "5"])

From 57771d0d39dfa4f810026895141575286b7e2517 Mon Sep 17 00:00:00 2001
From: Brent Yi 
Date: Wed, 13 Jul 2022 04:12:16 -0700
Subject: [PATCH 055/491] Fix Literal[True] and Literal[False] corner case

---
 dcargs/_instantiators.py | 19 +++++++++++--------
 tests/test_dcargs.py     | 17 +++++++++++++++++
 2 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py
index 4180f3a91..06b84b1c1 100644
--- a/dcargs/_instantiators.py
+++ b/dcargs/_instantiators.py
@@ -352,9 +352,9 @@ def _instantiator_from_union(
         metas.pop(0)
 
     metavar: Union[str, Tuple[str, ...]]
-    if isinstance(metas[0].metavar, str):
+    if all(map(lambda m: isinstance(m.metavar, str), metas)):
         metavar = "(" + "|".join(map(lambda x: cast(str, x.metavar), metas)) + ")"
-    else:
+    elif all(map(lambda m: isinstance(m.metavar, tuple), metas)):
         # Do our best to create a reasonable looking metavar.
         # This is imperfect!
         assert isinstance(metas[0].metavar, tuple)
@@ -364,6 +364,9 @@ def _instantiator_from_union(
                 zip(*map(lambda m: m.metavar, metas)),
             )
         )
+    else:
+        # Should never hit this case.
+        assert False
 
     def union_instantiator(string_or_strings: Union[str, List[str]]) -> Any:
         for instantiator, metadata in zip(instantiators, metas):
@@ -461,19 +464,19 @@ def _instantiator_from_literal(
     contained_type = type(next(iter(choices)))
     if issubclass(contained_type, enum.Enum):
         choices = tuple(map(lambda x: x.name, choices))
+    else:
+        choices = tuple(map(str, choices))
     instantiator, metadata = _instantiator_from_type_inner(
         contained_type,
         type_from_typevar,
         allow_sequences=False,
     )
-    assert (
-        # Choices provided by the contained type
-        metadata.choices is None
-        or len(set(choices) - set(metadata.choices)) == 0
-    )
+    if metadata.choices is not None:
+        assert all(map(lambda t: isinstance(t, str), metadata.choices))
+        assert len(set(choices) - set(metadata.choices)) == 0
     return instantiator, dataclasses.replace(
         metadata,
-        choices=tuple(map(str, choices)),
+        choices=choices,
         metavar="{" + ",".join(map(str, choices)) + "}",
         is_optional=False,
     )
diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py
index d0adbebc5..0f217eea3 100644
--- a/tests/test_dcargs.py
+++ b/tests/test_dcargs.py
@@ -261,6 +261,23 @@ class A:
         assert dcargs.cli(A, args=["--x", "3"])
 
 
+def test_literal_bool():
+    def main(x: Literal[True]) -> bool:
+        return x
+
+    assert dcargs.cli(main, args=["--x", "True"]) is True
+    with pytest.raises(SystemExit):
+        dcargs.cli(main, args=["--x", "False"])
+
+    def main2(x: Literal[True, False]) -> bool:
+        return x
+
+    assert dcargs.cli(main2, args=["--x", "True"]) is True
+    assert dcargs.cli(main2, args=["--x", "False"]) is False
+    with pytest.raises(SystemExit):
+        dcargs.cli(main2, args=["--x", "Tru"])
+
+
 def test_literal_enum():
     class Color(enum.Enum):
         RED = enum.auto()

From b6dfdc99a392f16a85c8faf93cf149daa4b100c7 Mon Sep 17 00:00:00 2001
From: Brent Yi 
Date: Thu, 14 Jul 2022 05:02:27 -0700
Subject: [PATCH 056/491] Minor: collection types, union metavars, tests

---
 README.md                 | 12 +++++-----
 dcargs/_fields.py         | 10 +++++++-
 dcargs/_instantiators.py  | 48 ++++++++++++++++++++++++++++++++-------
 dcargs/_parsers.py        |  6 ++++-
 dcargs/_strings.py        |  4 +++-
 tests/test_collections.py | 29 ++++++++++++++++++++++-
 tests/test_dcargs.py      | 21 +++++++++++++++--
 tests/test_errors.py      |  8 +++++++
 tests/test_helptext.py    | 10 ++++----
 9 files changed, 123 insertions(+), 25 deletions(-)

diff --git a/README.md b/README.md
index 622b6209a..0f172358f 100644
--- a/README.md
+++ b/README.md
@@ -313,7 +313,7 @@ if __name__ == "__main__":
 usage: 03_enums_and_containers.py [-h] --dataset-sources PATH [PATH ...]
                                   [--image-dimensions INT INT]
                                   [--optimizer-type {ADAM,SGD}]
-                                  [--checkpoint-interval (None|INT)]
+                                  [--checkpoint-interval None|INT]
 
 required arguments:
   --dataset-sources PATH [PATH ...]
@@ -325,7 +325,7 @@ optional arguments:
                         Height and width of some image data. (default: 32 32)
   --optimizer-type {ADAM,SGD}
                         Gradient-based optimizer to use. (default: ADAM)
-  --checkpoint-interval (None|INT)
+  --checkpoint-interval None|INT
                         Interval to save checkpoints at. (default: None)
 

 
@@ -786,8 +786,8 @@ if __name__ == "__main__":
 usage: 07_literals_and_unions.py [-h] [--enum {RED,GREEN,BLUE}]
                                  [--restricted-enum {RED,GREEN}]
                                  [--integer {0,1,2,3}] [--string {red,green}]
-                                 [--string-or-enum ({red,green}|{RED,GREEN,BLUE})]
-                                 [--tuple-of-string-or-enum ({red,green}|{RED,GREEN,BLUE}) [({red,green}|{RED,GREEN,BLUE}) ...]]
+                                 [--string-or-enum {red,green,RED,GREEN,BLUE}]
+                                 [--tuple-of-string-or-enum {red,green,RED,GREEN,BLUE} [{red,green,RED,GREEN,BLUE} ...]]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -797,9 +797,9 @@ optional arguments:
                         (default: RED)
   --integer {0,1,2,3}   (default: 0)
   --string {red,green}  (default: red)
-  --string-or-enum ({red,green}|{RED,GREEN,BLUE})
+  --string-or-enum {red,green,RED,GREEN,BLUE}
                         (default: red)
-  --tuple-of-string-or-enum ({red,green}|{RED,GREEN,BLUE}) [({red,green}|{RED,GREEN,BLUE}) ...]
+  --tuple-of-string-or-enum {red,green,RED,GREEN,BLUE} [{red,green,RED,GREEN,BLUE} ...]
                         (default: red RED)
diff --git a/dcargs/_fields.py b/dcargs/_fields.py index d7c1d180e..4677d4da9 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -143,7 +143,8 @@ def field_list_from_callable( # Generate field list from function signature. field_list = [] ignore_self = cls is not None - for param in inspect.signature(f).parameters.values(): + params = inspect.signature(f).parameters.values() + for param in params: # For `__init__`, skip self parameter. if ignore_self: ignore_self = False @@ -159,6 +160,13 @@ def field_list_from_callable( if helptext is None and cls is not None: helptext = _docstrings.get_field_docstring(cls, param.name) + if param.name not in hints: + raise TypeError( + f"Expected fully type-annotated callable, but {f} with arguments" + f" {tuple(map(lambda p: p.name, params))} has no annotation for" + f" '{param.name}'." + ) + field_list.append( Field( name=param.name, diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index 06b84b1c1..3abb9e6f2 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -38,11 +38,13 @@ import enum import inspect import warnings +from collections import deque from typing import ( Any, Callable, Dict, Hashable, + Iterable, List, Optional, Tuple, @@ -141,7 +143,7 @@ def instantiator(_unused_string: str) -> None: pass elif not callable(typ): raise UnsupportedTypeAnnotationError( - f"Expected {typ} to be a `(arg: str) -> T` type converter, but is not" + f"Expected {typ} to be an `(arg: str) -> T` type converter, but is not" " callable." ) else: @@ -151,7 +153,7 @@ def instantiator(_unused_string: str) -> None: for i, param in enumerate(signature.parameters.values()): if i == 0 and param.annotation not in (str, inspect.Parameter.empty): raise UnsupportedTypeAnnotationError( - f"Expected {typ} to be a `(arg: str) -> T` type converter, but got" + f"Expected {typ} to be an `(arg: str) -> T` type converter, but got" f" {signature}. You may have a nested type in a container, which is" " unsupported." ) @@ -165,7 +167,7 @@ def instantiator(_unused_string: str) -> None: # Raise an error if parameters look wrong. if not (param_count == 1 or (param_count == 0 and has_var_positional)): raise UnsupportedTypeAnnotationError( - f"Expected {typ} to be a `(arg: str) -> T` type converter, but got" + f"Expected {typ} to be an `(arg: str) -> T` type converter, but got" f" {signature}. You may have a nested type in a container, which is" " unsupported." ) @@ -238,7 +240,13 @@ def _instantiator_from_container_type( return instantiator_from_type(contained_type, type_from_typevar) for make, matched_origins in { - _instantiator_from_list_sequence_or_set: (collections.abc.Sequence, list, set), + _instantiator_from_list_sequence_or_set: ( + collections.abc.Sequence, + frozenset, + list, + set, + deque, + ), _instantiator_from_tuple: (tuple,), _instantiator_from_dict: (dict, collections.abc.Mapping), _instantiator_from_union: (Union,), @@ -305,6 +313,31 @@ def _instantiator_from_tuple( ) +def _join_union_metavars(metavars: Iterable[str]) -> str: + """Metavar generation helper for unions. + + Examples: + None, INT => NONE|INT + {0,1,2}, {3,4} => {0,1,2,3,4} + None, {0,1,2}, {3,4} => None|{0,1,2,3,4} + """ + metavars = tuple(metavars) + merged_metavars = [metavars[0]] + for i in range(1, len(metavars)): + prev = merged_metavars[-1] + curr = metavars[i] + if ( + prev.startswith("{") + and prev.endswith("}") + and curr.startswith("{") + and curr.endswith("}") + ): + merged_metavars[-1] = prev[:-1] + "," + curr[1:] + else: + merged_metavars.append(curr) + return "|".join(merged_metavars) + + def _instantiator_from_union( typ: Type, type_from_typevar: Dict[TypeVar, Type] ) -> Tuple[Instantiator, InstantiatorMetadata]: @@ -353,14 +386,13 @@ def _instantiator_from_union( metavar: Union[str, Tuple[str, ...]] if all(map(lambda m: isinstance(m.metavar, str), metas)): - metavar = "(" + "|".join(map(lambda x: cast(str, x.metavar), metas)) + ")" + metavar = _join_union_metavars(map(lambda x: cast(str, x.metavar), metas)) elif all(map(lambda m: isinstance(m.metavar, tuple), metas)): - # Do our best to create a reasonable looking metavar. - # This is imperfect! + # Do our best to create a reasonable looking metavar. This is imperfect! assert isinstance(metas[0].metavar, tuple) metavar = tuple( map( - lambda metavars: "(" + "|".join(metavars) + ")", + lambda metavars: _join_union_metavars(metavars), zip(*map(lambda m: m.metavar, metas)), ) ) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 11c699184..2569e3226 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -72,7 +72,11 @@ def _is_possibly_nested_type(typ: Any) -> bool: # Nested types like nested (data)classes should have fully type-annotated inputs. If # any inputs are unannotated (for example, in the case of pathlib.Path), we can # assume the type is not nested. - for param in inspect.signature(typ).parameters.values(): + try: + sig = inspect.signature(typ) + except ValueError: + return False + for param in sig.parameters.values(): if param.annotation is inspect.Parameter.empty: return False diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 173368445..6de2bc8b3 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -66,7 +66,9 @@ def instance_from_string(typ: Type[T], arg: str) -> T: assert len(get_args(typ)) == 0, f"Type {typ} cannot be instantiated." if typ is bool: return {"True": True, "False": False}[arg] # type: ignore - elif issubclass(typ, enum.Enum): + elif isinstance(typ, type) and issubclass(typ, enum.Enum): return typ[arg] # type: ignore + elif typ is bytes: + return bytes(arg, encoding="ascii") # type: ignore else: return typ(arg) # type: ignore diff --git a/tests/test_collections.py b/tests/test_collections.py index 2c6e31ce1..68dcdd3ab 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -1,6 +1,7 @@ +import collections import dataclasses import enum -from typing import Any, List, Optional, Sequence, Set, Tuple, Union +from typing import Any, Deque, FrozenSet, List, Optional, Sequence, Set, Tuple, Union import pytest from typing_extensions import Literal @@ -209,6 +210,32 @@ class A: dcargs.cli(A, args=[]) +def test_frozen_sets(): + @dataclasses.dataclass + class A: + x: FrozenSet[int] + + assert dcargs.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x=frozenset({1, 2, 3})) + with pytest.raises(SystemExit): + dcargs.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.cli(A, args=[]) + + +def test_deque(): + @dataclasses.dataclass + class A: + x: Deque[int] + + assert dcargs.cli(A, args=["--x", "1", "2", "3", "3"]) == A( + x=collections.deque([1, 2, 3, 3]) + ) + with pytest.raises(SystemExit): + dcargs.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + dcargs.cli(A, args=[]) + + def test_sets_with_default(): @dataclasses.dataclass class A: diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 0f217eea3..996acede9 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -1,7 +1,7 @@ import dataclasses import enum import pathlib -from typing import Any, ClassVar, Optional, TypeVar, Union +from typing import Any, AnyStr, ClassVar, Optional, TypeVar, Union import pytest from typing_extensions import Annotated, Final, Literal, TypeAlias @@ -203,10 +203,11 @@ class A: def test_union(): - def main(x: Union[int, str]) -> Union[int, str]: + def main(x: Union[Literal[1, 2], Literal[3, 4, 5], str]) -> Union[int, str]: return x assert dcargs.cli(main, args=["--x", "5"]) == 5 + assert dcargs.cli(main, args=["--x", "6"]) == "6" assert dcargs.cli(main, args=["--x", "five"]) == "five" @@ -394,3 +395,19 @@ def main(x: Any) -> Any: return x assert dcargs.cli(main, args=["--x", "hello"]) == "hello" + + +def test_bytes(): + def main(x: bytes) -> bytes: + return x + + assert dcargs.cli(main, args=["--x", "hello"]) == b"hello" + + +def test_any_str(): + def main(x: AnyStr) -> AnyStr: + return x + + # Use bytes when provided ascii-compatible inputs. + assert dcargs.cli(main, args=["--x", "hello"]) == b"hello" + assert dcargs.cli(main, args=["--x", "hello„"]) == "hello„" diff --git a/tests/test_errors.py b/tests/test_errors.py index a8fb1e678..07842d874 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -141,3 +141,11 @@ def main(a: Union[Tuple[int, int], Tuple[int, int, int]]) -> None: with pytest.raises(dcargs.UnsupportedTypeAnnotationError): dcargs.cli(main, args=["--a", "5", "5"]) + + +def test_missing_annotation(): + def main(a) -> None: + pass + + with pytest.raises(TypeError): + dcargs.cli(main, args=["--help"]) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 182893242..34f64f278 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -195,7 +195,7 @@ class Config: with contextlib.redirect_stdout(f): dcargs.cli(Config, args=["--help"]) helptext = f.getvalue() - assert " --x (None|INT) An optional variable. (default: None)\n" in helptext + assert " --x None|INT An optional variable. (default: None)\n" in helptext def test_helptext_hard_bool(): @@ -370,7 +370,7 @@ class OptionalLiteralHelptext: with contextlib.redirect_stdout(f): dcargs.cli(OptionalLiteralHelptext, args=["--help"]) helptext = f.getvalue() - assert "--x (None|{1,2,3}) A number. (default: None)\n" in helptext + assert "--x None|{1,2,3} A number. (default: None)\n" in helptext def test_multiple_subparsers_helptext(): @@ -439,6 +439,6 @@ class OptionalHelptext: dcargs.cli(OptionalHelptext, args=["--help"]) helptext = f.getvalue() assert cast(str, OptionalHelptext.__doc__) in helptext - assert "[--x (None|INT)]" in helptext - assert "--y (None|INT) [(None|INT) ...]\n" in helptext - assert "[--z (None|INT)]\n" in helptext + assert "[--x None|INT]" in helptext + assert "--y None|INT [None|INT ...]\n" in helptext + assert "[--z None|INT]\n" in helptext From 36a358075bf14dce703e9aa5f717de07d847c7c1 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 15 Jul 2022 04:54:28 -0700 Subject: [PATCH 057/491] Helptext housekeeping, unsupported types with defaults (creates "fixed args") --- README.md | 311 +++++++++++++--------- _update_readme.py | 6 +- dcargs/_arguments.py | 94 +++++-- dcargs/_calling.py | 52 ++-- dcargs/_cli.py | 4 +- dcargs/_fields.py | 18 +- dcargs/_parsers.py | 49 +--- dcargs/_strings.py | 9 + examples/04_flags.py | 1 + tests/test_dcargs.py | 11 +- tests/test_dict_namedtuple.py | 28 +- tests/test_helptext.py | 102 +++---- tests/test_unsupported_but_should_work.py | 12 +- 13 files changed, 429 insertions(+), 268 deletions(-) diff --git a/README.md b/README.md index 0f172358f..a529b76cd 100644 --- a/README.md +++ b/README.md @@ -166,17 +166,16 @@ if __name__ == "__main__": 
 $ python ./01_functions.py --help
-usage: 01_functions.py [-h] --field1 STR [--field2 INT] [--flag]
+usage: 01_functions.py [-h] --field1 STR [--field2 INT]
+                       [--flag]
 
 Function, whose arguments will be populated from a CLI interface.
 
-required arguments:
-  --field1 STR  A string field.
-
-optional arguments:
-  -h, --help    show this help message and exit
+arguments:
+  -h, --help            show this help message and exit
+  --field1 STR  A string field. (required)
   --field2 INT  A numeric field, with a default value. (default: 3)
-  --flag        A boolean flag.
+  --flag                A boolean flag.
 
@@ -228,18 +227,17 @@ if __name__ == "__main__":
 
 
 $ python ./02_dataclasses.py --help
-usage: 02_dataclasses.py [-h] --field1 STR [--field2 INT] [--flag]
+usage: 02_dataclasses.py [-h] --field1 STR [--field2 INT]
+                         [--flag]
 
 Description.
 This should show up in the helptext!
 
-required arguments:
-  --field1 STR  A string field.
-
-optional arguments:
-  -h, --help    show this help message and exit
+arguments:
+  -h, --help            show this help message and exit
+  --field1 STR  A string field. (required)
   --field2 INT  A numeric field, with a default value. (default: 3)
-  --flag        A boolean flag.
+  --flag                A boolean flag.
 

 
 
@@ -310,17 +308,16 @@ if __name__ == "__main__":
 
 
 $ python ./03_enums_and_containers.py --help
-usage: 03_enums_and_containers.py [-h] --dataset-sources PATH [PATH ...]
+usage: 03_enums_and_containers.py [-h] --dataset-sources PATH
+                                  [PATH ...]
                                   [--image-dimensions INT INT]
                                   [--optimizer-type {ADAM,SGD}]
                                   [--checkpoint-interval None|INT]
 
-required arguments:
-  --dataset-sources PATH [PATH ...]
-                        Paths to load training data from. This can be multiple!
-
-optional arguments:
+arguments:
   -h, --help            show this help message and exit
+  --dataset-sources PATH [PATH ...]
+                        Paths to load training data from. This can be multiple! (required)
   --image-dimensions INT INT
                         Height and width of some image data. (default: 32 32)
   --optimizer-type {ADAM,SGD}
@@ -383,6 +380,22 @@ if __name__ == "__main__":
 
 **Example usage:**
 
+
+$ python ./04_flags.py --help
+usage: 04_flags.py [-h] --boolean {True,False}
+                   [--optional-boolean None|{True,False}] [--flag-a]
+                   [--no-flag-b]
+
+arguments:
+  -h, --help            show this help message and exit
+  --boolean {True,False}
+                        Boolean. This expects an explicit "True" or "False". (required)
+  --optional-boolean None|{True,False}
+                        Optional boolean. Same as above, but can be omitted. (default: None)
+  --flag-a              Pass --flag-a in to set this value to True.
+  --no-flag-b           Pass --no-flag-b in to set this value to False. (default: True)
+

+
 
 $ python ./04_flags.py --boolean True
 Args(boolean=True, optional_boolean=None, flag_a=False, flag_b=True)
@@ -488,22 +501,23 @@ usage: 05_hierarchical_configs.py [-h]
                                   [--config.optimizer.weight-decay FLOAT]
                                   [--config.batch-size INT]
                                   [--config.train-steps INT]
-                                  [--config.seed INT] [--restore-checkpoint]
+                                  [--config.seed INT]
+                                  [--restore-checkpoint]
                                   [--checkpoint-interval INT]
                                   OUT_DIR
 
 Train a model.
 
 positional arguments:
-  OUT_DIR               Where to save logs and checkpoints.
+  OUT_DIR       Where to save logs and checkpoints. (required)
 
-optional arguments:
+arguments:
   -h, --help            show this help message and exit
   --restore-checkpoint  Set to restore an existing checkpoint.
   --checkpoint-interval INT
                         Training steps between each checkpoint save. (default: 1000)
 
-optional config.optimizer arguments:
+config.optimizer arguments:
   Various configurable options for our optimizer.
 
   --config.optimizer.algorithm {ADAM,SGD}
@@ -513,14 +527,15 @@ optional config.optimizer arguments:
   --config.optimizer.weight-decay FLOAT
                         Coefficient for L2 regularization. (default: 0.01)
 
-optional config arguments:
+config arguments:
   Experiment configuration.
 
   --config.batch-size INT
                         Batch size. (default: 32)
   --config.train-steps INT
                         Total number of training steps. (default: 100000)
-  --config.seed INT     Random seed. This is helpful for making sure that our experiments are all
+  --config.seed INT
+                        Random seed. This is helpful for making sure that our experiments are all
                         reproducible! (default: 0)
 

 
@@ -571,11 +586,22 @@ one of multiple possible base configurations, and then use the CLI to either ove
 ```python
 import dataclasses
 import os
-from typing import Literal, Tuple, Union
+from typing import Callable, Literal, Tuple, Union
 
 import dcargs
 
 
+# Learning rate schedulers.
+def no_lr_scheduler(step: int) -> float:
+    return 1.0
+
+
+def linear_warmup_1000_scheduler(step: int) -> float:
+    assert step >= 0
+    return min(1.0, step / 1000.0)
+
+
+# Optimizer configs.
 @dataclasses.dataclass
 class AdamOptimizer:
     # Adam learning rate.
@@ -591,6 +617,7 @@ class SgdOptimizer:
     learning_rate: float = 3e-4
 
 
+# Overall experiment config.
 @dataclasses.dataclass(frozen=True)
 class ExperimentConfig:
     # Dataset to run experiment on.
@@ -599,6 +626,11 @@ class ExperimentConfig:
     # Optimizer parameters.
     optimizer: Union[AdamOptimizer, SgdOptimizer]
 
+    # Learning rate scheduler. Fields with types that `dcargs.cli()` does not support
+    # instantiating from the CLI (such as Callables) will always be kept at their
+    # default value.
+    lr_scheduler: Callable[[int], float]
+
     # Model size.
     num_layers: int
     units: int
@@ -621,6 +653,7 @@ base_config_library = {
     "small": ExperimentConfig(
         dataset="mnist",
         optimizer=SgdOptimizer(),
+        lr_scheduler=no_lr_scheduler,
         batch_size=2048,
         num_layers=4,
         units=64,
@@ -632,6 +665,7 @@ base_config_library = {
     "big": ExperimentConfig(
         dataset="imagenet-50",
         optimizer=AdamOptimizer(),
+        lr_scheduler=linear_warmup_1000_scheduler,
         batch_size=32,
         num_layers=8,
         units=256,
@@ -668,24 +702,31 @@ if __name__ == "__main__":
 
 $ BASE_CONFIG=small python ./06_base_configs.py --help
 usage: 06_base_configs.py [-h] [--dataset {mnist,imagenet-50}]
-                          [--optimizer.learning-rate FLOAT] [--num-layers INT]
-                          [--units INT] [--batch-size INT] [--train-steps INT]
-                          --seed INT
-
-required arguments:
-  --seed INT            Random seed. This is helpful for making sure that our experiments are all
-                        reproducible!
+                          [--optimizer.learning-rate FLOAT]
+                          [--lr-scheduler fixed]
+                          [--num-layers INT] [--units INT]
+                          [--batch-size INT]
+                          [--train-steps INT] --seed INT
 
-optional arguments:
+arguments:
   -h, --help            show this help message and exit
   --dataset {mnist,imagenet-50}
                         Dataset to run experiment on. (default: mnist)
-  --num-layers INT      Model size. (default: 4)
-  --units INT           Model size. (default: 64)
-  --batch-size INT      Batch size. (default: 2048)
-  --train-steps INT     Total number of training steps. (default: 30000)
-
-optional optimizer arguments:
+  --lr-scheduler (fixed)
+                        Learning rate scheduler. Fields with types that `dcargs.cli()` does not support
+                        instantiating from the CLI (such as Callables) will always be kept at their
+                        default value. (value: '<function no_lr_scheduler at 0x7fb5254763a0>')
+  --num-layers INT
+                        Model size. (default: 4)
+  --units INT   Model size. (default: 64)
+  --batch-size INT
+                        Batch size. (default: 2048)
+  --train-steps INT
+                        Total number of training steps. (default: 30000)
+  --seed INT    Random seed. This is helpful for making sure that our experiments are all
+                        reproducible! (required)
+
+optimizer arguments:
   Optimizer parameters.
 
   --optimizer.learning-rate FLOAT
@@ -694,31 +735,38 @@ optional optimizer arguments:
 
 
 $ BASE_CONFIG=small python ./06_base_configs.py --seed 94720
-ExperimentConfig(dataset='mnist', optimizer=SgdOptimizer(learning_rate=0.0003), num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720)
+ExperimentConfig(dataset='mnist', optimizer=SgdOptimizer(learning_rate=0.0003), lr_scheduler=<function no_lr_scheduler at 0x7fc74a7733a0>, num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720)
 

 
 
 $ BASE_CONFIG=big python ./06_base_configs.py --help
 usage: 06_base_configs.py [-h] [--dataset {mnist,imagenet-50}]
                           [--optimizer.learning-rate FLOAT]
-                          [--optimizer.betas FLOAT FLOAT] [--num-layers INT]
-                          [--units INT] [--batch-size INT] [--train-steps INT]
-                          --seed INT
+                          [--optimizer.betas FLOAT FLOAT]
+                          [--lr-scheduler fixed]
+                          [--num-layers INT] [--units INT]
+                          [--batch-size INT]
+                          [--train-steps INT] --seed INT
 
-required arguments:
-  --seed INT            Random seed. This is helpful for making sure that our experiments are all
-                        reproducible!
-
-optional arguments:
+arguments:
   -h, --help            show this help message and exit
   --dataset {mnist,imagenet-50}
                         Dataset to run experiment on. (default: imagenet-50)
-  --num-layers INT      Model size. (default: 8)
-  --units INT           Model size. (default: 256)
-  --batch-size INT      Batch size. (default: 32)
-  --train-steps INT     Total number of training steps. (default: 100000)
+  --lr-scheduler (fixed)
+                        Learning rate scheduler. Fields with types that `dcargs.cli()` does not support
+                        instantiating from the CLI (such as Callables) will always be kept at their
+                        default value. (value: '<function linear_warmup_1000_scheduler at 0x7ff2be2a0dc0>')
+  --num-layers INT
+                        Model size. (default: 8)
+  --units INT   Model size. (default: 256)
+  --batch-size INT
+                        Batch size. (default: 32)
+  --train-steps INT
+                        Total number of training steps. (default: 100000)
+  --seed INT    Random seed. This is helpful for making sure that our experiments are all
+                        reproducible! (required)
 
-optional optimizer arguments:
+optimizer arguments:
   Optimizer parameters.
 
   --optimizer.learning-rate FLOAT
@@ -729,7 +777,7 @@ optional optimizer arguments:
 
 
 $ BASE_CONFIG=big python ./06_base_configs.py --seed 94720
-ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learning_rate=0.001, betas=(0.9, 0.999)), num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720)
+ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learning_rate=0.001, betas=(0.9, 0.999)), lr_scheduler=<function linear_warmup_1000_scheduler at 0x7fb97f898dc0>, num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720)
@@ -785,18 +833,21 @@ if __name__ == "__main__": $ python ./07_literals_and_unions.py --help usage: 07_literals_and_unions.py [-h] [--enum {RED,GREEN,BLUE}] [--restricted-enum {RED,GREEN}] - [--integer {0,1,2,3}] [--string {red,green}] + [--integer {0,1,2,3}] + [--string {red,green}] [--string-or-enum {red,green,RED,GREEN,BLUE}] [--tuple-of-string-or-enum {red,green,RED,GREEN,BLUE} [{red,green,RED,GREEN,BLUE} ...]] -optional arguments: +arguments: -h, --help show this help message and exit --enum {RED,GREEN,BLUE} (default: RED) --restricted-enum {RED,GREEN} (default: RED) - --integer {0,1,2,3} (default: 0) - --string {red,green} (default: red) + --integer {0,1,2,3} + (default: 0) + --string {red,green} + (default: red) --string-or-enum {red,green,RED,GREEN,BLUE} (default: red) --tuple-of-string-or-enum {red,green,RED,GREEN,BLUE} [{red,green,RED,GREEN,BLUE} ...] @@ -898,25 +949,26 @@ if __name__ == "__main__": $ python ./08_positional_args.py --help usage: 08_positional_args.py [-h] [--optimizer.algorithm {ADAM,SGD}] [--optimizer.learning-rate FLOAT] - [--optimizer.weight-decay FLOAT] [--force] - [--verbose] [--background-rgb FLOAT FLOAT FLOAT] + [--optimizer.weight-decay FLOAT] + [--force] [--verbose] + [--background-rgb FLOAT FLOAT FLOAT] SOURCE DEST Command-line interface defined using a function signature. Note that this docstring is parsed to generate helptext. positional arguments: - SOURCE Source path. - DEST Destination path. + SOURCE Source path. (required) + DEST Destination path. (required) -optional arguments: +arguments: -h, --help show this help message and exit --force Do not prompt before overwriting. --verbose Explain what is being done. --background-rgb FLOAT FLOAT FLOAT Background color. Red by default. (default: 1.0 0.0 0.0) -optional optimizer arguments: +optimizer arguments: Configuration for our optimizer object. --optimizer.algorithm {ADAM,SGD} @@ -989,7 +1041,7 @@ if __name__ == "__main__": $ python ./09_subparsers.py --help usage: 09_subparsers.py [-h] {checkout,commit} ... -optional arguments: +arguments: -h, --help show this help message and exit subcommands: @@ -1003,13 +1055,12 @@ usage: 09_subparsers.py commit [-h] --cmd.message STR [--cmd.all] Commit changes. -optional arguments: - -h, --help show this help message and exit +arguments: + -h, --help show this help message and exit -required cmd arguments: +cmd arguments: --cmd.message STR - -optional cmd arguments: + (required) --cmd.all @@ -1024,11 +1075,12 @@ usage: 09_subparsers.py checkout [-h] --cmd.branch STR Checkout a branch. -optional arguments: - -h, --help show this help message and exit +arguments: + -h, --help show this help message and exit -required cmd arguments: - --cmd.branch STR +cmd arguments: + --cmd.branch STR + (required) 
@@ -1125,7 +1177,7 @@ usage: 10_multiple_subparsers.py [-h] [{mnist-dataset,image-net-dataset}] ...
 
 Example training script.
 
-optional arguments:
+arguments:
   -h, --help            show this help message and exit
 
 optional subcommands:
@@ -1140,10 +1192,10 @@ usage: 10_multiple_subparsers.py mnist-dataset [-h] [--dataset.binary]
                                                [{adam-optimizer,sgd-optimizer}]
                                                ...
 
-optional arguments:
+arguments:
   -h, --help            show this help message and exit
 
-optional dataset arguments:
+dataset arguments:
   --dataset.binary      Set to load binary version of MNIST dataset.
 
 optional subcommands:
@@ -1211,16 +1263,17 @@ if __name__ == "__main__":
 
 $ python ./11_dictionaries.py --help
 usage: 11_dictionaries.py [-h] --standard-dict STR {True,False}
-                          [STR {True,False} ...] [--typed-dict.field1 STR]
-                          [--typed-dict.field2 INT] [--typed-dict.field3]
+                          [STR {True,False} ...]
+                          [--typed-dict.field1 STR]
+                          [--typed-dict.field2 INT]
+                          [--typed-dict.field3]
 
-required arguments:
-  --standard-dict STR {True,False} [STR {True,False} ...]
-
-optional arguments:
+arguments:
   -h, --help            show this help message and exit
+  --standard-dict STR {True,False} [STR {True,False} ...]
+                        (required)
 
-optional typed_dict arguments:
+typed_dict arguments:
 
   --typed-dict.field1 STR
                         A string field. (default: hey)
@@ -1274,18 +1327,17 @@ if __name__ == "__main__":
 
 
 $ python ./12_named_tuples.py --help
-usage: 12_named_tuples.py [-h] --field1 STR [--field2 INT] [--flag]
+usage: 12_named_tuples.py [-h] --field1 STR [--field2 INT]
+                          [--flag]
 
 Description.
 This should show up in the helptext!
 
-required arguments:
-  --field1 STR  A string field.
-
-optional arguments:
-  -h, --help    show this help message and exit
+arguments:
+  -h, --help            show this help message and exit
+  --field1 STR  A string field. (required)
   --field2 INT  A numeric field, with a default value. (default: 3)
-  --flag        A boolean flag.
+  --flag                A boolean flag.
 

 
 
@@ -1338,17 +1390,16 @@ if __name__ == "__main__":
 
 
 $ python ./13_standard_classes.py --help
-usage: 13_standard_classes.py [-h] --field1 STR --field2 INT [--flag]
+usage: 13_standard_classes.py [-h] --field1 STR --field2 INT
+                              [--flag]
 
 Arguments.
 
-required arguments:
-  --field1 STR  A string field.
-  --field2 INT  A numeric field.
-
-optional arguments:
-  -h, --help    show this help message and exit
-  --flag        A boolean flag.
+arguments:
+  -h, --help            show this help message and exit
+  --field1 STR  A string field. (required)
+  --field2 INT  A numeric field. (required)
+  --flag                A boolean flag.
 

 
 
@@ -1411,54 +1462,76 @@ if __name__ == "__main__":
 
 
 $ python ./14_generics.py --help
-usage: 14_generics.py [-h] --point-continuous.x FLOAT --point-continuous.y
-                      FLOAT --point-continuous.z FLOAT
-                      --point-continuous.frame-id STR --point-discrete.x INT
-                      --point-discrete.y INT --point-discrete.z INT
-                      --point-discrete.frame-id STR --shape.a.x FLOAT
-                      --shape.a.y FLOAT --shape.a.z FLOAT --shape.a.frame-id
-                      STR --shape.b.x FLOAT --shape.b.y FLOAT --shape.b.z
-                      FLOAT --shape.b.frame-id STR --shape.c.x FLOAT
-                      --shape.c.y FLOAT --shape.c.z FLOAT --shape.c.frame-id
-                      STR
-
-optional arguments:
+usage: 14_generics.py [-h] --point-continuous.x FLOAT
+                      --point-continuous.y FLOAT --point-continuous.z
+                      FLOAT --point-continuous.frame-id STR
+                      --point-discrete.x INT --point-discrete.y
+                      INT --point-discrete.z INT
+                      --point-discrete.frame-id STR --shape.a.x
+                      FLOAT --shape.a.y FLOAT --shape.a.z
+                      FLOAT --shape.a.frame-id STR --shape.b.x
+                      FLOAT --shape.b.y FLOAT --shape.b.z
+                      FLOAT --shape.b.frame-id STR --shape.c.x
+                      FLOAT --shape.c.y FLOAT --shape.c.z
+                      FLOAT --shape.c.frame-id STR
+
+arguments:
   -h, --help            show this help message and exit
 
-required point_continuous arguments:
+point_continuous arguments:
 
   --point-continuous.x FLOAT
+                        (required)
   --point-continuous.y FLOAT
+                        (required)
   --point-continuous.z FLOAT
+                        (required)
   --point-continuous.frame-id STR
+                        (required)
 
-required point_discrete arguments:
+point_discrete arguments:
 
   --point-discrete.x INT
+                        (required)
   --point-discrete.y INT
+                        (required)
   --point-discrete.z INT
+                        (required)
   --point-discrete.frame-id STR
+                        (required)
 
-required shape.a arguments:
+shape.a arguments:
 
   --shape.a.x FLOAT
+                        (required)
   --shape.a.y FLOAT
+                        (required)
   --shape.a.z FLOAT
+                        (required)
   --shape.a.frame-id STR
+                        (required)
 
-required shape.b arguments:
+shape.b arguments:
 
   --shape.b.x FLOAT
+                        (required)
   --shape.b.y FLOAT
+                        (required)
   --shape.b.z FLOAT
+                        (required)
   --shape.b.frame-id STR
+                        (required)
 
-required shape.c arguments:
+shape.c arguments:
 
   --shape.c.x FLOAT
+                        (required)
   --shape.c.y FLOAT
+                        (required)
   --shape.c.z FLOAT
-  --shape.c.frame-id STR
+                        (required)
+  --shape.c.frame-id STR
+                        (required)
diff --git a/_update_readme.py b/_update_readme.py index 096113b60..1928d370f 100644 --- a/_update_readme.py +++ b/_update_readme.py @@ -6,11 +6,11 @@ import inspect import os import pathlib -import re import shlex import subprocess import dcargs +import dcargs._strings @dataclasses.dataclass(frozen=True) @@ -74,9 +74,7 @@ def format_script_for_readme(path: pathlib.Path) -> str: encoding="utf8", env=dict(os.environ, **env_vars), ).stdout - output = re.sub( # Strip colorcodes. - r"\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))", "", output - ).strip() + output = dcargs._strings.strip_color_codes(output).strip() example_output_lines.extend( [ "", diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index b027aad1a..4443dd35f 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -8,6 +8,8 @@ import shlex from typing import Any, Dict, Mapping, Optional, Set, Tuple, Type, TypeVar, Union +import termcolor + from . import _fields, _instantiators try: @@ -33,10 +35,8 @@ def add_argument( # Get keyword arguments, with None values removed. kwargs = dataclasses.asdict(self.lowered) - kwargs = dict(filter(lambda kv: kv[1] is not None, kwargs.items())) - - # Pop lowered properties t kwargs.pop("instantiator") + kwargs = {k: v for k, v in kwargs.items() if v is not None} name_or_flag = kwargs.pop("name_or_flag") # Note that the name must be passed in as a position argument. @@ -53,6 +53,7 @@ def lowered(self) -> LoweredArgumentDefinition: _rule_generate_helptext, _rule_set_name_or_flag, _rule_positional_special_handling, + _rule_bold_metavar, ) return functools.reduce( lambda lowered, rule: rule(self, lowered), @@ -72,6 +73,13 @@ class LoweredArgumentDefinition: # mixed-type tuples. instantiator: Optional[_instantiators.Instantiator] = None + def is_fixed(self) -> bool: + """If the instantiator is set to `None`, even after all argument + transformations, it means that we weren't able to determine a valid instantiator + for an argument. We then mark the argument as 'fixed', with a value always equal + to the field default.""" + return self.instantiator is None + # From here on out, all fields correspond 1:1 to inputs to argparse's # add_argument() method. name_or_flag: str = "" @@ -138,11 +146,24 @@ def _rule_recursive_instantiator_from_type( tuples.""" if lowered.instantiator is not None: return lowered + try: + instantiator, metadata = _instantiators.instantiator_from_type( + arg.field.typ, # type: ignore + arg.type_from_typevar, + ) + except _instantiators.UnsupportedTypeAnnotationError as e: + if arg.field.default is None: + raise e + else: + # For fields with a default, we'll get by even if there's no instantiator + # available. + return dataclasses.replace( + lowered, + metavar=termcolor.colored("(fixed)", color="red"), + required=False, + default=_fields.MISSING, + ) - instantiator, metadata = _instantiators.instantiator_from_type( - arg.field.typ, # type: ignore - arg.type_from_typevar, - ) return dataclasses.replace( lowered, instantiator=instantiator, @@ -166,7 +187,11 @@ def as_str(x: Any) -> str: else: return str(x) - if lowered.default is None or lowered.action is not None: + if ( + lowered.default is None + or lowered.default is _fields.MISSING + or lowered.action is not None + ): return lowered elif lowered.nargs is not None and lowered.nargs != "?": if isinstance(lowered.default, Mapping): @@ -199,26 +224,38 @@ def _rule_generate_helptext( elif arg.field.positional: help_parts.append(str(lowered.metavar)) - if lowered.action is not None: + default = lowered.default + if lowered.is_fixed(): + # For fixed args, we'll be missing the lowered default. Use field default + # instead. + assert default is _fields.MISSING + default = arg.field.default + + if lowered.action == "store_true": # Don't show defaults for boolean flags. - # assert lowered.action in ("store_true", "store_false") - pass + assert lowered.action in ("store_true", "store_false") elif not lowered.required: + default_label = "value" if lowered.is_fixed() else "default" # Include default value in helptext. We intentionally don't use the % template # because the types of all arguments are set to strings, which will cause the # default to be casted to a string and introduce extra quotation marks. - if lowered.nargs is not None and hasattr(lowered.default, "__iter__"): - # For tuple types, we might have lowered.default as (0, 1, 2, 3). - # For list types, we might have lowered.default as [0, 1, 2, 3]. - # For set types, we might have lowered.default as {0, 1, 2, 3}. + if lowered.nargs is not None and hasattr(default, "__iter__"): + # For tuple types, we might have default as (0, 1, 2, 3). + # For list types, we might have default as [0, 1, 2, 3]. + # For set types, we might have default as {0, 1, 2, 3}. # # In all cases, we want to display (default: 0 1 2 3), for consistency with # the format that argparse expects when we set nargs. - assert lowered.default is not None # Just for type checker. - default_parts = map(shlex.quote, map(str, lowered.default)) - help_parts.append(f"(default: {' '.join(default_parts)})") + assert default is not None # Just for type checker. + default_parts = map(shlex.quote, map(str, default)) + default_text = f"({default_label}: {' '.join(default_parts)})" else: - help_parts.append(f"(default: {shlex.quote(str(lowered.default))})") + default_text = f"({default_label}: {shlex.quote(str(default))})" + help_parts.append(default_text) + else: + help_parts.append( + termcolor.colored("(required)", on_color="on_red") + ) return dataclasses.replace(lowered, help=" ".join(help_parts)) @@ -257,3 +294,22 @@ def _rule_positional_special_handling( required=None, nargs="?" if not lowered.required else lowered.nargs, ) + + +def _rule_bold_metavar( + arg: ArgumentDefinition, + lowered: LoweredArgumentDefinition, +) -> LoweredArgumentDefinition: + metavar = lowered.metavar + + def _format(x: str) -> str: + return termcolor.colored(x, attrs=["bold"]) + + if isinstance(metavar, str): + metavar = _format(metavar) + elif isinstance(metavar, tuple): + metavar = tuple(map(_format, metavar)) + else: + assert metavar is None + + return dataclasses.replace(lowered, metavar=metavar) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 70bed8b04..ab0b0cf75 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -7,7 +7,7 @@ from typing_extensions import get_args, get_origin -from . import _fields, _parsers, _resolver, _strings +from . import _arguments, _fields, _parsers, _resolver, _strings class InstantiationError(Exception): @@ -22,7 +22,7 @@ def call_from_args( f: Callable[..., T], parser_definition: _parsers.ParserSpecification, default_instance: Optional[T], - value_from_arg: Dict[str, Any], + value_from_prefixed_field_name: Dict[str, Any], field_name_prefix: str, avoid_subparsers: bool, ) -> Tuple[T, Set[str]]: @@ -36,15 +36,15 @@ def call_from_args( kwargs: Dict[str, Any] = {} consumed_keywords: Set[str] = set() - def get_value_from_arg(arg: str) -> Any: + def get_value_from_arg(prefixed_field_name: str) -> Any: """Helper for getting values from `value_from_arg` + doing some extra asserts.""" - assert arg in value_from_arg - assert arg not in consumed_keywords - consumed_keywords.add(arg) - return value_from_arg[arg] + assert prefixed_field_name in value_from_prefixed_field_name + assert prefixed_field_name not in consumed_keywords + consumed_keywords.add(prefixed_field_name) + return value_from_prefixed_field_name[prefixed_field_name] - arg_from_prefixed_field_name = {} + arg_from_prefixed_field_name: Dict[str, _arguments.ArgumentDefinition] = {} for arg in parser_definition.args: arg_from_prefixed_field_name[arg.prefix + arg.field.name] = arg @@ -62,14 +62,30 @@ def get_value_from_arg(arg: str) -> Any: if prefixed_field_name in arg_from_prefixed_field_name: # Standard arguments. arg = arg_from_prefixed_field_name[prefixed_field_name] - value = get_value_from_arg(prefixed_field_name) - if value is not None: - try: - assert arg.lowered.instantiator is not None - value = arg.lowered.instantiator(value) - except ValueError as e: + if not arg.lowered.is_fixed(): + value = get_value_from_arg(prefixed_field_name) + if value is not None: + try: + assert arg.lowered.instantiator is not None + value = arg.lowered.instantiator(value) + except ValueError as e: + raise InstantiationError( + f"Parsing error for {arg.lowered.name_or_flag}: {e.args[0]}" + ) + else: + assert arg.field.default is not _fields.MISSING + value = arg.field.default + if ( + value_from_prefixed_field_name.get(prefixed_field_name) + is not _fields.MISSING + ): + print( + (type(value_from_prefixed_field_name.get(prefixed_field_name))), + (type(_fields.MISSING)), + ) raise InstantiationError( - f"Parsing error for {arg.lowered.name_or_flag}: {e.args[0]}" + f"{arg.lowered.name_or_flag} was passed in, but is a fixed" + " argument that cannot be parsed" ) elif ( prefixed_field_name @@ -83,7 +99,7 @@ def get_value_from_arg(arg: str) -> Any: field_type, parser_definition, field.default, - value_from_arg, + value_from_prefixed_field_name, field_name_prefix=prefixed_field_name + _strings.NESTED_FIELD_DELIMETER, avoid_subparsers=avoid_subparsers, ) @@ -96,7 +112,7 @@ def get_value_from_arg(arg: str) -> Any: subparser_dest = _strings.SUBPARSER_DEST_FMT.format( name=prefixed_field_name ) - if subparser_dest in value_from_arg: + if subparser_dest in value_from_prefixed_field_name: subparser_name = get_value_from_arg(subparser_dest) else: default_instance = parser_definition.subparsers_from_name[ @@ -135,7 +151,7 @@ def get_value_from_arg(arg: str) -> Any: subparser_name ], field.default if type(field.default) is chosen_f else None, - value_from_arg, + value_from_prefixed_field_name, field_name_prefix=prefixed_field_name + _strings.NESTED_FIELD_DELIMETER, avoid_subparsers=avoid_subparsers, diff --git a/dcargs/_cli.py b/dcargs/_cli.py index eccdee353..52cb97375 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -103,7 +103,7 @@ def cli( formatter_class=argparse.RawTextHelpFormatter, ) parser_definition.apply(parser) - value_from_arg = vars(parser.parse_args(args=args)) + value_from_prefixed_field_name = vars(parser.parse_args(args=args)) try: # Attempt to call `f` using whatever was passed in. @@ -111,7 +111,7 @@ def cli( f, parser_definition, default_instance, - value_from_arg, + value_from_prefixed_field_name, field_name_prefix="", avoid_subparsers=avoid_subparsers, ) diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 4677d4da9..cca3f5bbd 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -20,8 +20,19 @@ class Field: positional: bool -class _MISSING_TYPE: - pass +class _MISSING_TYPE: # pragma: no cover + # Singleton pattern. + # https://www.python.org/download/releases/2.2/descrintro/#__new__ + def __new__(cls, *args, **kwds): + it = cls.__dict__.get("__it__") + if it is not None: + return it + cls.__it__ = it = object.__new__(cls) + it.init(*args, **kwds) + return it + + def init(self, *args, **kwds): + pass MISSING: Any = _MISSING_TYPE() @@ -96,7 +107,7 @@ def field_list_from_callable( if hasattr(default_instance, name): default = getattr(default_instance, name) if default in _missing_types or default_instance in _missing_types: - default = MISSING + default = None field_list.append( Field( @@ -153,6 +164,7 @@ def field_list_from_callable( # Get default value. default = param.default if default in _missing_types: + # TODO: we should _fields.MISSING. default = None # Get helptext from docstring. diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 2569e3226..53c29a773 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -238,32 +238,25 @@ def from_callable( def apply(self, parser: argparse.ArgumentParser) -> None: """Create defined arguments and subparsers.""" + # Generate helptext. parser.description = self.description + # Make argument groups. def format_group_name(nested_field_name: str, required: bool) -> str: - if required: - prefix = termcolor.colored("required", attrs=["bold"]) - else: - prefix = termcolor.colored("optional", attrs=["bold", "dark"]) - suffix = termcolor.colored("arguments", attrs=["bold"]) - + # if required: + # prefix = termcolor.colored("required", attrs=["bold"]) + # else: + # prefix = termcolor.colored("optional", attrs=["bold", "dark"]) if nested_field_name != "": - return " ".join( - [ - prefix, - termcolor.colored(nested_field_name, attrs=["bold"]), - suffix, - ] + return termcolor.colored( + nested_field_name + " arguments", attrs=["bold"] ) else: - return " ".join([prefix, suffix]) + return termcolor.colored("arguments", attrs=["bold"]) - optional_group_from_prefix: Dict[str, argparse._ArgumentGroup] = { + group_from_prefix: Dict[str, argparse._ArgumentGroup] = { "": parser._action_groups[1], } - required_group_from_prefix: Dict[str, argparse._ArgumentGroup] = { - "": parser.add_argument_group(format_group_name("", required=True)), - } # Break some API boundaries to rename the optional group. parser._action_groups[1].title = format_group_name("", required=False) @@ -278,29 +271,15 @@ def format_group_name(nested_field_name: str, required: bool) -> str: arg.add_argument(positional_group) continue - if arg.lowered.required: - target_groups, other_groups = ( - required_group_from_prefix, - optional_group_from_prefix, - ) - else: - target_groups, other_groups = ( - optional_group_from_prefix, - required_group_from_prefix, - ) - - if arg.prefix not in target_groups: + if arg.prefix not in group_from_prefix: nested_field_name = arg.prefix[:-1] - target_groups[arg.prefix] = parser.add_argument_group( + group_from_prefix[arg.prefix] = parser.add_argument_group( format_group_name(nested_field_name, required=arg.lowered.required), - # Add a description, but only to the first group for a field. description=self.helptext_from_nested_class_field_name.get( nested_field_name - ) - if arg.prefix not in other_groups - else None, + ), ) - arg.add_argument(target_groups[arg.prefix]) + arg.add_argument(group_from_prefix[arg.prefix]) # Create subparser tree. if len(self.subparsers_from_name) > 0: diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 6de2bc8b3..834c512ab 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -72,3 +72,12 @@ def instance_from_string(typ: Type[T], arg: str) -> T: return bytes(arg, encoding="ascii") # type: ignore else: return typ(arg) # type: ignore + + +@functools.lru_cache(maxsize=None) +def _strip_regex() -> re.Pattern: + return re.compile(r"\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))") + + +def strip_color_codes(x: str): + return _strip_regex().sub("", x) diff --git a/examples/04_flags.py b/examples/04_flags.py index 40ff68cd6..8d2c9f749 100644 --- a/examples/04_flags.py +++ b/examples/04_flags.py @@ -2,6 +2,7 @@ value, automatically converted to flags. Usage: +`python ./04_flags.py --help` `python ./04_flags.py --boolean True` `python ./04_flags.py --boolean False --flag-a` `python ./04_flags.py --boolean False --no-flag-b` diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 996acede9..05cb7cd86 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -1,7 +1,7 @@ import dataclasses import enum import pathlib -from typing import Any, AnyStr, ClassVar, Optional, TypeVar, Union +from typing import Any, AnyStr, Callable, ClassVar, Optional, TypeVar, Union import pytest from typing_extensions import Annotated, Final, Literal, TypeAlias @@ -411,3 +411,12 @@ def main(x: AnyStr) -> AnyStr: # Use bytes when provided ascii-compatible inputs. assert dcargs.cli(main, args=["--x", "hello"]) == b"hello" assert dcargs.cli(main, args=["--x", "hello„"]) == "hello„" + + +def test_fixed(): + def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]: + return x + + assert dcargs.cli(main, args=[])(3) == 6 + with pytest.raises(SystemExit): + dcargs.cli(main, args=["--x", "something"]) diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index 81857910c..719826966 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -7,6 +7,7 @@ from typing_extensions import Literal, TypedDict import dcargs +import dcargs._strings def test_basic_dict(): @@ -101,11 +102,14 @@ class HelptextTypedDict(TypedDict): with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(HelptextTypedDict, default_instance={"z": 3}, args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert cast(str, HelptextTypedDict.__doc__) in helptext - assert ":\n --x INT Documentation 1\n" in helptext - assert "--y INT Documentation 2\n" in helptext - assert "--z INT Documentation 3 (default: 3)\n" in helptext + assert "--x INT" in helptext + assert "--y INT" in helptext + assert "--z INT" in helptext + assert "Documentation 1 (required)" in helptext + assert "Documentation 2 (required)" in helptext + assert "Documentation 3 (default: 3)\n" in helptext def test_basic_namedtuple(): @@ -161,11 +165,11 @@ class HelptextNamedTupleDefault(NamedTuple): with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(HelptextNamedTupleDefault, args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert cast(str, HelptextNamedTupleDefault.__doc__) in helptext - assert ":\n --x INT Documentation 1\n" in helptext - assert "--y INT Documentation 2\n" in helptext - assert "--z INT Documentation 3 (default: 3)\n" in helptext + assert "--x INT Documentation 1 (required)\n" in helptext + assert "--y INT Documentation 2 (required)\n" in helptext + assert "--z INT Documentation 3 (default: 3)\n" in helptext def test_helptext_and_default_instance_namedtuple(): @@ -193,8 +197,8 @@ class HelptextNamedTuple(NamedTuple): ), args=["--help"], ) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert cast(str, HelptextNamedTuple.__doc__) in helptext - assert ":\n --x INT Documentation 1\n" in helptext - assert "--y INT Documentation 2\n" in helptext - assert "--z INT Documentation 3 (default: 3)\n" in helptext + assert "Documentation 1 (required)\n" in helptext + assert "Documentation 2 (required)\n" in helptext + assert "Documentation 3 (default: 3)\n" in helptext diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 34f64f278..9e654781d 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -9,6 +9,7 @@ from typing_extensions import Literal import dcargs +import dcargs._strings def test_helptext(): @@ -28,11 +29,14 @@ class Helptext: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(Helptext, args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert cast(str, Helptext.__doc__) in helptext - assert ":\n --x INT Documentation 1\n" in helptext - assert "--y INT Documentation 2\n" in helptext - assert "--z INT Documentation 3 (default: 3)\n" in helptext + assert "x INT" in helptext + assert "y INT" in helptext + assert "z INT" in helptext + assert "Documentation 1 (required)\n" in helptext + assert "Documentation 2 (required)\n" in helptext + assert "Documentation 3 (default: 3)\n" in helptext def test_helptext_inherited(): @@ -63,9 +67,9 @@ class ChildClass(UnrelatedParentClass, ActualParentClass): with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(ChildClass, args=["--help"]) - helptext = f.getvalue() - assert ":\n --x INT Documentation 1\n" in helptext - assert "--y INT Documentation 2\n" in helptext + helptext = dcargs._strings.strip_color_codes(f.getvalue()) + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext def test_helptext_nested(): @@ -96,7 +100,7 @@ def main_no_docstring(a: Inner) -> None: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(main_with_docstring, args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert "Documented in function" in helptext and str(Inner.__doc__) not in helptext assert "Args:" not in helptext assert "Hello world!" in helptext @@ -105,7 +109,7 @@ def main_no_docstring(a: Inner) -> None: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(main_no_docstring, args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) print(helptext) assert "Something" in helptext assert "Args:" not in helptext @@ -127,12 +131,11 @@ class HelptextWithVariousDefaults: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(HelptextWithVariousDefaults, args=["--help"]) - helptext = f.getvalue() - assert ( - "show this help message and exit\n --x PATH (default:" - " /some/path/to/a/file)\n" in helptext - ) - assert "--y {RED,GREEN,BLUE} (default: RED)\n" in helptext + helptext = dcargs._strings.strip_color_codes(f.getvalue()) + assert "show this help message and exit\n --x PATH" in helptext + assert "(default: /some/path/to/a/file)\n" in helptext + assert "--y {RED,GREEN,BLUE}" in helptext + assert "(default: RED)" in helptext def test_multiline_helptext(): @@ -154,16 +157,12 @@ class HelptextMultiline: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(HelptextMultiline, args=["--help"]) - helptext = f.getvalue() - assert " --x INT Documentation 1\n" in helptext - assert ( - " --y INT Documentation 2\n Next line of documentation 2\n" - in helptext - ) - assert ( - " --z INT Documentation 3\n Next line of documentation 3" - " (default: 3)\n" in helptext - ) + helptext = dcargs._strings.strip_color_codes(f.getvalue()) + assert "Documentation 1 (required)\n" in helptext + assert "Documentation 2\n" in helptext + assert "Next line of documentation 2 (required)\n" in helptext + assert "Documentation 3\n" in helptext + assert "Next line of documentation 3 (default: 3)\n" in helptext def test_grouped_helptext(): @@ -178,10 +177,10 @@ class HelptextGrouped: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(HelptextGrouped, args=["--help"]) - helptext = f.getvalue() - assert " --x INT Documentation 1\n" in helptext - assert " --y INT Description of both y and z.\n" in helptext - assert " --z INT Description of both y and z. (default: 3)\n" in helptext + helptext = dcargs._strings.strip_color_codes(f.getvalue()) + assert "Documentation 1 (required)\n" in helptext + assert "Description of both y and z. (required)\n" in helptext + assert "Description of both y and z. (default: 3)\n" in helptext def test_none_default_value_helptext(): @@ -194,7 +193,7 @@ class Config: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(Config, args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert " --x None|INT An optional variable. (default: None)\n" in helptext @@ -215,7 +214,7 @@ class HelptextHardString: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(HelptextHardString, args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert "--x Helptext. 2% milk.\n" in helptext @@ -237,10 +236,10 @@ class Child(Parent): with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(Child, args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) + assert "--x STR" in helptext assert ( - "--x STR Helptext. (default: 'This docstring may be tougher to parse!')\n" - in helptext + "Helptext. (default: 'This docstring may be tougher to parse!')\n" in helptext ) @@ -267,10 +266,10 @@ class Child2(Parent2): with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(Child2, args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) + assert "--x STR" in helptext assert ( - "--x STR Helptext! (default: 'This docstring may be tougher to parse?')\n" - in helptext + "Helptext! (default: 'This docstring may be tougher to parse?')\n" in helptext ) @@ -283,7 +282,7 @@ class TupleHelptext: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(TupleHelptext, args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert "--x INT STR FLOAT\n" in helptext @@ -296,8 +295,9 @@ class TupleHelptextDefaults: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(TupleHelptextDefaults, args=["--help"]) - helptext = f.getvalue() - assert "--x INT STR STR (default: 5 'hello world' hello)\n" in helptext + helptext = dcargs._strings.strip_color_codes(f.getvalue()) + assert "--x INT STR STR" in helptext + assert "(default: 5 'hello world' hello)\n" in helptext def test_generic_helptext(): @@ -311,7 +311,7 @@ class GenericTupleHelptext(Generic[T]): with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(GenericTupleHelptext[int], args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert "--x INT\n" in helptext @@ -326,7 +326,7 @@ class GenericTupleHelptext(Generic[T]): with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(GenericTupleHelptext[int], args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert "--x INT INT INT\n" in helptext @@ -341,7 +341,7 @@ class GenericTupleHelptext(Generic[T]): with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(GenericTupleHelptext[int], args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert "--x INT [INT ...]\n" in helptext @@ -355,8 +355,9 @@ class LiteralHelptext: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(LiteralHelptext, args=["--help"]) - helptext = f.getvalue() - assert "--x {1,2,3} A number.\n" in helptext + helptext = dcargs._strings.strip_color_codes(f.getvalue()) + assert "--x {1,2,3}" in helptext + assert "A number. (required)\n" in helptext def test_optional_literal_helptext(): @@ -369,8 +370,9 @@ class OptionalLiteralHelptext: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(OptionalLiteralHelptext, args=["--help"]) - helptext = f.getvalue() - assert "--x None|{1,2,3} A number. (default: None)\n" in helptext + helptext = dcargs._strings.strip_color_codes(f.getvalue()) + assert "--x None|{1,2,3}" in helptext + assert "A number. (default: None)\n" in helptext def test_multiple_subparsers_helptext(): @@ -401,7 +403,7 @@ class MultipleSubparsers: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(MultipleSubparsers, args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert "Field a description." in helptext assert "Field b description." not in helptext @@ -413,7 +415,7 @@ class MultipleSubparsers: dcargs.cli( MultipleSubparsers, args=["subcommand1", "subcommand1", "--help"] ) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert "Field a description." not in helptext assert "Field b description." not in helptext @@ -437,7 +439,7 @@ class OptionalHelptext: with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(OptionalHelptext, args=["--help"]) - helptext = f.getvalue() + helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert cast(str, OptionalHelptext.__doc__) in helptext assert "[--x None|INT]" in helptext assert "--y None|INT [None|INT ...]\n" in helptext diff --git a/tests/test_unsupported_but_should_work.py b/tests/test_unsupported_but_should_work.py index 572cac0c1..2452cc228 100644 --- a/tests/test_unsupported_but_should_work.py +++ b/tests/test_unsupported_but_should_work.py @@ -3,7 +3,6 @@ Includes things like omegaconf.MISSING, attrs, etc, which mostly work but either likely have corner cases or just seem sketchy. """ - import contextlib import io import pathlib @@ -14,6 +13,7 @@ import pytest import dcargs +import dcargs._strings def test_omegaconf_missing(): @@ -123,7 +123,9 @@ class Helptext: with contextlib.redirect_stdout(f): dcargs.cli(Helptext, args=["--help"]) helptext = f.getvalue() - assert cast(str, Helptext.__doc__) in helptext - assert ":\n --x INT Documentation 1\n" in helptext - assert "--y INT Documentation 2\n" in helptext - assert "--z INT Documentation 3 (default: 3)\n" in helptext + assert dcargs._strings.strip_color_codes(cast(str, Helptext.__doc__)) in helptext + + # Note that required detection seems to be broken here. + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + assert "Documentation 3" in helptext From 42c4ad7e60fc68f04fc964432f35238d98dcba62 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 15 Jul 2022 05:40:30 -0700 Subject: [PATCH 058/491] Add `prog` arg, tweak base config pattern --- README.md | 113 +++++++++++++++++------------------- _update_readme.py | 12 +++- dcargs/_arguments.py | 4 +- dcargs/_cli.py | 4 ++ examples/06_base_configs.py | 43 ++++++++------ 5 files changed, 94 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index a529b76cd..99144198e 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,8 @@ Args: f: Callable. Keyword Args: + prog: The name of the program printed in helptext. Mirrors argument from + `argparse.ArgumentParser()`. description: Description text for the parser, displayed when the --help flag is passed in. If not specified, `f`'s docstring is used. Mirrors argument from `argparse.ArgumentParser()`. @@ -585,23 +587,13 @@ one of multiple possible base configurations, and then use the CLI to either ove ```python import dataclasses -import os -from typing import Callable, Literal, Tuple, Union +import importlib +import sys +from typing import Dict, Literal, Tuple, Type, TypeVar, Union import dcargs -# Learning rate schedulers. -def no_lr_scheduler(step: int) -> float: - return 1.0 - - -def linear_warmup_1000_scheduler(step: int) -> float: - assert step >= 0 - return min(1.0, step / 1000.0) - - -# Optimizer configs. @dataclasses.dataclass class AdamOptimizer: # Adam learning rate. @@ -617,7 +609,6 @@ class SgdOptimizer: learning_rate: float = 3e-4 -# Overall experiment config. @dataclasses.dataclass(frozen=True) class ExperimentConfig: # Dataset to run experiment on. @@ -626,11 +617,6 @@ class ExperimentConfig: # Optimizer parameters. optimizer: Union[AdamOptimizer, SgdOptimizer] - # Learning rate scheduler. Fields with types that `dcargs.cli()` does not support - # instantiating from the CLI (such as Callables) will always be kept at their - # default value. - lr_scheduler: Callable[[int], float] - # Model size. num_layers: int units: int @@ -653,7 +639,6 @@ base_config_library = { "small": ExperimentConfig( dataset="mnist", optimizer=SgdOptimizer(), - lr_scheduler=no_lr_scheduler, batch_size=2048, num_layers=4, units=64, @@ -665,7 +650,6 @@ base_config_library = { "big": ExperimentConfig( dataset="imagenet-50", optimizer=AdamOptimizer(), - lr_scheduler=linear_warmup_1000_scheduler, batch_size=32, num_layers=8, units=256, @@ -674,24 +658,32 @@ base_config_library = { ), } -if __name__ == "__main__": - # Get base configuration name from environment. - base_config_name = os.environ.get("BASE_CONFIG") - if base_config_name is None or base_config_name not in base_config_library: - raise SystemExit( - f"BASE_CONFIG should be set to one of {tuple(base_config_library.keys())}" - ) + +T = TypeVar("T") + + +def cli_with_base_configs(cls: Type[T], base_library: Dict[str, T]) -> T: + # Get base configuration name from the first positional argument. + if len(sys.argv) < 2 or sys.argv[1] not in base_library: + valid_usages = map(lambda k: f"{sys.argv[0]} {k} --help", base_library.keys()) + raise SystemExit(f"usage:\n " + "\n ".join(valid_usages)) # Get base configuration from our library, and use it for default CLI parameters. - base_config = base_config_library[base_config_name] - config = dcargs.cli( - ExperimentConfig, - default_instance=base_config, + default_instance = base_library[sys.argv[1]] + return dcargs.cli( + cls, + prog=" ".join(sys.argv[:2]), + args=sys.argv[2:], + default_instance=default_instance, # `avoid_subparsers` will avoid making a subparser for unions when a default is # provided; in this case, it simplifies our CLI but makes it less expressive # (cannot switch away from the base optimizer types). avoid_subparsers=True, ) + + +if __name__ == "__main__": + config = cli_with_base_configs(ExperimentConfig, base_config_library) print(config) ``` @@ -700,22 +692,27 @@ if __name__ == "__main__": **Example usage:** 
-$ BASE_CONFIG=small python ./06_base_configs.py --help
-usage: 06_base_configs.py [-h] [--dataset {mnist,imagenet-50}]
-                          [--optimizer.learning-rate FLOAT]
-                          [--lr-scheduler fixed]
-                          [--num-layers INT] [--units INT]
-                          [--batch-size INT]
-                          [--train-steps INT] --seed INT
+$ python ./06_base_configs_argv.py
+usage:
+  examples/06_base_configs.py small --help
+  examples/06_base_configs.py big --help
+
+ +
+$ python ./06_base_configs_argv.py small --help
+usage: examples/06_base_configs.py small [-h]
+                                         [--dataset {mnist,imagenet-50}]
+                                         [--optimizer.learning-rate FLOAT]
+                                         [--num-layers INT]
+                                         [--units INT]
+                                         [--batch-size INT]
+                                         [--train-steps INT] --seed
+                                         INT
 
 arguments:
   -h, --help            show this help message and exit
   --dataset {mnist,imagenet-50}
                         Dataset to run experiment on. (default: mnist)
-  --lr-scheduler (fixed)
-                        Learning rate scheduler. Fields with types that `dcargs.cli()` does not support
-                        instantiating from the CLI (such as Callables) will always be kept at their
-                        default value. (value: '<function no_lr_scheduler at 0x7fb5254763a0>')
   --num-layers INT
                         Model size. (default: 4)
   --units INT   Model size. (default: 64)
@@ -734,28 +731,26 @@ optimizer arguments:
 
-$ BASE_CONFIG=small python ./06_base_configs.py --seed 94720
-ExperimentConfig(dataset='mnist', optimizer=SgdOptimizer(learning_rate=0.0003), lr_scheduler=<function no_lr_scheduler at 0x7fc74a7733a0>, num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720)
+$ python ./06_base_configs_argv.py small --seed 94720
+ExperimentConfig(dataset='mnist', optimizer=SgdOptimizer(learning_rate=0.0003), num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720)
 
-$ BASE_CONFIG=big python ./06_base_configs.py --help
-usage: 06_base_configs.py [-h] [--dataset {mnist,imagenet-50}]
-                          [--optimizer.learning-rate FLOAT]
-                          [--optimizer.betas FLOAT FLOAT]
-                          [--lr-scheduler fixed]
-                          [--num-layers INT] [--units INT]
-                          [--batch-size INT]
-                          [--train-steps INT] --seed INT
+$ python ./06_base_configs_argv.py big --help
+usage: examples/06_base_configs.py big [-h]
+                                       [--dataset {mnist,imagenet-50}]
+                                       [--optimizer.learning-rate FLOAT]
+                                       [--optimizer.betas FLOAT FLOAT]
+                                       [--num-layers INT]
+                                       [--units INT]
+                                       [--batch-size INT]
+                                       [--train-steps INT] --seed
+                                       INT
 
 arguments:
   -h, --help            show this help message and exit
   --dataset {mnist,imagenet-50}
                         Dataset to run experiment on. (default: imagenet-50)
-  --lr-scheduler (fixed)
-                        Learning rate scheduler. Fields with types that `dcargs.cli()` does not support
-                        instantiating from the CLI (such as Callables) will always be kept at their
-                        default value. (value: '<function linear_warmup_1000_scheduler at 0x7ff2be2a0dc0>')
   --num-layers INT
                         Model size. (default: 8)
   --units INT   Model size. (default: 256)
@@ -776,8 +771,8 @@ optimizer arguments:
 
-$ BASE_CONFIG=big python ./06_base_configs.py --seed 94720
-ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learning_rate=0.001, betas=(0.9, 0.999)), lr_scheduler=<function linear_warmup_1000_scheduler at 0x7fb97f898dc0>, num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720)
+$ python ./06_base_configs_argv.py big --seed 94720
+ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learning_rate=0.001, betas=(0.9, 0.999)), num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720)
diff --git a/_update_readme.py b/_update_readme.py index 1928d370f..304633f0c 100644 --- a/_update_readme.py +++ b/_update_readme.py @@ -68,13 +68,19 @@ def format_script_for_readme(path: pathlib.Path) -> str: env_vars = { k: v for (k, v) in map(lambda x: x.split("="), args[:python_index]) } - output = subprocess.run( + process_output = subprocess.run( args=["python", str(path)] + args[python_index + 2 :], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, encoding="utf8", env=dict(os.environ, **env_vars), - ).stdout - output = dcargs._strings.strip_color_codes(output).strip() + ) + if process_output.stderr != "": + output = dcargs._strings.strip_color_codes(process_output.stderr).strip() + elif process_output.stdout != "": + output = dcargs._strings.strip_color_codes(process_output.stdout).strip() + else: + assert False example_output_lines.extend( [ "", diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 4443dd35f..737f16e2e 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -253,9 +253,7 @@ def _rule_generate_helptext( default_text = f"({default_label}: {shlex.quote(str(default))})" help_parts.append(default_text) else: - help_parts.append( - termcolor.colored("(required)", on_color="on_red") - ) + help_parts.append(termcolor.colored("(required)", on_color="on_red")) return dataclasses.replace(lowered, help=" ".join(help_parts)) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 52cb97375..97ef04b4e 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -28,6 +28,7 @@ def parse( def cli( f: Callable[..., T], *, + prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[T] = None, @@ -71,6 +72,8 @@ def cli( f: Callable. Keyword Args: + prog: The name of the program printed in helptext. Mirrors argument from + `argparse.ArgumentParser()`. description: Description text for the parser, displayed when the --help flag is passed in. If not specified, `f`'s docstring is used. Mirrors argument from `argparse.ArgumentParser()`. @@ -100,6 +103,7 @@ def cli( # Parse using argparse! parser = argparse.ArgumentParser( + prog=prog, formatter_class=argparse.RawTextHelpFormatter, ) parser_definition.apply(parser) diff --git a/examples/06_base_configs.py b/examples/06_base_configs.py index 60da22687..61458da6a 100644 --- a/examples/06_base_configs.py +++ b/examples/06_base_configs.py @@ -3,15 +3,16 @@ (existing) or fill in (missing) values. Usage: -`BASE_CONFIG=small python ./06_base_configs.py --help` -`BASE_CONFIG=small python ./06_base_configs.py --seed 94720` -`BASE_CONFIG=big python ./06_base_configs.py --help` -`BASE_CONFIG=big python ./06_base_configs.py --seed 94720` +`python ./06_base_configs_argv.py` +`python ./06_base_configs_argv.py small --help` +`python ./06_base_configs_argv.py small --seed 94720` +`python ./06_base_configs_argv.py big --help` +`python ./06_base_configs_argv.py big --seed 94720` """ import dataclasses -import os -from typing import Literal, Tuple, Union +import sys +from typing import Dict, Literal, Tuple, Type, TypeVar, Union import dcargs @@ -80,22 +81,30 @@ class ExperimentConfig: ), } -if __name__ == "__main__": - # Get base configuration name from environment. - base_config_name = os.environ.get("BASE_CONFIG") - if base_config_name is None or base_config_name not in base_config_library: - raise SystemExit( - f"BASE_CONFIG should be set to one of {tuple(base_config_library.keys())}" - ) + +T = TypeVar("T") + + +def cli_with_base_configs(cls: Type[T], base_library: Dict[str, T]) -> T: + # Get base configuration name from the first positional argument. + if len(sys.argv) < 2 or sys.argv[1] not in base_library: + valid_usages = map(lambda k: f"{sys.argv[0]} {k} --help", base_library.keys()) + raise SystemExit("usage:\n " + "\n ".join(valid_usages)) # Get base configuration from our library, and use it for default CLI parameters. - base_config = base_config_library[base_config_name] - config = dcargs.cli( - ExperimentConfig, - default_instance=base_config, + default_instance = base_library[sys.argv[1]] + return dcargs.cli( + cls, + prog=" ".join(sys.argv[:2]), + args=sys.argv[2:], + default_instance=default_instance, # `avoid_subparsers` will avoid making a subparser for unions when a default is # provided; in this case, it simplifies our CLI but makes it less expressive # (cannot switch away from the base optimizer types). avoid_subparsers=True, ) + + +if __name__ == "__main__": + config = cli_with_base_configs(ExperimentConfig, base_config_library) print(config) From 0d3ed6df625a95f1ecd0c5ab9c3abd62aca5aa21 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 15 Jul 2022 17:37:19 -0700 Subject: [PATCH 059/491] Polish --- README.md | 217 ++++++++++++++++++------------------ _update_readme.py | 5 +- dcargs/_arguments.py | 38 ++----- dcargs/_calling.py | 5 +- dcargs/_cli.py | 9 +- dcargs/_fields.py | 2 +- dcargs/_instantiators.py | 73 ++++++------ dcargs/_parsers.py | 19 +--- examples/06_base_configs.py | 22 ++-- examples/09_subparsers.py | 2 - setup.py | 3 +- tests/test_dcargs.py | 9 +- tests/test_helptext.py | 28 +++-- 13 files changed, 209 insertions(+), 223 deletions(-) diff --git a/README.md b/README.md index 99144198e..0bd0376cc 100644 --- a/README.md +++ b/README.md @@ -132,8 +132,8 @@ Ultimately, we aim to enable configuration interfaces that are: 1. Functions +

- In the simplest case, `dcargs.cli()` can be used to run a function with arguments populated from the CLI. @@ -168,8 +168,7 @@ if __name__ == "__main__": 
 $ python ./01_functions.py --help
-usage: 01_functions.py [-h] --field1 STR [--field2 INT]
-                       [--flag]
+usage: 01_functions.py [-h] --field1 STR [--field2 INT] [--flag]
 
 Function, whose arguments will be populated from a CLI interface.
 
@@ -177,7 +176,7 @@ arguments:
   -h, --help            show this help message and exit
   --field1 STR  A string field. (required)
   --field2 INT  A numeric field, with a default value. (default: 3)
-  --flag                A boolean flag.
+  --flag                A boolean flag. (default: flag=False)
 
@@ -190,14 +189,15 @@ hello 3 False
 hello 3 True
+
2. Dataclasses +

- Common pattern: use `dcargs.cli()` to instantiate a dataclass. **Code ([link](examples/02_dataclasses.py)):** @@ -229,17 +229,15 @@ if __name__ == "__main__": 
 $ python ./02_dataclasses.py --help
-usage: 02_dataclasses.py [-h] --field1 STR [--field2 INT]
-                         [--flag]
+usage: 02_dataclasses.py [-h] --field1 STR [--field2 INT] [--flag]
 
-Description.
-This should show up in the helptext!
+Description. This should show up in the helptext!
 
 arguments:
   -h, --help            show this help message and exit
   --field1 STR  A string field. (required)
   --field2 INT  A numeric field, with a default value. (default: 3)
-  --flag                A boolean flag.
+  --flag                A boolean flag. (default: flag=False)
 
@@ -252,14 +250,15 @@ Args(field1='hello', field2=3, flag=False)
 Args(field1='hello', field2=3, flag=True)
+
3. Enums And Containers +

- We can generate argument parsers from more advanced type annotations, like enums and tuple types. @@ -310,21 +309,21 @@ if __name__ == "__main__": 
 $ python ./03_enums_and_containers.py --help
-usage: 03_enums_and_containers.py [-h] --dataset-sources PATH
-                                  [PATH ...]
+usage: 03_enums_and_containers.py [-h] --dataset-sources PATH [PATH ...]
                                   [--image-dimensions INT INT]
                                   [--optimizer-type {ADAM,SGD}]
-                                  [--checkpoint-interval None|INT]
+                                  [--checkpoint-interval {None}|INT]
 
 arguments:
   -h, --help            show this help message and exit
   --dataset-sources PATH [PATH ...]
-                        Paths to load training data from. This can be multiple! (required)
+                        Paths to load training data from. This can be
+                        multiple! (required)
   --image-dimensions INT INT
                         Height and width of some image data. (default: 32 32)
   --optimizer-type {ADAM,SGD}
                         Gradient-based optimizer to use. (default: ADAM)
-  --checkpoint-interval None|INT
+  --checkpoint-interval {None}|INT
                         Interval to save checkpoints at. (default: None)
@@ -338,14 +337,15 @@ TrainConfig(dataset_sources=(PosixPath('data'),), image_dimensions=(16 TrainConfig(dataset_sources=(PosixPath('data'),), image_dimensions=(32, 32), optimizer_type=<OptimizerType.SGD: 2>, checkpoint_interval=None) +
4. Flags +

- Booleans can either be expected to be explicitly passed in, or, if given a default value, automatically converted to flags. @@ -385,17 +385,21 @@ if __name__ == "__main__": 
 $ python ./04_flags.py --help
 usage: 04_flags.py [-h] --boolean {True,False}
-                   [--optional-boolean None|{True,False}] [--flag-a]
+                   [--optional-boolean {None,True,False}] [--flag-a]
                    [--no-flag-b]
 
 arguments:
   -h, --help            show this help message and exit
   --boolean {True,False}
-                        Boolean. This expects an explicit "True" or "False". (required)
-  --optional-boolean None|{True,False}
-                        Optional boolean. Same as above, but can be omitted. (default: None)
-  --flag-a              Pass --flag-a in to set this value to True.
-  --no-flag-b           Pass --no-flag-b in to set this value to False. (default: True)
+                        Boolean. This expects an explicit "True" or "False".
+                        (required)
+  --optional-boolean {None,True,False}
+                        Optional boolean. Same as above, but can be omitted.
+                        (default: None)
+  --flag-a              Pass --flag-a in to set this value to True. (default:
+                        flag_a=False)
+  --no-flag-b           Pass --no-flag-b in to set this value to False.
+                        (default: flag_b=True)
 
@@ -413,14 +417,15 @@ Args(boolean=False, optional_boolean=None, flag_a=True, flag_b=True)
 Args(boolean=False, optional_boolean=None, flag_a=False, flag_b=False)
+
5. Hierarchical Configs +

- Parsing of nested types (in this case nested dataclasses) enables hierarchical configuration objects that are both modular and highly expressive. @@ -503,21 +508,22 @@ usage: 05_hierarchical_configs.py [-h] [--config.optimizer.weight-decay FLOAT] [--config.batch-size INT] [--config.train-steps INT] - [--config.seed INT] - [--restore-checkpoint] + [--config.seed INT] [--restore-checkpoint] [--checkpoint-interval INT] OUT_DIR Train a model. positional arguments: - OUT_DIR Where to save logs and checkpoints. (required) + OUT_DIR Where to save logs and checkpoints. (required) arguments: -h, --help show this help message and exit - --restore-checkpoint Set to restore an existing checkpoint. + --restore-checkpoint Set to restore an existing checkpoint. (default: + restore_checkpoint=False) --checkpoint-interval INT - Training steps between each checkpoint save. (default: 1000) + Training steps between each checkpoint save. (default: + 1000) config.optimizer arguments: Various configurable options for our optimizer. @@ -537,8 +543,8 @@ config arguments: --config.train-steps INT Total number of training steps. (default: 100000) --config.seed INT - Random seed. This is helpful for making sure that our experiments are all - reproducible! (default: 0) + Random seed. This is helpful for making sure that our + experiments are all reproducible! (default: 0) 
@@ -571,14 +577,15 @@ seed: 0
 train_steps: 100000
+
6. Base Configs +

- We can integrate `dcargs.cli()` into common configuration patterns: here, we select one of multiple possible base configurations, and then use the CLI to either override (existing) or fill in (missing) values. @@ -586,30 +593,25 @@ one of multiple possible base configurations, and then use the CLI to either ove **Code ([link](examples/06_base_configs.py)):** ```python -import dataclasses -import importlib import sys +from dataclasses import dataclass from typing import Dict, Literal, Tuple, Type, TypeVar, Union import dcargs -@dataclasses.dataclass +@dataclass(frozen=True) class AdamOptimizer: - # Adam learning rate. learning_rate: float = 1e-3 - - # Moving average parameters. betas: Tuple[float, float] = (0.9, 0.999) -@dataclasses.dataclass +@dataclass(frozen=True) class SgdOptimizer: - # SGD learning rate. learning_rate: float = 3e-4 -@dataclasses.dataclass(frozen=True) +@dataclass(frozen=True) class ExperimentConfig: # Dataset to run experiment on. dataset: Literal["mnist", "imagenet-50"] @@ -663,10 +665,12 @@ T = TypeVar("T") def cli_with_base_configs(cls: Type[T], base_library: Dict[str, T]) -> T: + """Populate an instance of `cls`, using a CLI lets the user select from a library of + base configs.""" # Get base configuration name from the first positional argument. if len(sys.argv) < 2 or sys.argv[1] not in base_library: valid_usages = map(lambda k: f"{sys.argv[0]} {k} --help", base_library.keys()) - raise SystemExit(f"usage:\n " + "\n ".join(valid_usages)) + raise SystemExit("usage:\n " + "\n ".join(valid_usages)) # Get base configuration from our library, and use it for default CLI parameters. default_instance = base_library[sys.argv[1]] @@ -700,14 +704,11 @@ usage: 
 $ python ./06_base_configs_argv.py small --help
-usage: examples/06_base_configs.py small [-h]
-                                         [--dataset {mnist,imagenet-50}]
+usage: examples/06_base_configs.py small [-h] [--dataset {mnist,imagenet-50}]
                                          [--optimizer.learning-rate FLOAT]
-                                         [--num-layers INT]
-                                         [--units INT]
+                                         [--num-layers INT] [--units INT]
                                          [--batch-size INT]
-                                         [--train-steps INT] --seed
-                                         INT
+                                         [--train-steps INT] --seed INT
 
 arguments:
   -h, --help            show this help message and exit
@@ -720,14 +721,14 @@ arguments:
                         Batch size. (default: 2048)
   --train-steps INT
                         Total number of training steps. (default: 30000)
-  --seed INT    Random seed. This is helpful for making sure that our experiments are all
-                        reproducible! (required)
+  --seed INT    Random seed. This is helpful for making sure that our
+                        experiments are all reproducible! (required)
 
 optimizer arguments:
   Optimizer parameters.
 
   --optimizer.learning-rate FLOAT
-                        SGD learning rate. (default: 0.0003)
+                        (default: 0.0003)
 
@@ -737,15 +738,12 @@ ExperimentConfig(dataset='mnist', optimizer=SgdOptimizer(learning_rate
 
 
 $ python ./06_base_configs_argv.py big --help
-usage: examples/06_base_configs.py big [-h]
-                                       [--dataset {mnist,imagenet-50}]
+usage: examples/06_base_configs.py big [-h] [--dataset {mnist,imagenet-50}]
                                        [--optimizer.learning-rate FLOAT]
                                        [--optimizer.betas FLOAT FLOAT]
-                                       [--num-layers INT]
-                                       [--units INT]
-                                       [--batch-size INT]
-                                       [--train-steps INT] --seed
-                                       INT
+                                       [--num-layers INT] [--units INT]
+                                       [--batch-size INT] [--train-steps INT]
+                                       --seed INT
 
 arguments:
   -h, --help            show this help message and exit
@@ -758,16 +756,16 @@ arguments:
                         Batch size. (default: 32)
   --train-steps INT
                         Total number of training steps. (default: 100000)
-  --seed INT    Random seed. This is helpful for making sure that our experiments are all
-                        reproducible! (required)
+  --seed INT    Random seed. This is helpful for making sure that our
+                        experiments are all reproducible! (required)
 
 optimizer arguments:
   Optimizer parameters.
 
   --optimizer.learning-rate FLOAT
-                        Adam learning rate. (default: 0.001)
+                        (default: 0.001)
   --optimizer.betas FLOAT FLOAT
-                        Moving average parameters. (default: 0.9 0.999)
+                        (default: 0.9 0.999)
 

 
 
@@ -775,14 +773,15 @@ optimizer arguments:
 ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learning_rate=0.001, betas=(0.9, 0.999)), num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720)
 

 
+
7. Literals And Unions +

- `typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices; `typing.Union[]` can be used to restrict inputs to a fixed set of types. @@ -828,8 +827,7 @@ if __name__ == "__main__": $ python ./07_literals_and_unions.py --help usage: 07_literals_and_unions.py [-h] [--enum {RED,GREEN,BLUE}] [--restricted-enum {RED,GREEN}] - [--integer {0,1,2,3}] - [--string {red,green}] + [--integer {0,1,2,3}] [--string {red,green}] [--string-or-enum {red,green,RED,GREEN,BLUE}] [--tuple-of-string-or-enum {red,green,RED,GREEN,BLUE} [{red,green,RED,GREEN,BLUE} ...]] @@ -869,14 +867,15 @@ Args(enum=<Color.RED: 1>, restricted_enum=<Color.RED: 1>, integer=0, Args(enum=<Color.RED: 1>, restricted_enum=<Color.RED: 1>, integer=0, string='red', string_or_enum='red', tuple_of_string_or_enum=(<Color.RED: 1>, 'green', <Color.BLUE: 3>)) +
8. Positional Args +

- Positional-only arguments in functions are converted to positional CLI arguments. **Code ([link](examples/08_positional_args.py)):** @@ -944,24 +943,25 @@ if __name__ == "__main__": $ python ./08_positional_args.py --help usage: 08_positional_args.py [-h] [--optimizer.algorithm {ADAM,SGD}] [--optimizer.learning-rate FLOAT] - [--optimizer.weight-decay FLOAT] - [--force] [--verbose] - [--background-rgb FLOAT FLOAT FLOAT] + [--optimizer.weight-decay FLOAT] [--force] + [--verbose] [--background-rgb FLOAT FLOAT FLOAT] SOURCE DEST Command-line interface defined using a function signature. Note that this docstring is parsed to generate helptext. positional arguments: - SOURCE Source path. (required) - DEST Destination path. (required) + SOURCE Source path. (required) + DEST Destination path. (required) arguments: -h, --help show this help message and exit - --force Do not prompt before overwriting. - --verbose Explain what is being done. + --force Do not prompt before overwriting. (default: + force=False) + --verbose Explain what is being done. (default: verbose=False) --background-rgb FLOAT FLOAT FLOAT - Background color. Red by default. (default: 1.0 0.0 0.0) + Background color. Red by default. (default: 1.0 0.0 + 0.0) optimizer arguments: Configuration for our optimizer object. @@ -984,14 +984,15 @@ verbose=False background_rgb=(1.0, 0.0, 0.0) +
9. Subparsers +

- Unions over nested types (classes or dataclasses) are populated using subparsers. **Code ([link](examples/09_subparsers.py)):** @@ -1008,14 +1009,12 @@ import dcargs @dataclasses.dataclass(frozen=True) class Checkout: """Checkout a branch.""" - branch: str @dataclasses.dataclass(frozen=True) class Commit: """Commit changes.""" - message: str all: bool = False @@ -1056,7 +1055,7 @@ arguments: cmd arguments: --cmd.message STR (required) - --cmd.all + --cmd.all (default: all=False) 
@@ -1083,14 +1082,15 @@ cmd arguments:
 Checkout(branch='main')
+
10. Multiple Subparsers +

- Multiple unions over nested types are populated using a series of subparsers. **Code ([link](examples/10_multiple_subparsers.py)):** @@ -1176,7 +1176,7 @@ arguments: -h, --help show this help message and exit optional subcommands: - Dataset to train on. (default: mnist-dataset) + Dataset to train on. (default: mnist-dataset) [{mnist-dataset,image-net-dataset}] @@ -1192,6 +1192,7 @@ arguments: dataset arguments: --dataset.binary Set to load binary version of MNIST dataset. + (default: binary=False) optional subcommands: Optimizer to train with. (default: adam-optimizer) @@ -1205,14 +1206,15 @@ MnistDataset(binary=False) AdamOptimizer(learning_rate=0.0003, betas=(0.9, 0.999)) +
11. Dictionaries +

- Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation, or a `TypedDict` type. @@ -1258,10 +1260,8 @@ if __name__ == "__main__": 
 $ python ./11_dictionaries.py --help
 usage: 11_dictionaries.py [-h] --standard-dict STR {True,False}
-                          [STR {True,False} ...]
-                          [--typed-dict.field1 STR]
-                          [--typed-dict.field2 INT]
-                          [--typed-dict.field3]
+                          [STR {True,False} ...] [--typed-dict.field1 STR]
+                          [--typed-dict.field2 INT] [--typed-dict.field3]
 
 arguments:
   -h, --help            show this help message and exit
@@ -1274,7 +1274,7 @@ typed_dict arguments:
                         A string field. (default: hey)
   --typed-dict.field2 INT
                         A numeric field. (default: 3)
-  --typed-dict.field3   A boolean field.
+  --typed-dict.field3   A boolean field. (default: field3=False)
 
@@ -1283,14 +1283,15 @@ Standard dict: {'key1': True, 'key2': False}
 Typed dict: {'field1': 'hey', 'field2': 3, 'field3': False}
+
12. Named Tuples +

- Example using `dcargs.cli()` to instantiate a named tuple. **Code ([link](examples/12_named_tuples.py)):** @@ -1322,17 +1323,15 @@ if __name__ == "__main__": 
 $ python ./12_named_tuples.py --help
-usage: 12_named_tuples.py [-h] --field1 STR [--field2 INT]
-                          [--flag]
+usage: 12_named_tuples.py [-h] --field1 STR [--field2 INT] [--flag]
 
-Description.
-This should show up in the helptext!
+Description. This should show up in the helptext!
 
 arguments:
   -h, --help            show this help message and exit
   --field1 STR  A string field. (required)
   --field2 INT  A numeric field, with a default value. (default: 3)
-  --flag                A boolean flag.
+  --flag                A boolean flag. (default: flag=False)
 
@@ -1340,14 +1339,15 @@ arguments:
 TupleType(field1='hello', field2=3, flag=False)
+
13. Standard Classes +

- In addition to functions and dataclasses, we can also generate CLIs from (the constructors of) standard Python classes. @@ -1385,8 +1385,7 @@ if __name__ == "__main__": 
 $ python ./13_standard_classes.py --help
-usage: 13_standard_classes.py [-h] --field1 STR --field2 INT
-                              [--flag]
+usage: 13_standard_classes.py [-h] --field1 STR --field2 INT [--flag]
 
 Arguments.
 
@@ -1394,7 +1393,7 @@ arguments:
   -h, --help            show this help message and exit
   --field1 STR  A string field. (required)
   --field2 INT  A numeric field. (required)
-  --flag                A boolean flag.
+  --flag                A boolean flag. (default: flag=False)
 
@@ -1402,14 +1401,15 @@ arguments:
 ['hello', 7, False]
+
14. Generics +

- Example of parsing for generic dataclasses. **Code ([link](examples/14_generics.py)):** @@ -1457,18 +1457,16 @@ if __name__ == "__main__": 
 $ python ./14_generics.py --help
-usage: 14_generics.py [-h] --point-continuous.x FLOAT
-                      --point-continuous.y FLOAT --point-continuous.z
-                      FLOAT --point-continuous.frame-id STR
-                      --point-discrete.x INT --point-discrete.y
-                      INT --point-discrete.z INT
-                      --point-discrete.frame-id STR --shape.a.x
-                      FLOAT --shape.a.y FLOAT --shape.a.z
-                      FLOAT --shape.a.frame-id STR --shape.b.x
-                      FLOAT --shape.b.y FLOAT --shape.b.z
-                      FLOAT --shape.b.frame-id STR --shape.c.x
-                      FLOAT --shape.c.y FLOAT --shape.c.z
-                      FLOAT --shape.c.frame-id STR
+usage: 14_generics.py [-h] --point-continuous.x FLOAT --point-continuous.y
+                      FLOAT --point-continuous.z FLOAT
+                      --point-continuous.frame-id STR --point-discrete.x INT
+                      --point-discrete.y INT --point-discrete.z INT
+                      --point-discrete.frame-id STR --shape.a.x FLOAT
+                      --shape.a.y FLOAT --shape.a.z FLOAT --shape.a.frame-id
+                      STR --shape.b.x FLOAT --shape.b.y FLOAT --shape.b.z
+                      FLOAT --shape.b.frame-id STR --shape.c.x FLOAT
+                      --shape.c.y FLOAT --shape.c.z FLOAT --shape.c.frame-id
+                      STR
 
 arguments:
   -h, --help            show this help message and exit
@@ -1529,6 +1527,7 @@ shape.c arguments:
                         (required)
+
## Serialization diff --git a/_update_readme.py b/_update_readme.py index 304633f0c..7c0971d69 100644 --- a/_update_readme.py +++ b/_update_readme.py @@ -98,10 +98,12 @@ def format_script_for_readme(path: pathlib.Path) -> str: "", f"{index}. {title}", "", + "
", "
", - "", description_text, "", + "\n".join(example_usages), + "", f"**Code ([link]({path})):**", "", "```python", @@ -116,6 +118,7 @@ def format_script_for_readme(path: pathlib.Path) -> str: + example_output_lines + [ "", + "
", "", ] ) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 737f16e2e..4cf9a966f 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -53,7 +53,7 @@ def lowered(self) -> LoweredArgumentDefinition: _rule_generate_helptext, _rule_set_name_or_flag, _rule_positional_special_handling, - _rule_bold_metavar, + # _rule_bold_metavar, ) return functools.reduce( lambda lowered, rule: rule(self, lowered), @@ -159,7 +159,7 @@ def _rule_recursive_instantiator_from_type( # available. return dataclasses.replace( lowered, - metavar=termcolor.colored("(fixed)", color="red"), + metavar=termcolor.colored("(not parsable)", color="red"), required=False, default=_fields.MISSING, ) @@ -231,15 +231,16 @@ def _rule_generate_helptext( assert default is _fields.MISSING default = arg.field.default - if lowered.action == "store_true": - # Don't show defaults for boolean flags. - assert lowered.action in ("store_true", "store_false") - elif not lowered.required: + if not lowered.required: default_label = "value" if lowered.is_fixed() else "default" # Include default value in helptext. We intentionally don't use the % template # because the types of all arguments are set to strings, which will cause the # default to be casted to a string and introduce extra quotation marks. - if lowered.nargs is not None and hasattr(default, "__iter__"): + if lowered.action == "store_true": + default_text = f"(default: {arg.field.name}=False)" + elif lowered.action == "store_false": + default_text = f"(default: {arg.field.name}=True)" + elif lowered.nargs is not None and hasattr(default, "__iter__"): # For tuple types, we might have default as (0, 1, 2, 3). # For list types, we might have default as [0, 1, 2, 3]. # For set types, we might have default as {0, 1, 2, 3}. @@ -251,9 +252,9 @@ def _rule_generate_helptext( default_text = f"({default_label}: {' '.join(default_parts)})" else: default_text = f"({default_label}: {shlex.quote(str(default))})" - help_parts.append(default_text) + help_parts.append(termcolor.colored(default_text, attrs=["dark"])) else: - help_parts.append(termcolor.colored("(required)", on_color="on_red")) + help_parts.append(termcolor.colored("(required)", color="red", attrs=["bold"])) return dataclasses.replace(lowered, help=" ".join(help_parts)) @@ -292,22 +293,3 @@ def _rule_positional_special_handling( required=None, nargs="?" if not lowered.required else lowered.nargs, ) - - -def _rule_bold_metavar( - arg: ArgumentDefinition, - lowered: LoweredArgumentDefinition, -) -> LoweredArgumentDefinition: - metavar = lowered.metavar - - def _format(x: str) -> str: - return termcolor.colored(x, attrs=["bold"]) - - if isinstance(metavar, str): - metavar = _format(metavar) - elif isinstance(metavar, tuple): - metavar = tuple(map(_format, metavar)) - else: - assert metavar is None - - return dataclasses.replace(lowered, metavar=metavar) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index ab0b0cf75..0be5e9916 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -40,8 +40,6 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: """Helper for getting values from `value_from_arg` + doing some extra asserts.""" assert prefixed_field_name in value_from_prefixed_field_name - assert prefixed_field_name not in consumed_keywords - consumed_keywords.add(prefixed_field_name) return value_from_prefixed_field_name[prefixed_field_name] arg_from_prefixed_field_name: Dict[str, _arguments.ArgumentDefinition] = {} @@ -60,8 +58,11 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: ) if prefixed_field_name in arg_from_prefixed_field_name: + assert prefixed_field_name not in consumed_keywords + # Standard arguments. arg = arg_from_prefixed_field_name[prefixed_field_name] + consumed_keywords.add(prefixed_field_name) if not arg.lowered.is_fixed(): value = get_value_from_arg(prefixed_field_name) if value is not None: diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 97ef04b4e..5d8a6df25 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -1,6 +1,8 @@ import argparse from typing import Callable, Optional, Sequence, TypeVar +import argparse_color_formatter + from . import _calling, _parsers T = TypeVar("T") @@ -103,8 +105,7 @@ def cli( # Parse using argparse! parser = argparse.ArgumentParser( - prog=prog, - formatter_class=argparse.RawTextHelpFormatter, + prog=prog, formatter_class=argparse_color_formatter.ColorHelpFormatter ) parser_definition.apply(parser) value_from_prefixed_field_name = vars(parser.parse_args(args=args)) @@ -126,7 +127,5 @@ def cli( print(e.args[0]) raise SystemExit() - # print(consumed_keywords) - # print(value_from_arg.keys()) - # assert consumed_keywords == value_from_arg.keys() + # assert consumed_keywords == value_from_prefixed_field_name.keys() return out diff --git a/dcargs/_fields.py b/dcargs/_fields.py index cca3f5bbd..84a224396 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -22,7 +22,7 @@ class Field: class _MISSING_TYPE: # pragma: no cover # Singleton pattern. - # https://www.python.org/download/releases/2.2/descrintro/#__new__ + # https://www.python.org/download/releases/2.2/descrintro/#__new__ def __new__(cls, *args, **kwds): it = cls.__dict__.get("__it__") if it is not None: diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index 3abb9e6f2..90867ae10 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -37,7 +37,6 @@ import dataclasses import enum import inspect -import warnings from collections import deque from typing import ( Any, @@ -55,6 +54,7 @@ overload, ) +import termcolor from typing_extensions import Annotated, Final, Literal, get_args, get_origin from . import _strings @@ -93,6 +93,10 @@ class UnsupportedTypeAnnotationError(Exception): ) +def _format_metavar(x: str) -> str: + return termcolor.colored(x, attrs=["bold"]) + + def instantiator_from_type( typ: Type, type_from_typevar: Dict[TypeVar, Type] ) -> Tuple[Instantiator, InstantiatorMetadata]: @@ -111,23 +115,21 @@ def instantiator_from_type( type_from_typevar, ) + # Handle Any. if typ is Any: - warnings.warn("Found field with type `Any`, which will be parsed as a string.") - return lambda arg: arg, InstantiatorMetadata( - nargs=None, - metavar="ANY", - choices=None, - is_optional=False, - ) + raise UnsupportedTypeAnnotationError("`Any` is not a parsable type.") + # Handle NoneType. if typ is NoneType: - def instantiator(_unused_string: str) -> None: + def instantiator(string: str) -> None: + if string != "None": + raise ValueError("Only valid argument is `None`.") return None return instantiator, InstantiatorMetadata( nargs=None, - metavar="None", + metavar="{" + _format_metavar("None") + "}", choices=("None",), is_optional=False, ) @@ -149,28 +151,35 @@ def instantiator(_unused_string: str) -> None: else: param_count = 0 has_var_positional = False - signature = inspect.signature(typ) - for i, param in enumerate(signature.parameters.values()): - if i == 0 and param.annotation not in (str, inspect.Parameter.empty): + try: + signature = inspect.signature(typ) + except ValueError: + # No signature, this is often the case with pybind, etc. + signature = None + + if signature is not None: + # Some checks we can do if the signature is available! + for i, param in enumerate(signature.parameters.values()): + if i == 0 and param.annotation not in (str, inspect.Parameter.empty): + raise UnsupportedTypeAnnotationError( + f"Expected {typ} to be an `(arg: str) -> T` type converter, but" + f" got {signature}. You may have a nested type in a container," + " which is unsupported." + ) + if param.kind is inspect.Parameter.VAR_POSITIONAL: + has_var_positional = True + elif param.default is inspect.Parameter.empty and param.kind is not ( + inspect.Parameter.VAR_KEYWORD + ): + param_count += 1 + + # Raise an error if parameters look wrong. + if not (param_count == 1 or (param_count == 0 and has_var_positional)): raise UnsupportedTypeAnnotationError( f"Expected {typ} to be an `(arg: str) -> T` type converter, but got" f" {signature}. You may have a nested type in a container, which is" " unsupported." ) - if param.kind is inspect.Parameter.VAR_POSITIONAL: - has_var_positional = True - elif param.default is inspect.Parameter.empty and param.kind is not ( - inspect.Parameter.VAR_KEYWORD - ): - param_count += 1 - - # Raise an error if parameters look wrong. - if not (param_count == 1 or (param_count == 0 and has_var_positional)): - raise UnsupportedTypeAnnotationError( - f"Expected {typ} to be an `(arg: str) -> T` type converter, but got" - f" {signature}. You may have a nested type in a container, which is" - " unsupported." - ) # Special case `choices` for some types, as implemented in `instance_from_string()`. auto_choices: Optional[Tuple[str, ...]] = None @@ -181,9 +190,9 @@ def instantiator(_unused_string: str) -> None: return lambda arg: _strings.instance_from_string(typ, arg), InstantiatorMetadata( nargs=None, - metavar=typ.__name__.upper() + metavar=_format_metavar(typ.__name__.upper()) if auto_choices is None - else "{" + ",".join(map(str, auto_choices)) + "}", + else "{" + ",".join(map(_format_metavar, map(str, auto_choices))) + "}", choices=auto_choices, is_optional=False, ) @@ -307,7 +316,7 @@ def _instantiator_from_tuple( make(x) for make, x in zip(instantiators, strings) ), InstantiatorMetadata( nargs=len(types), - metavar=tuple(cast(str, m.metavar) for m in metas), + metavar=tuple(_format_metavar(cast(str, m.metavar)) for m in metas), choices=metas[0].choices, is_optional=False, ) @@ -319,7 +328,7 @@ def _join_union_metavars(metavars: Iterable[str]) -> str: Examples: None, INT => NONE|INT {0,1,2}, {3,4} => {0,1,2,3,4} - None, {0,1,2}, {3,4} => None|{0,1,2,3,4} + {0,1,2}, {3,4}, STR => {0,1,2,3,4}|STR """ metavars = tuple(metavars) merged_metavars = [metavars[0]] @@ -509,6 +518,6 @@ def _instantiator_from_literal( return instantiator, dataclasses.replace( metadata, choices=choices, - metavar="{" + ",".join(map(str, choices)) + "}", + metavar="{" + ",".join(map(_format_metavar, map(str, choices))) + "}", is_optional=False, ) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 53c29a773..2a12bc9cb 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -242,24 +242,17 @@ def apply(self, parser: argparse.ArgumentParser) -> None: parser.description = self.description # Make argument groups. - def format_group_name(nested_field_name: str, required: bool) -> str: - # if required: - # prefix = termcolor.colored("required", attrs=["bold"]) - # else: - # prefix = termcolor.colored("optional", attrs=["bold", "dark"]) - if nested_field_name != "": - return termcolor.colored( - nested_field_name + " arguments", attrs=["bold"] - ) - else: - return termcolor.colored("arguments", attrs=["bold"]) + def format_group_name(nested_field_name: str) -> str: + return termcolor.colored( + (nested_field_name + " arguments").strip(), attrs=["bold"] + ) group_from_prefix: Dict[str, argparse._ArgumentGroup] = { "": parser._action_groups[1], } # Break some API boundaries to rename the optional group. - parser._action_groups[1].title = format_group_name("", required=False) + parser._action_groups[1].title = format_group_name("") positional_group = parser.add_argument_group( termcolor.colored("positional arguments", attrs=["bold"]) ) @@ -274,7 +267,7 @@ def format_group_name(nested_field_name: str, required: bool) -> str: if arg.prefix not in group_from_prefix: nested_field_name = arg.prefix[:-1] group_from_prefix[arg.prefix] = parser.add_argument_group( - format_group_name(nested_field_name, required=arg.lowered.required), + format_group_name(nested_field_name), description=self.helptext_from_nested_class_field_name.get( nested_field_name ), diff --git a/examples/06_base_configs.py b/examples/06_base_configs.py index 61458da6a..09977ed22 100644 --- a/examples/06_base_configs.py +++ b/examples/06_base_configs.py @@ -10,29 +10,25 @@ `python ./06_base_configs_argv.py big --seed 94720` """ -import dataclasses import sys +from dataclasses import dataclass from typing import Dict, Literal, Tuple, Type, TypeVar, Union import dcargs -@dataclasses.dataclass +@dataclass(frozen=True) class AdamOptimizer: - # Adam learning rate. learning_rate: float = 1e-3 - - # Moving average parameters. betas: Tuple[float, float] = (0.9, 0.999) -@dataclasses.dataclass +@dataclass(frozen=True) class SgdOptimizer: - # SGD learning rate. learning_rate: float = 3e-4 -@dataclasses.dataclass(frozen=True) +@dataclass(frozen=True) class ExperimentConfig: # Dataset to run experiment on. dataset: Literal["mnist", "imagenet-50"] @@ -58,7 +54,7 @@ class ExperimentConfig: # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. -base_config_library = { +base_configs = { "small": ExperimentConfig( dataset="mnist", optimizer=SgdOptimizer(), @@ -85,7 +81,9 @@ class ExperimentConfig: T = TypeVar("T") -def cli_with_base_configs(cls: Type[T], base_library: Dict[str, T]) -> T: +def cli_from_base_configs(base_library: Dict[str, T]) -> T: + """Populate an instance of `cls`, where the first positional argument is used to + select from a library of named base configs.""" # Get base configuration name from the first positional argument. if len(sys.argv) < 2 or sys.argv[1] not in base_library: valid_usages = map(lambda k: f"{sys.argv[0]} {k} --help", base_library.keys()) @@ -94,7 +92,7 @@ def cli_with_base_configs(cls: Type[T], base_library: Dict[str, T]) -> T: # Get base configuration from our library, and use it for default CLI parameters. default_instance = base_library[sys.argv[1]] return dcargs.cli( - cls, + type(default_instance), prog=" ".join(sys.argv[:2]), args=sys.argv[2:], default_instance=default_instance, @@ -106,5 +104,5 @@ def cli_with_base_configs(cls: Type[T], base_library: Dict[str, T]) -> T: if __name__ == "__main__": - config = cli_with_base_configs(ExperimentConfig, base_config_library) + config = cli_from_base_configs(base_configs) print(config) diff --git a/examples/09_subparsers.py b/examples/09_subparsers.py index 34808114c..7098df75f 100644 --- a/examples/09_subparsers.py +++ b/examples/09_subparsers.py @@ -19,14 +19,12 @@ @dataclasses.dataclass(frozen=True) class Checkout: """Checkout a branch.""" - branch: str @dataclasses.dataclass(frozen=True) class Commit: """Commit changes.""" - message: str all: bool = False diff --git a/setup.py b/setup.py index d3044ec94..d5f23f995 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.1.6", + version="0.1.7", description="Strongly typed, zero-effort CLIs", long_description=long_description, long_description_content_type="text/markdown", @@ -21,6 +21,7 @@ "typing_extensions>=4.3.0", "pyyaml", "termcolor", + "argparse-color-formatter", "backports.cached-property; python_version < '3.8.0'", ], extras_require={ diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 05cb7cd86..a7bc44481 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -1,3 +1,4 @@ +import copy import dataclasses import enum import pathlib @@ -391,10 +392,10 @@ def add(a: SomeTypeAlias, b: SomeTypeAlias) -> SomeTypeAlias: @pytest.mark.filterwarnings("ignore::Warning") def test_any(): - def main(x: Any) -> Any: + def main(x: Any = 5) -> Any: return x - assert dcargs.cli(main, args=["--x", "hello"]) == "hello" + assert dcargs.cli(main, args=[]) == 5 def test_bytes(): @@ -420,3 +421,7 @@ def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]: assert dcargs.cli(main, args=[])(3) == 6 with pytest.raises(SystemExit): dcargs.cli(main, args=["--x", "something"]) + + +def test_missing_singleton(): + assert dcargs.MISSING is copy.deepcopy(dcargs.MISSING) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 9e654781d..bbf0bffc9 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -159,10 +159,10 @@ class HelptextMultiline: dcargs.cli(HelptextMultiline, args=["--help"]) helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert "Documentation 1 (required)\n" in helptext - assert "Documentation 2\n" in helptext - assert "Next line of documentation 2 (required)\n" in helptext - assert "Documentation 3\n" in helptext - assert "Next line of documentation 3 (default: 3)\n" in helptext + assert "Documentation 2" in helptext + assert "documentation 2" in helptext + assert "Documentation 3" in helptext + assert "documentation 3" in helptext def test_grouped_helptext(): @@ -194,7 +194,8 @@ class Config: with contextlib.redirect_stdout(f): dcargs.cli(Config, args=["--help"]) helptext = dcargs._strings.strip_color_codes(f.getvalue()) - assert " --x None|INT An optional variable. (default: None)\n" in helptext + assert " --x {None}|INT" in helptext + assert "An optional variable. (default: None)\n" in helptext def test_helptext_hard_bool(): @@ -238,9 +239,8 @@ class Child(Parent): dcargs.cli(Child, args=["--help"]) helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert "--x STR" in helptext - assert ( - "Helptext. (default: 'This docstring may be tougher to parse!')\n" in helptext - ) + assert "Helptext." in helptext + assert "(default: 'This docstring" in helptext def test_helptext_with_inheritance_overriden(): @@ -268,9 +268,7 @@ class Child2(Parent2): dcargs.cli(Child2, args=["--help"]) helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert "--x STR" in helptext - assert ( - "Helptext! (default: 'This docstring may be tougher to parse?')\n" in helptext - ) + assert "Helptext! (default: 'This" in helptext def test_tuple_helptext(): @@ -371,7 +369,7 @@ class OptionalLiteralHelptext: with contextlib.redirect_stdout(f): dcargs.cli(OptionalLiteralHelptext, args=["--help"]) helptext = dcargs._strings.strip_color_codes(f.getvalue()) - assert "--x None|{1,2,3}" in helptext + assert "--x {None,1,2,3}" in helptext assert "A number. (default: None)\n" in helptext @@ -441,6 +439,6 @@ class OptionalHelptext: dcargs.cli(OptionalHelptext, args=["--help"]) helptext = dcargs._strings.strip_color_codes(f.getvalue()) assert cast(str, OptionalHelptext.__doc__) in helptext - assert "[--x None|INT]" in helptext - assert "--y None|INT [None|INT ...]\n" in helptext - assert "[--z None|INT]\n" in helptext + assert "[--x {None}|INT]" in helptext + assert "--y {None}|INT [{None}|INT ...]\n" in helptext + assert "[--z {None}|INT]\n" in helptext From aee3609cefb485a9a0c1f6e471df30b96ec91f73 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 16 Jul 2022 17:51:38 -0700 Subject: [PATCH 060/491] Overhaul Unions, clean up internals --- README.md | 199 +++++++++++++-------------- _update_readme.py | 2 +- dcargs/__init__.py | 2 +- dcargs/_argparse_formatter.py | 9 ++ dcargs/_arguments.py | 55 +++++--- dcargs/_calling.py | 24 ++-- dcargs/_cli.py | 19 ++- dcargs/_fields.py | 67 +++++---- dcargs/_instantiators.py | 129 ++++++++++------- dcargs/_parsers.py | 19 ++- dcargs/_serialization.py | 6 +- examples/01_functions.py | 6 +- examples/02_dataclasses.py | 6 +- examples/06_base_configs.py | 2 +- examples/07_literals_and_unions.py | 22 +-- examples/09_subparsers.py | 2 + setup.py | 1 + tests/test_collections.py | 22 ++- tests/test_dcargs.py | 36 ++++- tests/test_dict_namedtuple.py | 14 +- tests/test_errors.py | 10 +- tests/test_helptext.py | 7 +- tests/test_nested.py | 15 ++ tests/test_positional_ignore_py37.py | 32 ++++- 24 files changed, 438 insertions(+), 268 deletions(-) create mode 100644 dcargs/_argparse_formatter.py diff --git a/README.md b/README.md index 0bd0376cc..5c82ded5f 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ Ultimately, we aim to enable configuration interfaces that are: 1. Functions
-
+ In the simplest case, `dcargs.cli()` can be used to run a function with arguments populated from the CLI. @@ -146,16 +146,14 @@ import dcargs def main( field1: str, field2: int = 3, - flag: bool = False, ) -> None: """Function, whose arguments will be populated from a CLI interface. Args: field1: A string field. field2: A numeric field, with a default value. - flag: A boolean flag. """ - print(field1, field2, flag) + print(field1, field2) if __name__ == "__main__": @@ -168,25 +166,24 @@ if __name__ == "__main__": 
 $ python ./01_functions.py --help
-usage: 01_functions.py [-h] --field1 STR [--field2 INT] [--flag]
+usage: 01_functions.py [-h] --field1 STR [--field2 INT]
 
 Function, whose arguments will be populated from a CLI interface.
 
 arguments:
   -h, --help            show this help message and exit
   --field1 STR  A string field. (required)
-  --field2 INT  A numeric field, with a default value. (default: 3)
-  --flag                A boolean flag. (default: flag=False)
+  --field2 INT  A numeric field, with a default value. (default: 3)
 
 $ python ./01_functions.py --field1 hello
-hello 3 False
+hello 3
 
-$ python ./01_functions.py --field1 hello --flag
-hello 3 True
+$ python ./01_functions.py --field1 hello --field2 10
+hello 10
@@ -197,8 +194,9 @@ hello 3 True 2. Dataclasses
-
-Common pattern: use `dcargs.cli()` to instantiate a dataclass. + +Common pattern: use `dcargs.cli()` to instantiate a dataclass. The outputted instance +can be used as a typed alternative for an argparse namespace. **Code ([link](examples/02_dataclasses.py)):** @@ -215,7 +213,6 @@ class Args: field1: str # A string field. field2: int = 3 # A numeric field, with a default value. - flag: bool = False # A boolean flag. if __name__ == "__main__": @@ -229,25 +226,24 @@ if __name__ == "__main__": 
 $ python ./02_dataclasses.py --help
-usage: 02_dataclasses.py [-h] --field1 STR [--field2 INT] [--flag]
+usage: 02_dataclasses.py [-h] --field1 STR [--field2 INT]
 
 Description. This should show up in the helptext!
 
 arguments:
   -h, --help            show this help message and exit
   --field1 STR  A string field. (required)
-  --field2 INT  A numeric field, with a default value. (default: 3)
-  --flag                A boolean flag. (default: flag=False)
+  --field2 INT  A numeric field, with a default value. (default: 3)
 
 $ python ./02_dataclasses.py --field1 hello
-Args(field1='hello', field2=3, flag=False)
+Args(field1='hello', field2=3)
 
-$ python ./02_dataclasses.py --field1 hello --flag
-Args(field1='hello', field2=3, flag=True)
+$ python ./02_dataclasses.py --field1 hello --field2 5
+Args(field1='hello', field2=5)
@@ -258,7 +254,7 @@ Args(field1='hello', field2=3, flag=True) 3. Enums And Containers
-
+ We can generate argument parsers from more advanced type annotations, like enums and tuple types. @@ -345,7 +341,7 @@ TrainConfig(dataset_sources=(PosixPath('data'),), image_dimensions=(32 4. Flags
-
+ Booleans can either be expected to be explicitly passed in, or, if given a default value, automatically converted to flags. @@ -384,9 +380,8 @@ if __name__ == "__main__": 
 $ python ./04_flags.py --help
-usage: 04_flags.py [-h] --boolean {True,False}
-                   [--optional-boolean {None,True,False}] [--flag-a]
-                   [--no-flag-b]
+usage: 04_flags.py [-h] --boolean {True,False} --optional-boolean
+                   {None,True,False} [--flag-a] [--no-flag-b]
 
 arguments:
   -h, --help            show this help message and exit
@@ -395,26 +390,32 @@ arguments:
                         (required)
   --optional-boolean {None,True,False}
                         Optional boolean. Same as above, but can be omitted.
-                        (default: None)
-  --flag-a              Pass --flag-a in to set this value to True. (default:
-                        flag_a=False)
-  --no-flag-b           Pass --no-flag-b in to set this value to False.
-                        (default: flag_b=True)
+                        (required)
+  --flag-a              Pass --flag-a in to set this value to True. (sets
+                        flag_a=True)
+  --no-flag-b           Pass --no-flag-b in to set this value to False. (sets
+                        flag_b=False)
 
 $ python ./04_flags.py --boolean True
-Args(boolean=True, optional_boolean=None, flag_a=False, flag_b=True)
+usage: 04_flags.py [-h] --boolean {True,False} --optional-boolean
+                   {None,True,False} [--flag-a] [--no-flag-b]
+04_flags.py: error: the following arguments are required: --optional-boolean
 
 $ python ./04_flags.py --boolean False --flag-a
-Args(boolean=False, optional_boolean=None, flag_a=True, flag_b=True)
+usage: 04_flags.py [-h] --boolean {True,False} --optional-boolean
+                   {None,True,False} [--flag-a] [--no-flag-b]
+04_flags.py: error: the following arguments are required: --optional-boolean
 
 $ python ./04_flags.py --boolean False --no-flag-b
-Args(boolean=False, optional_boolean=None, flag_a=False, flag_b=False)
+usage: 04_flags.py [-h] --boolean {True,False} --optional-boolean
+                   {None,True,False} [--flag-a] [--no-flag-b]
+04_flags.py: error: the following arguments are required: --optional-boolean
@@ -425,7 +426,7 @@ Args(boolean=False, optional_boolean=None, flag_a=False, flag_b=False) 5. Hierarchical Configs
-
+ Parsing of nested types (in this case nested dataclasses) enables hierarchical configuration objects that are both modular and highly expressive. @@ -519,8 +520,8 @@ positional arguments: arguments: -h, --help show this help message and exit - --restore-checkpoint Set to restore an existing checkpoint. (default: - restore_checkpoint=False) + --restore-checkpoint Set to restore an existing checkpoint. (sets + restore_checkpoint=True) --checkpoint-interval INT Training steps between each checkpoint save. (default: 1000) @@ -585,7 +586,7 @@ train_steps: 100000 6. Base Configs
-
+ We can integrate `dcargs.cli()` into common configuration patterns: here, we select one of multiple possible base configurations, and then use the CLI to either override (existing) or fill in (missing) values. @@ -595,7 +596,7 @@ one of multiple possible base configurations, and then use the CLI to either ove ```python import sys from dataclasses import dataclass -from typing import Dict, Literal, Tuple, Type, TypeVar, Union +from typing import Dict, Literal, Tuple, TypeVar, Union import dcargs @@ -637,7 +638,7 @@ class ExperimentConfig: # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. -base_config_library = { +base_configs = { "small": ExperimentConfig( dataset="mnist", optimizer=SgdOptimizer(), @@ -664,9 +665,9 @@ base_config_library = { T = TypeVar("T") -def cli_with_base_configs(cls: Type[T], base_library: Dict[str, T]) -> T: - """Populate an instance of `cls`, using a CLI lets the user select from a library of - base configs.""" +def cli_from_base_configs(base_library: Dict[str, T]) -> T: + """Populate an instance of `cls`, where the first positional argument is used to + select from a library of named base configs.""" # Get base configuration name from the first positional argument. if len(sys.argv) < 2 or sys.argv[1] not in base_library: valid_usages = map(lambda k: f"{sys.argv[0]} {k} --help", base_library.keys()) @@ -675,7 +676,7 @@ def cli_with_base_configs(cls: Type[T], base_library: Dict[str, T]) -> T: # Get base configuration from our library, and use it for default CLI parameters. default_instance = base_library[sys.argv[1]] return dcargs.cli( - cls, + type(default_instance), prog=" ".join(sys.argv[:2]), args=sys.argv[2:], default_instance=default_instance, @@ -687,7 +688,7 @@ def cli_with_base_configs(cls: Type[T], base_library: Dict[str, T]) -> T: if __name__ == "__main__": - config = cli_with_base_configs(ExperimentConfig, base_config_library) + config = cli_from_base_configs(base_configs) print(config) ``` @@ -781,7 +782,7 @@ ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learni 7. Literals And Unions
-
+ `typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices; `typing.Union[]` can be used to restrict inputs to a fixed set of types. @@ -790,7 +791,7 @@ ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learni ```python import dataclasses import enum -from typing import Literal, Tuple, Union +from typing import Literal, Optional, Tuple, Union import dcargs @@ -803,11 +804,21 @@ class Color(enum.Enum): @dataclasses.dataclass(frozen=True) class Args: - enum: Color = Color.RED + # We can use Literal[] to restrict the set of allowable inputs, for example, over + # enums. restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED - integer: Literal[0, 1, 2, 3] = 0 - string: Literal["red", "green"] = "red" + + # Literals can also be marked Optional. + integer: Optional[Literal[0, 1, 2, 3]] = None + + # Unions can be used to specify multiple allowable types. + union_over_types: Union[int, str] = 0 string_or_enum: Union[Literal["red", "green"], Color] = "red" + + # Unions also work over more complex nested types. + union_over_tuples: Union[Tuple[int, int], Tuple[str]] = ("1",) + + # And can be nested in other types. tuple_of_string_or_enum: Tuple[Union[Literal["red", "green"], Color], ...] = ( "red", Color.RED, @@ -825,46 +836,31 @@ if __name__ == "__main__": 
 $ python ./07_literals_and_unions.py --help
-usage: 07_literals_and_unions.py [-h] [--enum {RED,GREEN,BLUE}]
-                                 [--restricted-enum {RED,GREEN}]
-                                 [--integer {0,1,2,3}] [--string {red,green}]
+usage: 07_literals_and_unions.py [-h] [--restricted-enum {RED,GREEN}]
+                                 [--integer {None,0,1,2,3}]
+                                 [--union-over-types INT|STR]
                                  [--string-or-enum {red,green,RED,GREEN,BLUE}]
+                                 [--union-over-tuples {INT INT}|STR]
                                  [--tuple-of-string-or-enum {red,green,RED,GREEN,BLUE} [{red,green,RED,GREEN,BLUE} ...]]
 
 arguments:
   -h, --help            show this help message and exit
-  --enum {RED,GREEN,BLUE}
-                        (default: RED)
   --restricted-enum {RED,GREEN}
-                        (default: RED)
-  --integer {0,1,2,3}
-                        (default: 0)
-  --string {red,green}
-                        (default: red)
+                        We can use Literal[] to restrict the set of allowable
+                        inputs, for example, over enums. (default: RED)
+  --integer {None,0,1,2,3}
+                        Literals can also be marked Optional. (default: None)
+  --union-over-types INT|STR
+                        Unions can be used to specify multiple allowable
+                        types. (default: 0)
   --string-or-enum {red,green,RED,GREEN,BLUE}
-                        (default: red)
+                        Unions can be used to specify multiple allowable
+                        types. (default: red)
+  --union-over-tuples {INT INT}|STR
+                        Unions also work over more complex nested types.
+                        (default: 1)
   --tuple-of-string-or-enum {red,green,RED,GREEN,BLUE} [{red,green,RED,GREEN,BLUE} ...]
-                        (default: red RED)
-
- -
-$ python ./07_literals_and_unions.py --enum RED --restricted-enum GREEN --integer 3 --string green
-Args(enum=<Color.RED: 1>, restricted_enum=<Color.GREEN: 2>, integer=3, string='green', string_or_enum='red', tuple_of_string_or_enum=('red', <Color.RED: 1>))
-
- -
-$ python ./07_literals_and_unions.py --string-or-enum green
-Args(enum=<Color.RED: 1>, restricted_enum=<Color.RED: 1>, integer=0, string='red', string_or_enum='green', tuple_of_string_or_enum=('red', <Color.RED: 1>))
-
- -
-$ python ./07_literals_and_unions.py --string-or-enum RED
-Args(enum=<Color.RED: 1>, restricted_enum=<Color.RED: 1>, integer=0, string='red', string_or_enum=<Color.RED: 1>, tuple_of_string_or_enum=('red', <Color.RED: 1>))
-
- -
-$ python ./07_literals_and_unions.py --tuple-of-string-or-enum RED green BLUE
-Args(enum=<Color.RED: 1>, restricted_enum=<Color.RED: 1>, integer=0, string='red', string_or_enum='red', tuple_of_string_or_enum=(<Color.RED: 1>, 'green', <Color.BLUE: 3>))
+                        And can be nested in other types. (default: red RED)
@@ -875,7 +871,7 @@ Args(enum=<Color.RED: 1>, restricted_enum=<Color.RED: 1>, integer=0, 8. Positional Args
-
+ Positional-only arguments in functions are converted to positional CLI arguments. **Code ([link](examples/08_positional_args.py)):** @@ -956,9 +952,8 @@ positional arguments: arguments: -h, --help show this help message and exit - --force Do not prompt before overwriting. (default: - force=False) - --verbose Explain what is being done. (default: verbose=False) + --force Do not prompt before overwriting. (sets force=True) + --verbose Explain what is being done. (sets verbose=True) --background-rgb FLOAT FLOAT FLOAT Background color. Red by default. (default: 1.0 0.0 0.0) @@ -992,7 +987,7 @@ background_rgb=(1.0, 0.0, 0.0) 9. Subparsers
-
+ Unions over nested types (classes or dataclasses) are populated using subparsers. **Code ([link](examples/09_subparsers.py)):** @@ -1009,12 +1004,14 @@ import dcargs @dataclasses.dataclass(frozen=True) class Checkout: """Checkout a branch.""" + branch: str @dataclasses.dataclass(frozen=True) class Commit: """Commit changes.""" + message: str all: bool = False @@ -1033,7 +1030,7 @@ if __name__ == "__main__": 
 $ python ./09_subparsers.py --help
-usage: 09_subparsers.py [-h] {checkout,commit} ...
+usage: 09_subparsers.py [-h] {checkout,commit}
 
 arguments:
   -h, --help         show this help message and exit
@@ -1055,7 +1052,7 @@ arguments:
 cmd arguments:
   --cmd.message STR
                         (required)
-  --cmd.all             (default: all=False)
+  --cmd.all             (sets all=True)
 
@@ -1090,7 +1087,7 @@ Checkout(branch='main')
 10. Multiple Subparsers
 
 

-

+
 Multiple unions over nested types are populated using a series of subparsers.
 
 **Code ([link](examples/10_multiple_subparsers.py)):**
@@ -1168,7 +1165,7 @@ AdamOptimizer(learning_rate=0.001, betas=(0.9, 0.999))
 
 
 $ python ./10_multiple_subparsers.py --help
-usage: 10_multiple_subparsers.py [-h] [{mnist-dataset,image-net-dataset}] ...
+usage: 10_multiple_subparsers.py [-h] [{mnist-dataset,image-net-dataset}]
 
 Example training script.
 
@@ -1191,8 +1188,8 @@ arguments:
   -h, --help            show this help message and exit
 
 dataset arguments:
-  --dataset.binary      Set to load binary version of MNIST dataset.
-                        (default: binary=False)
+  --dataset.binary      Set to load binary version of MNIST dataset. (sets
+                        binary=True)
 
 optional subcommands:
   Optimizer to train with. (default: adam-optimizer)
@@ -1214,7 +1211,7 @@ AdamOptimizer(learning_rate=0.0003, betas=(0.9, 0.999))
 11. Dictionaries
 
 

-

+
 Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation,
 or a `TypedDict` type.
 
@@ -1260,12 +1257,12 @@ if __name__ == "__main__":
 
 $ python ./11_dictionaries.py --help
 usage: 11_dictionaries.py [-h] --standard-dict STR {True,False}
-                          [STR {True,False} ...] [--typed-dict.field1 STR]
-                          [--typed-dict.field2 INT] [--typed-dict.field3]
+                          [--typed-dict.field1 STR] [--typed-dict.field2 INT]
+                          [--typed-dict.field3]
 
 arguments:
   -h, --help            show this help message and exit
-  --standard-dict STR {True,False} [STR {True,False} ...]
+  --standard-dict STR {True,False}
                         (required)
 
 typed_dict arguments:
@@ -1274,7 +1271,7 @@ typed_dict arguments:
                         A string field. (default: hey)
   --typed-dict.field2 INT
                         A numeric field. (default: 3)
-  --typed-dict.field3   A boolean field. (default: field3=False)
+  --typed-dict.field3   A boolean field. (sets field3=True)
 

 
 
@@ -1291,7 +1288,7 @@ Typed dict: {'field1': 'hey', 'field2': 3, 'f
 12. Named Tuples
 
 

-

+
 Example using `dcargs.cli()` to instantiate a named tuple.
 
 **Code ([link](examples/12_named_tuples.py)):**
@@ -1331,7 +1328,7 @@ arguments:
   -h, --help            show this help message and exit
   --field1 STR  A string field. (required)
   --field2 INT  A numeric field, with a default value. (default: 3)
-  --flag                A boolean flag. (default: flag=False)
+  --flag                A boolean flag. (sets flag=True)
 

 
 
@@ -1347,7 +1344,7 @@ TupleType(field1='hello', field2=3, flag=False)
 13. Standard Classes
 
 

-

+
 In addition to functions and dataclasses, we can also generate CLIs from (the
 constructors of) standard Python classes.
 
@@ -1393,7 +1390,7 @@ arguments:
   -h, --help            show this help message and exit
   --field1 STR  A string field. (required)
   --field2 INT  A numeric field. (required)
-  --flag                A boolean flag. (default: flag=False)
+  --flag                A boolean flag. (sets flag=True)
 

 
 
@@ -1409,7 +1406,7 @@ arguments:
 14. Generics
 
 

-

+
 Example of parsing for generic dataclasses.
 
 **Code ([link](examples/14_generics.py)):**
diff --git a/_update_readme.py b/_update_readme.py
index 7c0971d69..ea6d90256 100644
--- a/_update_readme.py
+++ b/_update_readme.py
@@ -99,7 +99,7 @@ def format_script_for_readme(path: pathlib.Path) -> str:
             f"{index}. {title}",
             "",
             "
",
-            "
",
+            "",
             description_text,
             "",
             "\n".join(example_usages),
diff --git a/dcargs/__init__.py b/dcargs/__init__.py
index c2ba5e4c4..2e368c857 100644
--- a/dcargs/__init__.py
+++ b/dcargs/__init__.py
@@ -1,5 +1,5 @@
 from ._cli import cli, parse
-from ._fields import MISSING
+from ._fields import MISSING_PUBLIC as MISSING
 from ._instantiators import UnsupportedTypeAnnotationError
 from ._serialization import from_yaml, to_yaml
 
diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py
new file mode 100644
index 000000000..262622cb8
--- /dev/null
+++ b/dcargs/_argparse_formatter.py
@@ -0,0 +1,9 @@
+from argparse_color_formatter import ColorHelpFormatter as ColorHelpFormatterBase
+
+
+class ColorHelpFormatter(ColorHelpFormatterBase):
+    def _format_args(self, action, default_metavar):
+        """Override _format_args() to ignore nargs and always expect single string
+        metavars."""
+        get_metavar = self._metavar_formatter(action, default_metavar)
+        return get_metavar(1)[0]
diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py
index 4cf9a966f..d922acbc6 100644
--- a/dcargs/_arguments.py
+++ b/dcargs/_arguments.py
@@ -6,7 +6,7 @@
 import functools
 import itertools
 import shlex
-from typing import Any, Dict, Mapping, Optional, Set, Tuple, Type, TypeVar, Union
+from typing import Any, Dict, Mapping, Optional, Set, Type, TypeVar, Union
 
 import termcolor
 
@@ -39,6 +39,12 @@ def add_argument(
         kwargs = {k: v for k, v in kwargs.items() if v is not None}
         name_or_flag = kwargs.pop("name_or_flag")
 
+        # We're actually going to skip the default field: if an argument is unset, the
+        # MISSING value will be detected in _calling.py and the field default will
+        # directly be used. This helps reduce the likelihood of issues with converting
+        # the field default to a string format, then back to the desired type.
+        kwargs["default"] = _fields.MISSING_NONPROP
+
         # Note that the name must be passed in as a position argument.
         parser.add_argument(name_or_flag, **kwargs)
 
@@ -89,7 +95,9 @@ def is_fixed(self) -> bool:
     action: Optional[str] = None
     nargs: Optional[Union[int, str]] = None
     choices: Optional[Set[Any]] = None
-    metavar: Optional[Union[str, Tuple[str, ...]]] = None
+    # Note: unlike in vanilla argparse, our metavar is always a string. We handle
+    # sequences, multiple arguments, etc, manually.
+    metavar: Optional[str] = None
     help: Optional[str] = None
 
 
@@ -100,7 +108,7 @@ def _rule_handle_defaults(
     """Set `required=True` if a default value is set."""
 
     # Mark lowered as required if a default is set.
-    if arg.field.default is None or arg.field.default is _fields.MISSING:
+    if arg.field.default in _fields.MISSING_SINGLETONS:
         return dataclasses.replace(lowered, default=None, required=True)
 
     return dataclasses.replace(lowered, default=arg.field.default)
@@ -152,7 +160,7 @@ def _rule_recursive_instantiator_from_type(
             arg.type_from_typevar,
         )
     except _instantiators.UnsupportedTypeAnnotationError as e:
-        if arg.field.default is None:
+        if arg.field.default in _fields.MISSING_SINGLETONS:
             raise e
         else:
             # For fields with a default, we'll get by even if there's no instantiator
@@ -161,7 +169,7 @@ def _rule_recursive_instantiator_from_type(
                 lowered,
                 metavar=termcolor.colored("(not parsable)", color="red"),
                 required=False,
-                default=_fields.MISSING,
+                default=_fields.MISSING_PROP,
             )
 
     return dataclasses.replace(
@@ -169,7 +177,6 @@ def _rule_recursive_instantiator_from_type(
         instantiator=instantiator,
         choices=metadata.choices,
         nargs=metadata.nargs,
-        required=(not metadata.is_optional) and lowered.required,
         metavar=metadata.metavar,
     )
 
@@ -189,7 +196,7 @@ def as_str(x: Any) -> str:
 
     if (
         lowered.default is None
-        or lowered.default is _fields.MISSING
+        or lowered.default in _fields.MISSING_SINGLETONS
         or lowered.action is not None
     ):
         return lowered
@@ -228,7 +235,7 @@ def _rule_generate_helptext(
     if lowered.is_fixed():
         # For fixed args, we'll be missing the lowered default. Use field default
         # instead.
-        assert default is _fields.MISSING
+        assert default in _fields.MISSING_SINGLETONS
         default = arg.field.default
 
     if not lowered.required:
@@ -237,9 +244,9 @@ def _rule_generate_helptext(
         # because the types of all arguments are set to strings, which will cause the
         # default to be casted to a string and introduce extra quotation marks.
         if lowered.action == "store_true":
-            default_text = f"(default: {arg.field.name}=False)"
+            default_text = f"(sets {arg.field.name}=True)"
         elif lowered.action == "store_false":
-            default_text = f"(default: {arg.field.name}=True)"
+            default_text = f"(sets {arg.field.name}=False)"
         elif lowered.nargs is not None and hasattr(default, "__iter__"):
             # For tuple types, we might have default as (0, 1, 2, 3).
             # For list types, we might have default as [0, 1, 2, 3].
@@ -254,7 +261,7 @@ def _rule_generate_helptext(
             default_text = f"({default_label}: {shlex.quote(str(default))})"
         help_parts.append(termcolor.colored(default_text, attrs=["dark"]))
     else:
-        help_parts.append(termcolor.colored("(required)", color="red", attrs=["bold"]))
+        help_parts.append(termcolor.colored("(required)", on_color="on_red"))
 
     return dataclasses.replace(lowered, help=" ".join(help_parts))
 
@@ -282,14 +289,26 @@ def _rule_positional_special_handling(
     if not arg.field.positional:
         return lowered
 
-    if lowered.nargs is not None and not lowered.required:
-        raise _instantiators.UnsupportedTypeAnnotationError(
-            "Optional sequences are not supported for positional arguments!"
-        )
+    metavar = (arg.prefix + arg.field.name).upper()
+    if lowered.nargs == "+":
+        metavar = f"{metavar} [{metavar} ...]"
+    elif isinstance(lowered.nargs, int):
+        metavar = " ".join((metavar,) * lowered.nargs)
+
+    if lowered.required:
+        nargs = lowered.nargs
+    else:
+        metavar = "[" + metavar + "]"
+        if lowered.nargs is None:
+            nargs = "?"
+        else:
+            # If lowered.nargs is either + or an int.
+            nargs = "*"
+
     return dataclasses.replace(
         lowered,
         dest=None,
-        metavar=(arg.prefix + arg.field.name).upper(),
-        required=None,
-        nargs="?" if not lowered.required else lowered.nargs,
+        required=None,  # Can't be passed in for positionals.
+        metavar=metavar,
+        nargs=nargs,
     )
diff --git a/dcargs/_calling.py b/dcargs/_calling.py
index 0be5e9916..580bd51e3 100644
--- a/dcargs/_calling.py
+++ b/dcargs/_calling.py
@@ -3,7 +3,7 @@
 
 from __future__ import annotations
 
-from typing import Any, Callable, Dict, List, Optional, Set, Tuple, TypeVar, Union
+from typing import Any, Callable, Dict, List, Set, Tuple, TypeVar, Union
 
 from typing_extensions import get_args, get_origin
 
@@ -21,7 +21,7 @@ class InstantiationError(Exception):
 def call_from_args(
     f: Callable[..., T],
     parser_definition: _parsers.ParserSpecification,
-    default_instance: Optional[T],
+    default_instance: Union[T, _fields.NonpropagatingMissingType],
     value_from_prefixed_field_name: Dict[str, Any],
     field_name_prefix: str,
     avoid_subparsers: bool,
@@ -65,7 +65,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any:
             consumed_keywords.add(prefixed_field_name)
             if not arg.lowered.is_fixed():
                 value = get_value_from_arg(prefixed_field_name)
-                if value is not None:
+                if value not in _fields.MISSING_SINGLETONS:
                     try:
                         assert arg.lowered.instantiator is not None
                         value = arg.lowered.instantiator(value)
@@ -73,20 +73,16 @@ def get_value_from_arg(prefixed_field_name: str) -> Any:
                         raise InstantiationError(
                             f"Parsing error for {arg.lowered.name_or_flag}: {e.args[0]}"
                         )
+                else:
+                    value = arg.field.default
             else:
-                assert arg.field.default is not _fields.MISSING
+                assert arg.field.default not in _fields.MISSING_SINGLETONS
                 value = arg.field.default
-                if (
-                    value_from_prefixed_field_name.get(prefixed_field_name)
-                    is not _fields.MISSING
-                ):
-                    print(
-                        (type(value_from_prefixed_field_name.get(prefixed_field_name))),
-                        (type(_fields.MISSING)),
-                    )
+                parsed_value = value_from_prefixed_field_name.get(prefixed_field_name)
+                if parsed_value not in _fields.MISSING_SINGLETONS:
                     raise InstantiationError(
-                        f"{arg.lowered.name_or_flag} was passed in, but is a fixed"
-                        " argument that cannot be parsed"
+                        f"{arg.lowered.name_or_flag}={parsed_value} was passed in, but"
+                        " is a fixed argument that cannot be parsed"
                     )
         elif (
             prefixed_field_name
diff --git a/dcargs/_cli.py b/dcargs/_cli.py
index 5d8a6df25..cca008f2b 100644
--- a/dcargs/_cli.py
+++ b/dcargs/_cli.py
@@ -1,9 +1,7 @@
 import argparse
-from typing import Callable, Optional, Sequence, TypeVar
+from typing import Callable, Optional, Sequence, TypeVar, Union
 
-import argparse_color_formatter
-
-from . import _calling, _parsers
+from . import _argparse_formatter, _calling, _fields, _parsers
 
 T = TypeVar("T")
 
@@ -92,20 +90,27 @@ def cli(
         The output of `f(...)`.
     """
 
+    default_instance_internal: Union[_fields.NonpropagatingMissingType, T]
+    if default_instance is None:
+        default_instance_internal = _fields.MISSING_NONPROP
+    else:
+        default_instance_internal = default_instance
+    del default_instance
+
     # Map a callable to the relevant CLI arguments + subparsers.
     parser_definition = _parsers.ParserSpecification.from_callable(
         f,
         description=description,
         parent_classes=set(),  # Used for recursive calls.
         parent_type_from_typevar=None,  # Used for recursive calls.
-        default_instance=default_instance,  # Overrides for default values.
+        default_instance=default_instance_internal,  # Overrides for default values.
         prefix="",  # Used for recursive calls.
         avoid_subparsers=avoid_subparsers,
     )
 
     # Parse using argparse!
     parser = argparse.ArgumentParser(
-        prog=prog, formatter_class=argparse_color_formatter.ColorHelpFormatter
+        prog=prog, formatter_class=_argparse_formatter.ColorHelpFormatter
     )
     parser_definition.apply(parser)
     value_from_prefixed_field_name = vars(parser.parse_args(args=args))
@@ -115,7 +120,7 @@ def cli(
         out, consumed_keywords = _calling.call_from_args(
             f,
             parser_definition,
-            default_instance,
+            default_instance_internal,
             value_from_prefixed_field_name,
             field_name_prefix="",
             avoid_subparsers=avoid_subparsers,
diff --git a/dcargs/_fields.py b/dcargs/_fields.py
index 84a224396..3f2d7e725 100644
--- a/dcargs/_fields.py
+++ b/dcargs/_fields.py
@@ -3,7 +3,7 @@
 import dataclasses
 import inspect
 import warnings
-from typing import Any, Callable, List, Optional, Type, TypeVar
+from typing import Any, Callable, List, Optional, Type, TypeVar, Union
 
 import docstring_parser
 from typing_extensions import get_type_hints, is_typeddict
@@ -20,7 +20,7 @@ class Field:
     positional: bool
 
 
-class _MISSING_TYPE:  # pragma: no cover
+class _Singleton:
     # Singleton pattern.
     # https://www.python.org/download/releases/2.2/descrintro/#__new__
     def __new__(cls, *args, **kwds):
@@ -35,16 +35,38 @@ def init(self, *args, **kwds):
         pass
 
 
-MISSING: Any = _MISSING_TYPE()
+class PropagatingMissingType(_Singleton):  # pragma: no cover
+    pass
+
+
+class NonpropagatingMissingType(_Singleton):
+    pass
+
+
+# We have two types of missing sentinels: a propagating missing value, which when set as
+# a default will set all child values of nested structures as missing as well, and a
+# nonpropagating missing sentinel, which does not override child defaults.
+MISSING_PROP = PropagatingMissingType()
+MISSING_NONPROP = NonpropagatingMissingType()
+
+# Note that our "public" missing API will always be the propagating missing sentinel.
+MISSING_PUBLIC: Any = MISSING_PROP
 """Sentinel value to mark fields as missing. Should generally only be used to mark
 fields passed in as a `default_instance` for `dcargs.cli()` as required."""
 
-_missing_types = [dataclasses.MISSING, MISSING, inspect.Parameter.empty]
+
+MISSING_TYPE = Union[PropagatingMissingType, NonpropagatingMissingType]
+MISSING_SINGLETONS = [
+    dataclasses.MISSING,
+    MISSING_PROP,
+    MISSING_NONPROP,
+    inspect.Parameter.empty,
+]
 try:
     # Undocumented feature: support omegaconf dataclasses out of the box.
     import omegaconf
 
-    _missing_types.append(omegaconf.MISSING)
+    MISSING_SINGLETONS.append(omegaconf.MISSING)
 except ImportError:
     pass
 
@@ -52,7 +74,8 @@ def init(self, *args, **kwds):
 
 
 def field_list_from_callable(
-    f: Callable[..., T], default_instance: Optional[T]
+    f: Callable[..., T],
+    default_instance: Union[T, PropagatingMissingType, NonpropagatingMissingType],
 ) -> List[Field]:
     """Generate a list of generic 'field' objects corresponding to the inputs of some
     annotated callable.
@@ -74,18 +97,16 @@ def field_list_from_callable(
     if cls is not None and is_typeddict(cls):
         # Handle typed dictionaries.
         field_list = []
-        no_default_instance = (
-            default_instance is None or default_instance in _missing_types
-        )
+        no_default_instance = default_instance in MISSING_SINGLETONS
         assert no_default_instance or isinstance(default_instance, dict)
         for name, typ in get_type_hints(cls).items():
             field_list.append(
                 Field(
                     name=name,
                     typ=typ,
-                    default=MISSING
+                    default=MISSING_PROP
                     if no_default_instance
-                    else default_instance.get(name, MISSING),  # type: ignore
+                    else default_instance.get(name, MISSING_PROP),  # type: ignore
                     helptext=_docstrings.get_field_docstring(cls, name),
                     positional=False,
                 )
@@ -103,11 +124,11 @@ def field_list_from_callable(
         # Note that _field_types is removed in Python 3.9.
         for name, typ in _resolver.get_type_hints(cls).items():
             # Get default, with priority for `default_instance`.
-            default = field_defaults.get(name)
+            default = field_defaults.get(name, MISSING_NONPROP)
             if hasattr(default_instance, name):
                 default = getattr(default_instance, name)
-            if default in _missing_types or default_instance in _missing_types:
-                default = None
+            if default_instance is MISSING_PROP:
+                default = MISSING_PROP
 
             field_list.append(
                 Field(
@@ -139,7 +160,7 @@ def field_list_from_callable(
     else:
         # Handle general callables.
         assert (
-            default_instance is None
+            default_instance in MISSING_SINGLETONS
         ), "`default_instance` is only supported for dataclass and TypedDict types."
 
         # Get type annotations, docstrings.
@@ -163,9 +184,6 @@ def field_list_from_callable(
 
             # Get default value.
             default = param.default
-            if default in _missing_types:
-                # TODO: we should _fields.MISSING.
-                default = None
 
             # Get helptext from docstring.
             helptext = docstring_from_arg_name.get(param.name)
@@ -213,11 +231,14 @@ def _get_dataclass_field_default(
     """Helper for getting the default instance for a field."""
     # If the dataclass's parent is explicitly marked MISSING, mark this field as missing
     # as well.
-    if parent_default_instance in _missing_types:
-        return MISSING
+    if parent_default_instance is MISSING_PROP:
+        return MISSING_PROP
 
     # Try grabbing default from parent instance.
-    if parent_default_instance is not None:
+    if (
+        parent_default_instance not in MISSING_SINGLETONS
+        and parent_default_instance is not None
+    ):
         # Populate default from some parent, eg `default_instance` in `dcargs.cli()`.
         if hasattr(parent_default_instance, field.name):
             return getattr(parent_default_instance, field.name)
@@ -230,7 +251,7 @@ def _get_dataclass_field_default(
             )
 
     # Try grabbing default from dataclass field.
-    if field.default not in _missing_types:
+    if field.default not in MISSING_SINGLETONS:
         default = field.default
         if dataclasses.is_dataclass(default):
             _ensure_dataclass_instance_used_as_default_is_frozen(field, default)
@@ -254,4 +275,4 @@ def _get_dataclass_field_default(
 
     # Otherwise, no default. This is different from MISSING, because MISSING propagates
     # to children. We could revisit this design to make it clearer.
-    return None
+    return MISSING_NONPROP
diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py
index 90867ae10..d3c1b69b4 100644
--- a/dcargs/_instantiators.py
+++ b/dcargs/_instantiators.py
@@ -76,9 +76,10 @@
 @dataclasses.dataclass
 class InstantiatorMetadata:
     nargs: Optional[Union[str, int]]
-    metavar: Union[str, Tuple[str, ...]]
+    # Note: unlike in vanilla argparse, our metavar is always a string. We handle
+    # sequences, multiple arguments, etc, manually.
+    metavar: str
     choices: Optional[Tuple[str, ...]]
-    is_optional: bool
 
 
 class UnsupportedTypeAnnotationError(Exception):
@@ -123,15 +124,15 @@ def instantiator_from_type(
     if typ is NoneType:
 
         def instantiator(string: str) -> None:
-            if string != "None":
-                raise ValueError("Only valid argument is `None`.")
+            # Note that other inputs should be caught by `choices` before the
+            # instantiator runs.
+            assert string == "None"
             return None
 
         return instantiator, InstantiatorMetadata(
             nargs=None,
             metavar="{" + _format_metavar("None") + "}",
             choices=("None",),
-            is_optional=False,
         )
 
     # Address container types. If a matching container is found, this will recursively
@@ -194,7 +195,6 @@ def instantiator(string: str) -> None:
         if auto_choices is None
         else "{" + ",".join(map(_format_metavar, map(str, auto_choices))) + "}",
         choices=auto_choices,
-        is_optional=False,
     )
 
 
@@ -289,9 +289,8 @@ def _instantiator_from_tuple(
         )
         return lambda strings: tuple([make(x) for x in strings]), InstantiatorMetadata(
             nargs="+",
-            metavar=inner_meta.metavar,
+            metavar=f"{inner_meta.metavar} [{inner_meta.metavar} ...]",
             choices=inner_meta.choices,
-            is_optional=False,
         )
 
     else:
@@ -312,13 +311,19 @@ def _instantiator_from_tuple(
                 " must match. This restricts mixing enums & literals with other"
                 " types."
             )
-        return lambda strings: tuple(
-            make(x) for make, x in zip(instantiators, strings)
-        ), InstantiatorMetadata(
+
+        def tuple_instantiator(strings: List[str]) -> Tuple[Any, ...]:
+            if len(strings) != len(instantiators):
+                raise ValueError(
+                    f"expected {len(instantiators)} arguments, but got {strings}"
+                )
+            out = tuple(make(x) for make, x in zip(instantiators, strings))
+            return out
+
+        return tuple_instantiator, InstantiatorMetadata(
             nargs=len(types),
-            metavar=tuple(_format_metavar(cast(str, m.metavar)) for m in metas),
+            metavar=" ".join((_format_metavar(cast(str, m.metavar)) for m in metas)),
             choices=metas[0].choices,
-            is_optional=False,
         )
 
 
@@ -329,6 +334,12 @@ def _join_union_metavars(metavars: Iterable[str]) -> str:
         None, INT => NONE|INT
         {0,1,2}, {3,4} => {0,1,2,3,4}
         {0,1,2}, {3,4}, STR => {0,1,2,3,4}|STR
+        {None}, INT [INT ...] => {None}|{INT [INT ...]}
+        STR, INT [INT ...] => STR|{INT [INT ...]}
+        STR, INT INT => STR|{INT INT}
+
+    The curly brackets are unfortunately overloaded but parentheses, square brackets,
+    and angle brackets all seem to interfere with some argparse internals.
     """
     metavars = tuple(metavars)
     merged_metavars = [metavars[0]]
@@ -344,6 +355,11 @@ def _join_union_metavars(metavars: Iterable[str]) -> str:
             merged_metavars[-1] = prev[:-1] + "," + curr[1:]
         else:
             merged_metavars.append(curr)
+
+    for i, m in enumerate(merged_metavars):
+        if " " in m:
+            merged_metavars[i] = "{" + m + "}"
+
     return "|".join(merged_metavars)
 
 
@@ -380,52 +396,68 @@ def _instantiator_from_union(
                 nargs = b.nargs
                 first = False
             elif nargs != b.nargs:
-                raise UnsupportedTypeAnnotationError(
-                    f"All options in {typ} must expect the same number of CLI"
-                    " arguments. (this is a limitation from argparse's nargs"
-                    " option)"
-                )
+                # Just be as general as possible if we see inconsistencies.
+                nargs = "+"
 
-    # Set metavar.
-    if nargs is not None and options[0] is NoneType:
-        # For optional types + sequences, the only way to set a field to None will
-        # be to omit its corresponding argument.
-        instantiators.pop(0)
-        metas.pop(0)
-
-    metavar: Union[str, Tuple[str, ...]]
-    if all(map(lambda m: isinstance(m.metavar, str), metas)):
-        metavar = _join_union_metavars(map(lambda x: cast(str, x.metavar), metas))
-    elif all(map(lambda m: isinstance(m.metavar, tuple), metas)):
-        # Do our best to create a reasonable looking metavar. This is imperfect!
-        assert isinstance(metas[0].metavar, tuple)
-        metavar = tuple(
-            map(
-                lambda metavars: _join_union_metavars(metavars),
-                zip(*map(lambda m: m.metavar, metas)),
-            )
-        )
-    else:
-        # Should never hit this case.
-        assert False
+    metavar: str
+    metavar = _join_union_metavars(map(lambda x: cast(str, x.metavar), metas))
 
     def union_instantiator(string_or_strings: Union[str, List[str]]) -> Any:
-        for instantiator, metadata in zip(instantiators, metas):
-            if metadata.choices is None or string_or_strings in metadata.choices:
+        metadata: InstantiatorMetadata
+        errors = []
+        for i, (instantiator, metadata) in enumerate(zip(instantiators, metas)):
+            # Check choices.
+            if metadata.choices is not None and (
+                (
+                    isinstance(string_or_strings, str)
+                    and string_or_strings not in metadata.choices
+                )
+                or (
+                    isinstance(string_or_strings, list)
+                    and any(x not in metadata.choices for x in string_or_strings)
+                )
+            ):
+                errors.append(
+                    f"{options[i]}: {string_or_strings} does not match choices"
+                    f" {metadata.choices}"
+                )
+                continue
+
+            # Try passing input directly into instantiator.
+            if metadata.nargs == nargs or (
+                isinstance(metadata.nargs, int) and nargs == "+"
+            ):
                 try:
                     return instantiator(string_or_strings)  # type: ignore
-                except ValueError:
+                except ValueError as e:
+                    # Failed, try next instantiator.
+                    errors.append(f"{options[i]}: {e.args[0]}")
+
+            # Try passing unwrapped length-1 input into instantiator.
+            elif (
+                metadata.nargs is None
+                and nargs is not None
+                and len(string_or_strings) == 1
+            ):
+                try:
+                    return instantiator(string_or_strings[0])  # type: ignore
+                except ValueError as e:
                     # Failed, try next instantiator.
-                    pass
+                    errors.append(f"{options[i]}: {e.args[0]}")
+            else:
+                errors.append(
+                    f"{options[i]}: did not attempt,"
+                    f" {metadata} {nargs} {len(string_or_strings)}"
+                )
         raise ValueError(
-            f"No type in {options} could be instantiated from {string_or_strings}."
+            f"no type in {options} could be instantiated from"
+            f" {string_or_strings}.\n\nGot errors:  \n- " + "\n- ".join(errors)
         )
 
     return union_instantiator, InstantiatorMetadata(
         nargs=nargs,
         metavar=metavar,
         choices=None,
-        is_optional=NoneType in options,
     )
 
 
@@ -466,7 +498,6 @@ def dict_instantiator(strings: List[str]) -> Any:
         nargs="+",
         metavar=f"{key_metadata.metavar} {val_metadata.metavar}",
         choices=None,
-        is_optional=False,
     )
 
 
@@ -488,9 +519,8 @@ def _instantiator_from_list_sequence_or_set(
         [make(x) for x in strings]
     ), InstantiatorMetadata(
         nargs="+",
-        metavar=inner_meta.metavar,
+        metavar=f"{inner_meta.metavar} [{inner_meta.metavar} ...]",
         choices=inner_meta.choices,
-        is_optional=False,
     )
 
 
@@ -519,5 +549,4 @@ def _instantiator_from_literal(
         metadata,
         choices=choices,
         metavar="{" + ",".join(map(_format_metavar, map(str, choices))) + "}",
-        is_optional=False,
     )
diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py
index 2a12bc9cb..5aa7b5614 100644
--- a/dcargs/_parsers.py
+++ b/dcargs/_parsers.py
@@ -99,7 +99,9 @@ def from_callable(
         description: Optional[str],
         parent_classes: Set[Type],
         parent_type_from_typevar: Optional[Dict[TypeVar, Type]],
-        default_instance: Optional[T],
+        default_instance: Union[
+            T, _fields.PropagatingMissingType, _fields.NonpropagatingMissingType
+        ],
         prefix: str,
         avoid_subparsers: bool,
     ) -> ParserSpecification:
@@ -330,7 +332,7 @@ class SubparsersSpecification:
     description: Optional[str]
     parser_from_name: Dict[str, ParserSpecification]
     required: bool
-    default_instance: Optional[Any]
+    default_instance: Any
     can_be_none: bool  # If underlying type is Optional[Something].
 
     @staticmethod
@@ -371,12 +373,15 @@ def from_field(
         can_be_none = options != options_no_none  # Optional[] type.
         if can_be_none:
             required = False
-        elif field.default is not None:
+        elif field.default not in _fields.MISSING_SINGLETONS:
             required = False
 
         # If there are any required arguments in the default subparser, we should mark
         # the subparser group as a whole as required.
-        if field.default is not None:
+        if (
+            field.default is not None
+            and field.default not in _fields.MISSING_SINGLETONS
+        ):
             default_parser = parser_from_name[
                 _strings.subparser_name_from_type(type(field.default))
             ]
@@ -393,7 +398,7 @@ def from_field(
         description_parts = []
         if field.helptext is not None:
             description_parts.append(field.helptext)
-        if not required and field.default is not None:
+        if not required and field.default not in _fields.MISSING_SINGLETONS:
             default = _strings.subparser_name_from_type(type(field.default))
             description_parts.append(f" (default: {default})")
 
@@ -405,6 +410,8 @@ def from_field(
             description=" ".join(description_parts),
             parser_from_name=parser_from_name,
             required=required,
-            default_instance=field.default,
+            default_instance=field.default
+            if field.default not in _fields.MISSING_SINGLETONS
+            else None,
             can_be_none=can_be_none,
         )
diff --git a/dcargs/_serialization.py b/dcargs/_serialization.py
index 7279efae8..73a431f5d 100644
--- a/dcargs/_serialization.py
+++ b/dcargs/_serialization.py
@@ -116,7 +116,7 @@ def make_enum_constructor(typ: Type):
 
     DataclassLoader.add_constructor(
         tag=MISSING_YAML_TAG_PREFIX,
-        constructor=lambda *_unused: _fields.MISSING,  # type: ignore
+        constructor=lambda *_unused: _fields.MISSING_PROP,  # type: ignore
     )
 
     return DataclassLoader
@@ -125,7 +125,7 @@ def make_enum_constructor(typ: Type):
 def _make_dumper(instance: Any) -> Type[yaml.Dumper]:
     class DataclassDumper(yaml.Dumper):
         def ignore_aliases(self, data):
-            return super().ignore_aliases(data) or data is _fields.MISSING
+            return super().ignore_aliases(data) or data is _fields.MISSING_PROP
 
     contained_types = list(_get_contained_special_types_from_instance(instance))
     contained_type_names = list(map(lambda cls: cls.__name__, contained_types))
@@ -163,7 +163,7 @@ def representer(dumper: DataclassDumper, data: Any) -> yaml.Node:
         DataclassDumper.add_representer(typ, make_representer(name))
 
     DataclassDumper.add_representer(
-        type(_fields.MISSING),
+        type(_fields.MISSING_PROP),
         lambda dumper, data: dumper.represent_scalar(
             tag=MISSING_YAML_TAG_PREFIX, value=""
         ),
diff --git a/examples/01_functions.py b/examples/01_functions.py
index 819aedeba..c51af61f8 100644
--- a/examples/01_functions.py
+++ b/examples/01_functions.py
@@ -4,7 +4,7 @@
 Usage:
 `python ./01_functions.py --help`
 `python ./01_functions.py --field1 hello`
-`python ./01_functions.py --field1 hello --flag`
+`python ./01_functions.py --field1 hello --field2 10`
 """
 
 import dcargs
@@ -13,16 +13,14 @@
 def main(
     field1: str,
     field2: int = 3,
-    flag: bool = False,
 ) -> None:
     """Function, whose arguments will be populated from a CLI interface.
 
     Args:
         field1: A string field.
         field2: A numeric field, with a default value.
-        flag: A boolean flag.
     """
-    print(field1, field2, flag)
+    print(field1, field2)
 
 
 if __name__ == "__main__":
diff --git a/examples/02_dataclasses.py b/examples/02_dataclasses.py
index 64fc65e48..4993138bc 100644
--- a/examples/02_dataclasses.py
+++ b/examples/02_dataclasses.py
@@ -1,9 +1,10 @@
-"""Common pattern: use `dcargs.cli()` to instantiate a dataclass.
+"""Common pattern: use `dcargs.cli()` to instantiate a dataclass. The outputted instance
+can be used as a typed alternative for an argparse namespace.
 
 Usage:
 `python ./02_dataclasses.py --help`
 `python ./02_dataclasses.py --field1 hello`
-`python ./02_dataclasses.py --field1 hello --flag`
+`python ./02_dataclasses.py --field1 hello --field2 5`
 """
 
 import dataclasses
@@ -18,7 +19,6 @@ class Args:
 
     field1: str  # A string field.
     field2: int = 3  # A numeric field, with a default value.
-    flag: bool = False  # A boolean flag.
 
 
 if __name__ == "__main__":
diff --git a/examples/06_base_configs.py b/examples/06_base_configs.py
index 09977ed22..7994790b0 100644
--- a/examples/06_base_configs.py
+++ b/examples/06_base_configs.py
@@ -12,7 +12,7 @@
 
 import sys
 from dataclasses import dataclass
-from typing import Dict, Literal, Tuple, Type, TypeVar, Union
+from typing import Dict, Literal, Tuple, TypeVar, Union
 
 import dcargs
 
diff --git a/examples/07_literals_and_unions.py b/examples/07_literals_and_unions.py
index 9438ccd95..ea3300837 100644
--- a/examples/07_literals_and_unions.py
+++ b/examples/07_literals_and_unions.py
@@ -3,15 +3,11 @@
 
 Usage:
 `python ./07_literals_and_unions.py --help`
-`python ./07_literals_and_unions.py --enum RED --restricted-enum GREEN --integer 3 --string green`
-`python ./07_literals_and_unions.py --string-or-enum green`
-`python ./07_literals_and_unions.py --string-or-enum RED`
-`python ./07_literals_and_unions.py --tuple-of-string-or-enum RED green BLUE`
 """
 
 import dataclasses
 import enum
-from typing import Literal, Tuple, Union
+from typing import Literal, Optional, Tuple, Union
 
 import dcargs
 
@@ -24,11 +20,21 @@ class Color(enum.Enum):
 
 @dataclasses.dataclass(frozen=True)
 class Args:
-    enum: Color = Color.RED
+    # We can use Literal[] to restrict the set of allowable inputs, for example, over
+    # enums.
     restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED
-    integer: Literal[0, 1, 2, 3] = 0
-    string: Literal["red", "green"] = "red"
+
+    # Literals can also be marked Optional.
+    integer: Optional[Literal[0, 1, 2, 3]] = None
+
+    # Unions can be used to specify multiple allowable types.
+    union_over_types: Union[int, str] = 0
     string_or_enum: Union[Literal["red", "green"], Color] = "red"
+
+    # Unions also work over more complex nested types.
+    union_over_tuples: Union[Tuple[int, int], Tuple[str]] = ("1",)
+
+    # And can be nested in other types.
     tuple_of_string_or_enum: Tuple[Union[Literal["red", "green"], Color], ...] = (
         "red",
         Color.RED,
diff --git a/examples/09_subparsers.py b/examples/09_subparsers.py
index 7098df75f..34808114c 100644
--- a/examples/09_subparsers.py
+++ b/examples/09_subparsers.py
@@ -19,12 +19,14 @@
 @dataclasses.dataclass(frozen=True)
 class Checkout:
     """Checkout a branch."""
+
     branch: str
 
 
 @dataclasses.dataclass(frozen=True)
 class Commit:
     """Commit changes."""
+
     message: str
     all: bool = False
 
diff --git a/setup.py b/setup.py
index d5f23f995..383fe10cf 100644
--- a/setup.py
+++ b/setup.py
@@ -30,6 +30,7 @@
             "pytest-cov",
             "omegaconf",
             "attrs",
+            "torch",
         ],
         "type-checking": [
             "mypy",
diff --git a/tests/test_collections.py b/tests/test_collections.py
index 68dcdd3ab..d7b4f3701 100644
--- a/tests/test_collections.py
+++ b/tests/test_collections.py
@@ -99,7 +99,7 @@ class A:
 def test_tuples_variable_optional():
     @dataclasses.dataclass
     class A:
-        x: Optional[Tuple[int, ...]]
+        x: Optional[Tuple[int, ...]] = None
 
     assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3))
     with pytest.raises(SystemExit):
@@ -250,7 +250,7 @@ class A:
 def test_optional_sequences():
     @dataclasses.dataclass
     class A:
-        x: Optional[Sequence[int]]
+        x: Optional[Sequence[int]] = None
 
     assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3])
     with pytest.raises(SystemExit):
@@ -261,7 +261,7 @@ class A:
 def test_optional_lists():
     @dataclasses.dataclass
     class A:
-        x: Optional[List[int]]
+        x: Optional[List[int]] = None
 
     assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3])
     with pytest.raises(SystemExit):
@@ -296,3 +296,19 @@ def main(a: Union[Tuple[str, float, str], Tuple[str, str, float]]) -> Any:
 
     assert dcargs.cli(main, args="--a 3.3 hey 7.0".split(" ")) == ("3.3", "hey", 7.0)
     assert dcargs.cli(main, args="--a 3 3 hey".split(" ")) == ("3", 3.0, "hey")
+
+
+def test_union_over_collections_3():
+    def main(a: Union[Tuple[int, int], Tuple[int, int, int]]) -> Tuple[int, ...]:
+        return a
+
+    assert dcargs.cli(main, args=["--a", "5", "5"]) == (5, 5)
+    assert dcargs.cli(main, args=["--a", "1", "2", "3"]) == (1, 2, 3)
+
+    with pytest.raises(SystemExit):
+        dcargs.cli(main, args=["--a", "5", "5", "2", "1"])
+
+    with pytest.raises(SystemExit):
+        dcargs.cli(main, args=["--a"])
+    with pytest.raises(SystemExit):
+        dcargs.cli(main, args=[])
diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py
index a7bc44481..86b9c3221 100644
--- a/tests/test_dcargs.py
+++ b/tests/test_dcargs.py
@@ -2,9 +2,10 @@
 import dataclasses
 import enum
 import pathlib
-from typing import Any, AnyStr, Callable, ClassVar, Optional, TypeVar, Union
+from typing import Any, AnyStr, Callable, ClassVar, List, Optional, TypeVar, Union
 
 import pytest
+import torch
 from typing_extensions import Annotated, Final, Literal, TypeAlias
 
 import dcargs
@@ -198,12 +199,32 @@ class A:
 def test_optional():
     @dataclasses.dataclass
     class A:
-        x: Optional[int]
+        x: Optional[int] = None
 
     assert dcargs.cli(A, args=[]) == A(x=None)
 
 
-def test_union():
+def test_union_basic():
+    def main(x: Union[int, str]) -> Union[int, str]:
+        return x
+
+    assert dcargs.cli(main, args=["--x", "5"]) == 5
+    assert dcargs.cli(main, args=["--x", "6"]) == 6
+    assert dcargs.cli(main, args=["--x", "five"]) == "five"
+
+
+def test_union_with_list():
+    def main(x: Union[int, str, List[bool]]) -> Any:
+        return x
+
+    assert dcargs.cli(main, args=["--x", "5"]) == 5
+    assert dcargs.cli(main, args=["--x", "6"]) == 6
+    assert dcargs.cli(main, args=["--x", "five"]) == "five"
+    assert dcargs.cli(main, args=["--x", "True"]) == "True"
+    assert dcargs.cli(main, args=["--x", "True", "False"]) == [True, False]
+
+
+def test_union_literal():
     def main(x: Union[Literal[1, 2], Literal[3, 4, 5], str]) -> Union[int, str]:
         return x
 
@@ -299,7 +320,7 @@ class A:
 def test_optional_literal():
     @dataclasses.dataclass
     class A:
-        x: Optional[Literal[0, 1, 2]]
+        x: Optional[Literal[0, 1, 2]] = None
 
     assert dcargs.cli(A, args=["--x", "1"]) == A(x=1)
     with pytest.raises(SystemExit):
@@ -425,3 +446,10 @@ def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]:
 
 def test_missing_singleton():
     assert dcargs.MISSING is copy.deepcopy(dcargs.MISSING)
+
+
+def test_torch_device():
+    def main(device: torch.device) -> torch.device:
+        return device
+
+    assert dcargs.cli(main, args=["--device", "cpu"]) == torch.device("cpu")
diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py
index 719826966..862c45a77 100644
--- a/tests/test_dict_namedtuple.py
+++ b/tests/test_dict_namedtuple.py
@@ -169,7 +169,8 @@ class HelptextNamedTupleDefault(NamedTuple):
     assert cast(str, HelptextNamedTupleDefault.__doc__) in helptext
     assert "--x INT  Documentation 1 (required)\n" in helptext
     assert "--y INT  Documentation 2 (required)\n" in helptext
-    assert "--z INT  Documentation 3 (default: 3)\n" in helptext
+    assert "--z INT" in helptext
+    assert "Documentation 3 (default: 3)\n" in helptext
 
 
 def test_helptext_and_default_instance_namedtuple():
@@ -184,13 +185,19 @@ class HelptextNamedTuple(NamedTuple):
         z: int
         """Documentation 3"""
 
+    with pytest.raises(SystemExit):
+        dcargs.cli(
+            HelptextNamedTuple,
+            default_instance=dcargs.MISSING,
+            args=[],
+        )
+
     f = io.StringIO()
     with pytest.raises(SystemExit):
         with contextlib.redirect_stdout(f):
             dcargs.cli(
                 HelptextNamedTuple,
                 default_instance=HelptextNamedTuple(
-                    # Sketchy, unsupported behavior...
                     x=dcargs.MISSING,
                     y=dcargs.MISSING,
                     z=3,
@@ -201,4 +208,5 @@ class HelptextNamedTuple(NamedTuple):
     assert cast(str, HelptextNamedTuple.__doc__) in helptext
     assert "Documentation 1 (required)\n" in helptext
     assert "Documentation 2 (required)\n" in helptext
-    assert "Documentation 3 (default: 3)\n" in helptext
+    assert "Documentation 3" in helptext
+    assert "(default: 3)\n" in helptext
diff --git a/tests/test_errors.py b/tests/test_errors.py
index 07842d874..b1edb6162 100644
--- a/tests/test_errors.py
+++ b/tests/test_errors.py
@@ -1,5 +1,5 @@
 import dataclasses
-from typing import Generic, List, Tuple, TypeVar, Union
+from typing import Generic, List, Tuple, TypeVar
 
 import pytest
 from typing_extensions import Literal
@@ -135,14 +135,6 @@ class ChildClass(UnrelatedParentClass, ActualParentClass[int]):
         dcargs.cli(ChildClass, args=["--x", "1", "--y", "2", "--z", "3"])
 
 
-def test_unsupported_union():
-    def main(a: Union[Tuple[int, int], Tuple[int, int, int]]) -> None:
-        pass
-
-    with pytest.raises(dcargs.UnsupportedTypeAnnotationError):
-        dcargs.cli(main, args=["--a", "5", "5"])
-
-
 def test_missing_annotation():
     def main(a) -> None:
         pass
diff --git a/tests/test_helptext.py b/tests/test_helptext.py
index bbf0bffc9..6e39fb61c 100644
--- a/tests/test_helptext.py
+++ b/tests/test_helptext.py
@@ -216,7 +216,8 @@ class HelptextHardString:
         with contextlib.redirect_stdout(f):
             dcargs.cli(HelptextHardString, args=["--help"])
     helptext = dcargs._strings.strip_color_codes(f.getvalue())
-    assert "--x         Helptext. 2% milk.\n" in helptext
+    assert "--x" in helptext
+    assert "Helptext. 2% milk." in helptext
 
 
 def test_helptext_with_inheritance():
@@ -361,7 +362,7 @@ class LiteralHelptext:
 def test_optional_literal_helptext():
     @dataclasses.dataclass
     class OptionalLiteralHelptext:
-        x: Optional[Literal[1, 2, 3]]
+        x: Optional[Literal[1, 2, 3]] = None
         """A number."""
 
     f = io.StringIO()
@@ -439,6 +440,6 @@ class OptionalHelptext:
             dcargs.cli(OptionalHelptext, args=["--help"])
     helptext = dcargs._strings.strip_color_codes(f.getvalue())
     assert cast(str, OptionalHelptext.__doc__) in helptext
-    assert "[--x {None}|INT]" in helptext
+    assert "--x {None}|INT" in helptext
     assert "--y {None}|INT [{None}|INT ...]\n" in helptext
     assert "[--z {None}|INT]\n" in helptext
diff --git a/tests/test_nested.py b/tests/test_nested.py
index 069588473..55ab661e8 100644
--- a/tests/test_nested.py
+++ b/tests/test_nested.py
@@ -21,6 +21,21 @@ class Nested:
         dcargs.cli(Nested, args=["--x", "1"])
 
 
+def test_nested_default_instance():
+    @dataclasses.dataclass
+    class B:
+        y: int = 1
+
+    @dataclasses.dataclass
+    class Nested:
+        x: int = 2
+        b: B = B()
+
+    assert dcargs.cli(
+        Nested, args=[], default_instance=Nested(x=1, b=B(y=2))
+    ) == Nested(x=1, b=B(y=2))
+
+
 def test_nested_default():
     @dataclasses.dataclass
     class B:
diff --git a/tests/test_positional_ignore_py37.py b/tests/test_positional_ignore_py37.py
index dd66f26cf..5c6af1a59 100644
--- a/tests/test_positional_ignore_py37.py
+++ b/tests/test_positional_ignore_py37.py
@@ -101,10 +101,30 @@ def main(
         dcargs.cli(main, args=["True", "false"])
 
 
-def test_unsupported_positional():
-    # Not super clear how to parse optional positional sequences...
-    def main(a: Optional[List[int]], /) -> None:
-        pass
+def test_optional_list():
+    def main(a: Optional[List[int]], /) -> Optional[List[int]]:
+        return a
 
-    with pytest.raises(dcargs.UnsupportedTypeAnnotationError):
-        dcargs.cli(main, args=["--a", "5", "5"])
+    assert dcargs.cli(main, args=["None"]) is None
+    assert dcargs.cli(main, args=["1", "2"]) == [1, 2]
+    with pytest.raises(SystemExit):
+        dcargs.cli(main, args=[])
+    with pytest.raises(SystemExit):
+        dcargs.cli(main, args=["hm"])
+
+
+def test_optional_list_with_default():
+    def main(a: Optional[List[int]] = None, /) -> Optional[List[int]]:
+        return a
+
+    assert dcargs.cli(main, args=["None"]) is None
+    assert dcargs.cli(main, args=["5", "5"]) == [5, 5]
+    with pytest.raises(SystemExit):
+        dcargs.cli(main, args=["None", "5"])
+
+
+def test_positional_tuple():
+    def main(x: Tuple[int, int], y: Tuple[str, str], /):
+        return x, y
+
+    assert dcargs.cli(main, args="1 2 3 4".split(" ")) == ((1, 2), ("3", "4"))

From 745c13f949308427db9e880b6d81792135827a1e Mon Sep 17 00:00:00 2001
From: Brent Yi 
Date: Mon, 18 Jul 2022 01:42:53 -0700
Subject: [PATCH 061/491] Nested collections, ANSI-aware wrapping, polish

---
 README.md                                 | 428 ++++++++++++----------
 _update_readme.py                         |  42 ++-
 dcargs/_argparse_formatter.py             |  24 +-
 dcargs/_arguments.py                      |  64 ++--
 dcargs/_cli.py                            |  13 +-
 dcargs/_docstrings.py                     |   3 +-
 dcargs/_fields.py                         |  17 +-
 dcargs/_instantiators.py                  | 213 +++++++----
 dcargs/_parsers.py                        |  21 +-
 dcargs/_resolver.py                       |   4 +-
 dcargs/_serialization.py                  |   2 +-
 dcargs/_strings.py                        |  47 +--
 examples/05_hierarchical_configs.py       |   1 -
 examples/06_base_configs.py               |   9 +-
 examples/11_dictionaries.py               |  19 +-
 setup.py                                  |   1 -
 tests/test_collections.py                 |  70 ++++
 tests/test_dict_namedtuple.py             |  18 +-
 tests/test_errors.py                      |  40 +-
 tests/test_helptext.py                    | 229 ++++++------
 tests/test_unsupported_but_should_work.py |   2 +-
 21 files changed, 764 insertions(+), 503 deletions(-)

diff --git a/README.md b/README.md
index 5c82ded5f..7d53c0f2a 100644
--- a/README.md
+++ b/README.md
@@ -1,30 +1,98 @@
 # dcargs
 
+```bash
+pip install dcargs
+```
+
 ![build](https://github.com/brentyi/dcargs/workflows/build/badge.svg)
 ![mypy](https://github.com/brentyi/dcargs/workflows/mypy/badge.svg?branch=master)
 ![lint](https://github.com/brentyi/dcargs/workflows/lint/badge.svg)
 [![codecov](https://codecov.io/gh/brentyi/dcargs/branch/master/graph/badge.svg)](https://codecov.io/gh/brentyi/dcargs)
 
 - [Overview](#overview)
+- [API](#api)
 - [Examples](#examples)
 - [Serialization](#serialization)
 - [Alternative tools](#alternative-tools)
 
 ## Overview
 
-```bash
-pip install dcargs
+**`dcargs`** is a library for typed CLI interfaces and configuration objects.
+
+Our core interface generates argument parsers from type-annotated callables. In
+the simplest case, this can be used as a drop-in replacement for `argparse`:
+
+
+
+
+
+
+
+
+
+
+
with argparsewith dcargs

+
+```python
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--a", type=int, required=True)
+parser.add_argument("--b", type=int, default=1)
+args = parser.parse_args()
+
+print(args.a + args.b)
 ```
 
-**`dcargs`** is a library for typed CLI interfaces and configuration objects.
+
+
+```python
+import dcargs
+
+def main(a: int, b: int = 3) -> None:
+    print(a + b)
+
+dcargs.cli(main)
+```
+
+

 
-Our core interface generates an argument parser from a type-annotated callable
-_`f`_, which may be a function, class, or dataclass:
+The broader goal is a replacement for tools like `argparse`, `hydra`, and
+`ml_collections` that's:
+
+- **Low effort.** Standard Python type annotations, docstrings, and default
+  values are parsed to automatically generate argument parsers with informative
+  helptext.
+
+- **Expressive.** `dcargs.cli()` understands functions, classes, dataclasses,
+  and _nested_ classes and dataclasses, as well as frequently used annotations
+  like unions, literals, collections, and generics, which can be composed into
+  hierarchical configuration objects built on standard Python features.
+
+- **Typed.** Unlike dynamic configuration namespaces produced by libraries like
+  `argparse`, `YACS`, `abseil`, `hydra`, or `ml_collections`, typed outputs mean
+  that IDE-assisted autocomplete, rename, refactor, and go-to-definition
+  operations work out-of-the-box, as well as static checking tools like `mypy`
+  and `pyright`.
+
+- **Modular.** Most approaches to configuration objects require a centralized
+  definition of all configurable fields. Hierarchically nesting configuration
+  structures, however, makes it easy to distribute definitions, defaults, and
+  documentation of configurable fields across modules or source files. A model
+  configuration dataclass, for example, can be co-located in its entirety with
+  the model implementation and dropped into any experiment configuration with an
+  import — this eliminates redundancy and makes entire modules easy to port
+  across codebases.
+
+## API
+
+
 
 ```python
-dcargs.cli(
+def cli(
     f: Callable[..., T],
     *,
+    prog: Optional[str] = None,
     description: Optional[str] = None,
     args: Optional[Sequence[str]] = None,
     default_instance: Optional[T] = None,
@@ -32,8 +100,10 @@ dcargs.cli(
 ) -> T
 ```
 
+
+
 

-Docstring
+Docstring
 
 
 
@@ -98,33 +168,6 @@ Returns:
 
 

 
----
-
-The goal is a tool that's lightweight enough for simple interactive scripts, but
-flexible enough to replace heavier configuration frameworks like `hydra` and
-`ml_collections`. Notably, `dcargs.cli()` supports _nested_ classes and
-dataclasses, which enable expressive hierarchical configuration objects built on
-standard Python features.
-
-Ultimately, we aim to enable configuration interfaces that are:
-
-- **Low-effort.** Type annotations, docstrings, and default values can be used
-  to automatically generate argument parsers with informative helptext. This
-  includes bells and whistles like enums, containers, etc.
-- **Strongly typed.** Unlike dynamic configuration namespaces produced by
-  libraries like `argparse`, `YACS`, `abseil`, `hydra`, or `ml_collections`,
-  typed outputs mean that IDE-assisted autocomplete, rename, refactor,
-  go-to-definition operations work out-of-the-box, as do static checking tools
-  like `mypy` and `pyright`.
-- **Modular.** Most approaches to configuration objects require a centralized
-  definition of all configurable fields. Supporting hierarchically nested
-  configuration structures, however, makes it easy to distribute definitions,
-  defaults, and documentation of configurable fields across modules or source
-  files. A model configuration dataclass, for example, can be co-located in its
-  entirety with the model implementation and dropped into any experiment
-  configuration with an import — this eliminates redundancy and makes the entire
-  module easy to port across codebases.
-
 ## Examples
 
 
@@ -134,8 +177,8 @@ Ultimately, we aim to enable configuration interfaces that are:
 
 

 
-In the simplest case, `dcargs.cli()` can be used to run a function with arguments
-populated from the CLI.
+In the simplest case, `dcargs.cli()` can be used to run a function with
+arguments populated from the CLI.
 
 **Code ([link](examples/01_functions.py)):**
 
@@ -171,7 +214,7 @@ usage: 01_functions.py [-h] --field1 STR [--field2 INT]
 Function, whose arguments will be populated from a CLI interface.
 
 arguments:
-  -h, --help            show this help message and exit
+  -h, --help    show this help message and exit
   --field1 STR  A string field. (required)
   --field2 INT  A numeric field, with a default value. (default: 3)
 

@@ -195,8 +238,8 @@ hello 10
 
 

 
-Common pattern: use `dcargs.cli()` to instantiate a dataclass. The outputted instance
-can be used as a typed alternative for an argparse namespace.
+Common pattern: use `dcargs.cli()` to instantiate a dataclass. The outputted
+instance can be used as a typed alternative for an argparse namespace.
 
 **Code ([link](examples/02_dataclasses.py)):**
 
@@ -228,10 +271,11 @@ if __name__ == "__main__":
 $ python ./02_dataclasses.py --help
 usage: 02_dataclasses.py [-h] --field1 STR [--field2 INT]
 
-Description. This should show up in the helptext!
+Description.
+This should show up in the helptext!
 
 arguments:
-  -h, --help            show this help message and exit
+  -h, --help    show this help message and exit
   --field1 STR  A string field. (required)
   --field2 INT  A numeric field, with a default value. (default: 3)
 

@@ -255,8 +299,8 @@ Args(field1='hello', field2=5)
 
 

 
-We can generate argument parsers from more advanced type annotations, like enums and
-tuple types.
+We can generate argument parsers from more advanced type annotations, like enums
+and tuple types.
 
 **Code ([link](examples/03_enums_and_containers.py)):**
 
@@ -316,11 +360,14 @@ arguments:
                         Paths to load training data from. This can be
                         multiple! (required)
   --image-dimensions INT INT
-                        Height and width of some image data. (default: 32 32)
+                        Height and width of some image data. (default: 32
+                        32)
   --optimizer-type {ADAM,SGD}
-                        Gradient-based optimizer to use. (default: ADAM)
+                        Gradient-based optimizer to use. (default:
+                        ADAM)
   --checkpoint-interval {None}|INT
-                        Interval to save checkpoints at. (default: None)
+                        Interval to save checkpoints at. (default:
+                        None)
 
@@ -342,8 +389,8 @@ TrainConfig(dataset_sources=(PosixPath('data'),), image_dimensions=(32
 
 

 
-Booleans can either be expected to be explicitly passed in, or, if given a default
-value, automatically converted to flags.
+Booleans can either be expected to be explicitly passed in, or, if given a
+default value, automatically converted to flags.
 
 **Code ([link](examples/04_flags.py)):**
 
@@ -391,10 +438,10 @@ arguments:
   --optional-boolean {None,True,False}
                         Optional boolean. Same as above, but can be omitted.
                         (required)
-  --flag-a              Pass --flag-a in to set this value to True. (sets
+  --flag-a              Pass --flag-a in to set this value to True. (sets:
                         flag_a=True)
-  --no-flag-b           Pass --no-flag-b in to set this value to False. (sets
-                        flag_b=False)
+  --no-flag-b           Pass --no-flag-b in to set this value to False.
+                        (sets: flag_b=False)
 
@@ -475,7 +522,6 @@ class ExperimentConfig:
 
 def train(
     out_dir: pathlib.Path,
-    /,
     config: ExperimentConfig,
     restore_checkpoint: bool = False,
     checkpoint_interval: int = 1000,
@@ -503,7 +549,7 @@ if __name__ == "__main__":
 
 
 $ python ./05_hierarchical_configs.py --help
-usage: 05_hierarchical_configs.py [-h]
+usage: 05_hierarchical_configs.py [-h] --out-dir PATH
                                   [--config.optimizer.algorithm {ADAM,SGD}]
                                   [--config.optimizer.learning-rate FLOAT]
                                   [--config.optimizer.weight-decay FLOAT]
@@ -511,30 +557,30 @@ usage: 05_hierarchical_configs.py [-h]
                                   [--config.train-steps INT]
                                   [--config.seed INT] [--restore-checkpoint]
                                   [--checkpoint-interval INT]
-                                  OUT_DIR
 
 Train a model.
 
-positional arguments:
-  OUT_DIR               Where to save logs and checkpoints. (required)
-
 arguments:
   -h, --help            show this help message and exit
-  --restore-checkpoint  Set to restore an existing checkpoint. (sets
+  --out-dir PATH  Where to save logs and checkpoints.
+                        (required)
+  --restore-checkpoint  Set to restore an existing checkpoint. (sets:
                         restore_checkpoint=True)
   --checkpoint-interval INT
-                        Training steps between each checkpoint save. (default:
-                        1000)
+                        Training steps between each checkpoint save.
+                        (default: 1000)
 
 config.optimizer arguments:
   Various configurable options for our optimizer.
 
   --config.optimizer.algorithm {ADAM,SGD}
-                        Gradient-based optimizer to use. (default: ADAM)
+                        Gradient-based optimizer to use. (default:
+                        ADAM)
   --config.optimizer.learning-rate FLOAT
                         Learning rate to use. (default: 0.0003)
   --config.optimizer.weight-decay FLOAT
-                        Coefficient for L2 regularization. (default: 0.01)
+                        Coefficient for L2 regularization. (default:
+                        0.01)
 
 config arguments:
   Experiment configuration.
@@ -542,40 +588,36 @@ config arguments:
   --config.batch-size INT
                         Batch size. (default: 32)
   --config.train-steps INT
-                        Total number of training steps. (default: 100000)
-  --config.seed INT
-                        Random seed. This is helpful for making sure that our
+                        Total number of training steps. (default:
+                        100000)
+  --config.seed INT  Random seed. This is helpful for making sure that our
                         experiments are all reproducible! (default: 0)
 

 
 
 $ python ./05_hierarchical_configs.py . --config.optimizer.algorithm SGD
-out_dir=PosixPath('.'), restore_checkpoint=False, checkpoint_interval=1000
-config=ExperimentConfig(optimizer=OptimizerConfig(algorithm=<OptimizerType.SGD: 2>, learning_rate=0.0003, weight_decay=0.01), batch_size=32, train_steps=100000, seed=0)
-# dcargs YAML.
-!dataclass:ExperimentConfig
-batch_size: 32
-optimizer: !dataclass:OptimizerConfig
-  algorithm: !enum:OptimizerType 'SGD'
-  learning_rate: 0.0003
-  weight_decay: 0.01
-seed: 0
-train_steps: 100000
+usage: 05_hierarchical_configs.py [-h] --out-dir PATH
+                                  [--config.optimizer.algorithm {ADAM,SGD}]
+                                  [--config.optimizer.learning-rate FLOAT]
+                                  [--config.optimizer.weight-decay FLOAT]
+                                  [--config.batch-size INT]
+                                  [--config.train-steps INT]
+                                  [--config.seed INT] [--restore-checkpoint]
+                                  [--checkpoint-interval INT]
+05_hierarchical_configs.py: error: the following arguments are required: --out-dir
 

 
 
 $ python ./05_hierarchical_configs.py . --restore-checkpoint
-out_dir=PosixPath('.'), restore_checkpoint=True, checkpoint_interval=1000
-config=ExperimentConfig(optimizer=OptimizerConfig(algorithm=<OptimizerType.ADAM: 1>, learning_rate=0.0003, weight_decay=0.01), batch_size=32, train_steps=100000, seed=0)
-# dcargs YAML.
-!dataclass:ExperimentConfig
-batch_size: 32
-optimizer: !dataclass:OptimizerConfig
-  algorithm: !enum:OptimizerType 'ADAM'
-  learning_rate: 0.0003
-  weight_decay: 0.01
-seed: 0
-train_steps: 100000
+usage: 05_hierarchical_configs.py [-h] --out-dir PATH
+                                  [--config.optimizer.algorithm {ADAM,SGD}]
+                                  [--config.optimizer.learning-rate FLOAT]
+                                  [--config.optimizer.weight-decay FLOAT]
+                                  [--config.batch-size INT]
+                                  [--config.train-steps INT]
+                                  [--config.seed INT] [--restore-checkpoint]
+                                  [--checkpoint-interval INT]
+05_hierarchical_configs.py: error: the following arguments are required: --out-dir
@@ -587,16 +629,18 @@ train_steps: 100000
-We can integrate `dcargs.cli()` into common configuration patterns: here, we select -one of multiple possible base configurations, and then use the CLI to either override -(existing) or fill in (missing) values. +We can integrate `dcargs.cli()` into common configuration patterns: here, we +select one of multiple possible base configurations, and then use the CLI to +either override (existing) or fill in (missing) values. **Code ([link](examples/06_base_configs.py)):** ```python import sys from dataclasses import dataclass -from typing import Dict, Literal, Tuple, TypeVar, Union +from typing import Callable, Dict, Literal, Tuple, TypeVar, Union + +from torch import nn import dcargs @@ -634,6 +678,9 @@ class ExperimentConfig: # reproducible! seed: int + # Activation to use. Not specifiable via the commandline. + activation: Callable[[], nn.Module] + # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type @@ -649,6 +696,7 @@ base_configs = { # The dcargs.MISSING sentinel allows us to specify that the seed should have no # default, and needs to be populated from the CLI. seed=dcargs.MISSING, + activation=nn.ReLU, ), "big": ExperimentConfig( dataset="imagenet-50", @@ -658,6 +706,7 @@ base_configs = { units=256, train_steps=100_000, seed=dcargs.MISSING, + activation=nn.GELU, ), } @@ -710,20 +759,23 @@ usage: examples/06_base_configs.py small [-h] [--dataset {mnist,imagenet-50}] [--num-layers INT] [--units INT] [--batch-size INT] [--train-steps INT] --seed INT + [--activation {<class 'torch.nn.modules.activation.ReLU'>}] arguments: -h, --help show this help message and exit --dataset {mnist,imagenet-50} Dataset to run experiment on. (default: mnist) - --num-layers INT - Model size. (default: 4) + --num-layers INT Model size. (default: 4) --units INT Model size. (default: 64) - --batch-size INT - Batch size. (default: 2048) - --train-steps INT - Total number of training steps. (default: 30000) + --batch-size INT Batch size. (default: 2048) + --train-steps INT Total number of training steps. (default: + 30000) --seed INT Random seed. This is helpful for making sure that our - experiments are all reproducible! (required) + experiments are all reproducible! + (required) + --activation {<class 'torch.nn.modules.activation.ReLU'>} + Activation to use. Not specifiable via the + commandline. (not parsable) optimizer arguments: Optimizer parameters. @@ -734,7 +786,7 @@ optimizer arguments: 
 $ python ./06_base_configs_argv.py small --seed 94720
-ExperimentConfig(dataset='mnist', optimizer=SgdOptimizer(learning_rate=0.0003), num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720)
+ExperimentConfig(dataset='mnist', optimizer=SgdOptimizer(learning_rate=0.0003), num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720, activation=<class 'torch.nn.modules.activation.ReLU'>)
 
@@ -745,20 +797,24 @@ usage: examples/06_base_configs.py big [-h] [--dataset {mnist,imagenet-50}]
                                        [--num-layers INT] [--units INT]
                                        [--batch-size INT] [--train-steps INT]
                                        --seed INT
+                                       [--activation {<class 'torch.nn.modules.activation.GELU'>}]
 
 arguments:
   -h, --help            show this help message and exit
   --dataset {mnist,imagenet-50}
-                        Dataset to run experiment on. (default: imagenet-50)
-  --num-layers INT
-                        Model size. (default: 8)
+                        Dataset to run experiment on. (default:
+                        imagenet-50)
+  --num-layers INT  Model size. (default: 8)
   --units INT   Model size. (default: 256)
-  --batch-size INT
-                        Batch size. (default: 32)
-  --train-steps INT
-                        Total number of training steps. (default: 100000)
+  --batch-size INT  Batch size. (default: 32)
+  --train-steps INT  Total number of training steps. (default:
+                        100000)
   --seed INT    Random seed. This is helpful for making sure that our
-                        experiments are all reproducible! (required)
+                        experiments are all reproducible!
+                        (required)
+  --activation {<class 'torch.nn.modules.activation.GELU'>}
+                        Activation to use. Not specifiable via the
+                        commandline. (not parsable)
 
 optimizer arguments:
   Optimizer parameters.
@@ -771,7 +827,7 @@ optimizer arguments:
 
 
 $ python ./06_base_configs_argv.py big --seed 94720
-ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learning_rate=0.001, betas=(0.9, 0.999)), num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720)
+ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learning_rate=0.001, betas=(0.9, 0.999)), num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720, activation=<class 'torch.nn.modules.activation.GELU'>)
@@ -783,8 +839,9 @@ ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learni
-`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices; -`typing.Union[]` can be used to restrict inputs to a fixed set of types. +`typing.Literal[]` can be used to restrict inputs to a fixed set of literal +choices; `typing.Union[]` can be used to restrict inputs to a fixed set of +types. **Code ([link](examples/07_literals_and_unions.py)):** @@ -847,9 +904,11 @@ arguments: -h, --help show this help message and exit --restricted-enum {RED,GREEN} We can use Literal[] to restrict the set of allowable - inputs, for example, over enums. (default: RED) + inputs, for example, over enums. (default: + RED) --integer {None,0,1,2,3} - Literals can also be marked Optional. (default: None) + Literals can also be marked Optional. (default: + None) --union-over-types INT|STR Unions can be used to specify multiple allowable types. (default: 0) @@ -860,7 +919,8 @@ arguments: Unions also work over more complex nested types. (default: 1) --tuple-of-string-or-enum {red,green,RED,GREEN,BLUE} [{red,green,RED,GREEN,BLUE} ...] - And can be nested in other types. (default: red RED) + And can be nested in other types. (default: red + RED)
@@ -872,7 +932,8 @@ arguments:
-Positional-only arguments in functions are converted to positional CLI arguments. +Positional-only arguments in functions are converted to positional CLI +arguments. **Code ([link](examples/08_positional_args.py)):** @@ -952,21 +1013,25 @@ positional arguments: arguments: -h, --help show this help message and exit - --force Do not prompt before overwriting. (sets force=True) - --verbose Explain what is being done. (sets verbose=True) + --force Do not prompt before overwriting. (sets: + force=True) + --verbose Explain what is being done. (sets: + verbose=True) --background-rgb FLOAT FLOAT FLOAT - Background color. Red by default. (default: 1.0 0.0 - 0.0) + Background color. Red by default. (default: 1.0 + 0.0 0.0) optimizer arguments: Configuration for our optimizer object. --optimizer.algorithm {ADAM,SGD} - Gradient-based optimizer to use. (default: ADAM) + Gradient-based optimizer to use. (default: + ADAM) --optimizer.learning-rate FLOAT Learning rate to use. (default: 0.0003) --optimizer.weight-decay FLOAT - Coefficient for L2 regularization. (default: 0.01) + Coefficient for L2 regularization. (default: + 0.01) 
@@ -988,7 +1053,8 @@ background_rgb=(1.0, 0.0, 0.0)
 
 

 
-Unions over nested types (classes or dataclasses) are populated using subparsers.
+Unions over nested types (classes or dataclasses) are populated using
+subparsers.
 
 **Code ([link](examples/09_subparsers.py)):**
 
@@ -1047,12 +1113,11 @@ usage: 09_subparsers.py commit [-h] --cmd.message STR [--cmd.all]
 Commit changes.
 
 arguments:
-  -h, --help            show this help message and exit
+  -h, --help         show this help message and exit
 
 cmd arguments:
-  --cmd.message STR
-                        (required)
-  --cmd.all             (sets all=True)
+  --cmd.message STR  (required)
+  --cmd.all          (sets: all=True)
 
@@ -1067,11 +1132,10 @@ usage: 09_subparsers.py checkout [-h] --cmd.branch STR
 Checkout a branch.
 
 arguments:
-  -h, --help            show this help message and exit
+  -h, --help        show this help message and exit
 
 cmd arguments:
-  --cmd.branch STR
-                        (required)
+  --cmd.branch STR  (required)
 
@@ -1173,7 +1237,7 @@ arguments:
   -h, --help            show this help message and exit
 
 optional subcommands:
-  Dataset to train on. (default: mnist-dataset)
+  Dataset to train on.  (default: mnist-dataset)
 
   [{mnist-dataset,image-net-dataset}]
@@ -1182,17 +1246,16 @@ optional subcommands: $ python ./10_multiple_subparsers.py mnist-dataset --help usage: 10_multiple_subparsers.py mnist-dataset [-h] [--dataset.binary] [{adam-optimizer,sgd-optimizer}] - ... arguments: -h, --help show this help message and exit dataset arguments: - --dataset.binary Set to load binary version of MNIST dataset. (sets - binary=True) + --dataset.binary Set to load binary version of MNIST dataset. + (sets: binary=True) optional subcommands: - Optimizer to train with. (default: adam-optimizer) + Optimizer to train with. (default: adam-optimizer) [{adam-optimizer,sgd-optimizer}] @@ -1212,32 +1275,34 @@ AdamOptimizer(learning_rate=0.0003, betas=(0.9, 0.999))
-Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation, -or a `TypedDict` type. +Dictionary inputs can be specified using either a standard `Dict[K, V]` +annotation, or a `TypedDict` type. -Note that setting `total=False` for `TypedDict` is currently not (but reasonably could be) -supported. +Note that setting `total=False` for `TypedDict` is currently not (but reasonably +could be) supported. **Code ([link](examples/11_dictionaries.py)):** ```python -from typing import Dict, TypedDict +from typing import Dict, Tuple, TypedDict import dcargs class DictionarySchema(TypedDict): - field1: str # A string field. - field2: int # A numeric field. - field3: bool # A boolean field. + learning_rate: float + betas: Tuple[float, float] def main( - standard_dict: Dict[str, bool], + standard_dict: Dict[str, float] = { + "learning_rate": 3e-4, + "beta1": 0.9, + "beta2": 0.999, + }, typed_dict: DictionarySchema = { - "field1": "hey", - "field2": 3, - "field3": False, + "learning_rate": 3e-4, + "betas": (0.9, 0.999), }, ) -> None: assert isinstance(standard_dict, dict) @@ -1256,28 +1321,22 @@ if __name__ == "__main__": 
 $ python ./11_dictionaries.py --help
-usage: 11_dictionaries.py [-h] --standard-dict STR {True,False}
-                          [--typed-dict.field1 STR] [--typed-dict.field2 INT]
-                          [--typed-dict.field3]
+usage: 11_dictionaries.py [-h] [--standard-dict STR FLOAT [STR FLOAT ...]]
+                          [--typed-dict.learning-rate FLOAT]
+                          [--typed-dict.betas FLOAT FLOAT]
 
 arguments:
   -h, --help            show this help message and exit
-  --standard-dict STR {True,False}
-                        (required)
+  --standard-dict STR FLOAT [STR FLOAT ...]
+                        (default: learning_rate 0.0003 beta1 0.9 beta2
+                        0.999)
 
 typed_dict arguments:
 
-  --typed-dict.field1 STR
-                        A string field. (default: hey)
-  --typed-dict.field2 INT
-                        A numeric field. (default: 3)
-  --typed-dict.field3   A boolean field. (sets field3=True)
-
- -
-$ python ./11_dictionaries.py --standard-dict key1 True key2 False
-Standard dict: {'key1': True, 'key2': False}
-Typed dict: {'field1': 'hey', 'field2': 3, 'field3': False}
+  --typed-dict.learning-rate FLOAT
+                        (default: 0.0003)
+  --typed-dict.betas FLOAT FLOAT
+                        (default: 0.9 0.999)
@@ -1322,13 +1381,14 @@ if __name__ == "__main__": $ python ./12_named_tuples.py --help usage: 12_named_tuples.py [-h] --field1 STR [--field2 INT] [--flag] -Description. This should show up in the helptext! +Description. +This should show up in the helptext! arguments: - -h, --help show this help message and exit + -h, --help show this help message and exit --field1 STR A string field. (required) --field2 INT A numeric field, with a default value. (default: 3) - --flag A boolean flag. (sets flag=True) + --flag A boolean flag. (sets: flag=True) 
@@ -1387,10 +1447,10 @@ usage: 13_standard_classes.py [-h] --field1 STR --field2 INT [--flag]
 Arguments.
 
 arguments:
-  -h, --help            show this help message and exit
+  -h, --help    show this help message and exit
   --field1 STR  A string field. (required)
   --field2 INT  A numeric field. (required)
-  --flag                A boolean flag. (sets flag=True)
+  --flag        A boolean flag. (sets: flag=True)
 
@@ -1492,34 +1552,25 @@ point_discrete arguments:
 
 shape.a arguments:
 
-  --shape.a.x FLOAT
-                        (required)
-  --shape.a.y FLOAT
-                        (required)
-  --shape.a.z FLOAT
-                        (required)
+  --shape.a.x FLOAT  (required)
+  --shape.a.y FLOAT  (required)
+  --shape.a.z FLOAT  (required)
   --shape.a.frame-id STR
                         (required)
 
 shape.b arguments:
 
-  --shape.b.x FLOAT
-                        (required)
-  --shape.b.y FLOAT
-                        (required)
-  --shape.b.z FLOAT
-                        (required)
+  --shape.b.x FLOAT  (required)
+  --shape.b.y FLOAT  (required)
+  --shape.b.z FLOAT  (required)
   --shape.b.frame-id STR
                         (required)
 
 shape.c arguments:
 
-  --shape.c.x FLOAT
-                        (required)
-  --shape.c.y FLOAT
-                        (required)
-  --shape.c.z FLOAT
-                        (required)
+  --shape.c.x FLOAT  (required)
+  --shape.c.y FLOAT  (required)
+  --shape.c.z FLOAT  (required)
   --shape.c.frame-id STR
                         (required)
@@ -1544,6 +1595,12 @@ PyYAML, etc), explicit type references enable custom tags that are robust against code reorganization and refactor, while a PyYAML backend enables serialization of arbitrary Python objects. +Note that we generally prefer to use YAML purely for serialization, as opposed +to a configuration interface that humans are expected to manually write or +modify. Specifying things like loadable base configurations can be done directly +in Python, which enables all of the usual autocompletion and type checking +features. + ## Alternative tools The core functionality of `dcargs` — generating argument parsers from type @@ -1567,4 +1624,5 @@ Note that most of these other libraries are generally aimed specifically at _dataclasses_ rather than general typed callables, but offer other features that you might find useful, such as registration for custom types (`pyrallis`), different approaches for serialization and config files (`tap`, `pyrallis`), -simultaneous parsing of multiple dataclasses (`simple-parsing`), etc. +simultaneous parsing of multiple dataclasses (`simple-parsing`), etc. Pull +requests are welcome if you're missing any of these. :slightly_smiling_face: diff --git a/_update_readme.py b/_update_readme.py index ea6d90256..4b0dcbac3 100644 --- a/_update_readme.py +++ b/_update_readme.py @@ -14,12 +14,15 @@ @dataclasses.dataclass(frozen=True) -class Constants: +class Markers: + signature_start: str = "" + signature_end: str = "" + docstring_start: str = "" - docstring_marker_end: str = "" + docstring_end: str = "" - examples_marker_start: str = "" - examples_marker_end: str = "" + examples_start: str = "" + examples_end: str = "" def replace_between_markers( @@ -76,9 +79,9 @@ def format_script_for_readme(path: pathlib.Path) -> str: env=dict(os.environ, **env_vars), ) if process_output.stderr != "": - output = dcargs._strings.strip_color_codes(process_output.stderr).strip() + output = dcargs._strings.strip_ansi_sequences(process_output.stderr).strip() elif process_output.stdout != "": - output = dcargs._strings.strip_color_codes(process_output.stdout).strip() + output = dcargs._strings.strip_ansi_sequences(process_output.stdout).strip() else: assert False example_output_lines.extend( @@ -124,7 +127,7 @@ def format_script_for_readme(path: pathlib.Path) -> str: ) -def get_examples_str(examples_dir: pathlib.Path, constants: Constants) -> str: +def get_examples_str(examples_dir: pathlib.Path, markers: Markers) -> str: script_paths = filter( lambda p: not p.name.startswith("_"), sorted(examples_dir.glob("*.py")) ) @@ -143,26 +146,39 @@ def get_examples_str(examples_dir: pathlib.Path, constants: Constants) -> str: def main( readme_path: pathlib.Path = REPO_ROOT / "README.md", examples_dir: pathlib.Path = REPO_ROOT / "examples", - constants: Constants = Constants(), + markers: Markers = Markers(), ) -> None: """Helper script for generating the examples list in the README.""" # Read. content = readme_path.read_text(encoding="utf8") + # Update signature. + signature_lines, _ = inspect.getsourcelines(dcargs.cli) + for i in range(len(signature_lines)): + if signature_lines[i].endswith(":\n"): + signature_lines = signature_lines[: i + 1] + break + content = replace_between_markers( + content, + markers.signature_start, + markers.signature_end, + "\n```python\n" + "".join(signature_lines).strip()[:-1] + "\n```\n", + ) + # Update examples. content = replace_between_markers( content, - constants.examples_marker_start, - constants.examples_marker_end, - get_examples_str(examples_dir, constants), + markers.examples_start, + markers.examples_end, + get_examples_str(examples_dir, markers), ) # Update docstring. content = replace_between_markers( content, - constants.docstring_start, - constants.docstring_marker_end, + markers.docstring_start, + markers.docstring_end, f"\n\n```\n{inspect.getdoc(dcargs.cli)}\n```\n\n", ) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index 262622cb8..fe6ffcba5 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -1,7 +1,27 @@ -from argparse_color_formatter import ColorHelpFormatter as ColorHelpFormatterBase +import argparse +import contextlib +from typing import Any +from . import _strings -class ColorHelpFormatter(ColorHelpFormatterBase): + +@contextlib.contextmanager +def argparse_ansi_monkey_patch(): + """Temporary monkey patch for making argparse ignore ANSI codes when wrapping usage + text.""" + + def monkeypatched_len(obj: Any) -> int: + if isinstance(obj, str): + return len(_strings.strip_ansi_sequences(obj)) + else: + return len(obj) + + argparse.len = monkeypatched_len # type: ignore + yield + del argparse.len # type: ignore + + +class ArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): def _format_args(self, action, default_metavar): """Override _format_args() to ignore nargs and always expect single string metavars.""" diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index d922acbc6..efc216d97 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -1,3 +1,6 @@ +"""Rules for taking high-level field definitions and lowering them into inputs for +argparse's `add_argument()`.""" + from __future__ import annotations import argparse @@ -6,7 +9,18 @@ import functools import itertools import shlex -from typing import Any, Dict, Mapping, Optional, Set, Type, TypeVar, Union +from typing import ( + Any, + Dict, + Mapping, + Optional, + Sequence, + Set, + Tuple, + Type, + TypeVar, + Union, +) import termcolor @@ -25,7 +39,7 @@ class ArgumentDefinition: """Structure containing everything needed to define an argument.""" prefix: str # Prefix for nesting. - field: _fields.Field + field: _fields.FieldDefinition type_from_typevar: Dict[TypeVar, Type] def add_argument( @@ -167,7 +181,9 @@ def _rule_recursive_instantiator_from_type( # available. return dataclasses.replace( lowered, - metavar=termcolor.colored("(not parsable)", color="red"), + metavar=termcolor.colored( + "{" + str(arg.field.default) + "}", color="red" + ), required=False, default=_fields.MISSING_PROP, ) @@ -188,11 +204,17 @@ def _rule_convert_defaults_to_strings( """Sets all default values to strings, as required as input to our instantiator functions. Special-cased for enums.""" - def as_str(x: Any) -> str: - if isinstance(x, enum.Enum): - return x.name + def as_str(x: Any) -> Tuple[str, ...]: + if isinstance(x, str): + return (x,) + elif isinstance(x, enum.Enum): + return (x.name,) + elif isinstance(x, Mapping): + return tuple(itertools.chain(*map(as_str, itertools.chain(*x.items())))) + elif isinstance(x, Sequence): + return tuple(itertools.chain(*map(as_str, x))) else: - return str(x) + return (str(x),) if ( lowered.default is None @@ -200,16 +222,9 @@ def as_str(x: Any) -> str: or lowered.action is not None ): return lowered - elif lowered.nargs is not None and lowered.nargs != "?": - if isinstance(lowered.default, Mapping): - return dataclasses.replace( - lowered, - default=tuple(map(as_str, itertools.chain(*lowered.default.items()))), - ) - else: - return dataclasses.replace( - lowered, default=tuple(map(as_str, lowered.default)) - ) + elif lowered.nargs is None: + (str_default,) = as_str(lowered.default) + return dataclasses.replace(lowered, default=str_default) else: return dataclasses.replace(lowered, default=as_str(lowered.default)) @@ -239,14 +254,15 @@ def _rule_generate_helptext( default = arg.field.default if not lowered.required: - default_label = "value" if lowered.is_fixed() else "default" # Include default value in helptext. We intentionally don't use the % template # because the types of all arguments are set to strings, which will cause the # default to be casted to a string and introduce extra quotation marks. - if lowered.action == "store_true": - default_text = f"(sets {arg.field.name}=True)" + if lowered.instantiator is None: + default_text = "(fixed)" + elif lowered.action == "store_true": + default_text = f"(sets: {arg.field.name}=True)" elif lowered.action == "store_false": - default_text = f"(sets {arg.field.name}=False)" + default_text = f"(sets: {arg.field.name}=False)" elif lowered.nargs is not None and hasattr(default, "__iter__"): # For tuple types, we might have default as (0, 1, 2, 3). # For list types, we might have default as [0, 1, 2, 3]. @@ -256,12 +272,12 @@ def _rule_generate_helptext( # the format that argparse expects when we set nargs. assert default is not None # Just for type checker. default_parts = map(shlex.quote, map(str, default)) - default_text = f"({default_label}: {' '.join(default_parts)})" + default_text = f"(default: {' '.join(default_parts)})" else: - default_text = f"({default_label}: {shlex.quote(str(default))})" + default_text = f"(default: {shlex.quote(str(default))})" help_parts.append(termcolor.colored(default_text, attrs=["dark"])) else: - help_parts.append(termcolor.colored("(required)", on_color="on_red")) + help_parts.append(termcolor.colored("(required)", color="red", attrs=["bold"])) return dataclasses.replace(lowered, help=" ".join(help_parts)) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index cca008f2b..7ed80cd05 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -1,3 +1,5 @@ +"""Core public API.""" + import argparse from typing import Callable, Optional, Sequence, TypeVar, Union @@ -109,11 +111,12 @@ def cli( ) # Parse using argparse! - parser = argparse.ArgumentParser( - prog=prog, formatter_class=_argparse_formatter.ColorHelpFormatter - ) - parser_definition.apply(parser) - value_from_prefixed_field_name = vars(parser.parse_args(args=args)) + with _argparse_formatter.argparse_ansi_monkey_patch(): + parser = argparse.ArgumentParser( + prog=prog, formatter_class=_argparse_formatter.ArgparseHelpFormatter + ) + parser_definition.apply(parser) + value_from_prefixed_field_name = vars(parser.parse_args(args=args)) try: # Attempt to call `f` using whatever was passed in. diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 405cf3109..7030436b5 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -1,4 +1,5 @@ -"""Helpers for parsing dataclass docstrings. Used for helptext generation.""" +"""Helpers for parsing docstrings. Used for helptext generation.""" + import dataclasses import functools import inspect diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 3f2d7e725..1c905587c 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -1,5 +1,6 @@ -"""Abstractions for pulling out 'field' abstractions, which specify inputs, from -general callables.""" +"""Abstractions for pulling out 'field' definitions, which specify inputs, types, and +defaults, from general callables.""" + import dataclasses import inspect import warnings @@ -12,7 +13,7 @@ @dataclasses.dataclass(frozen=True) -class Field: +class FieldDefinition: name: str typ: Type default: Any @@ -76,7 +77,7 @@ class NonpropagatingMissingType(_Singleton): def field_list_from_callable( f: Callable[..., T], default_instance: Union[T, PropagatingMissingType, NonpropagatingMissingType], -) -> List[Field]: +) -> List[FieldDefinition]: """Generate a list of generic 'field' objects corresponding to the inputs of some annotated callable. @@ -101,7 +102,7 @@ def field_list_from_callable( assert no_default_instance or isinstance(default_instance, dict) for name, typ in get_type_hints(cls).items(): field_list.append( - Field( + FieldDefinition( name=name, typ=typ, default=MISSING_PROP @@ -131,7 +132,7 @@ def field_list_from_callable( default = MISSING_PROP field_list.append( - Field( + FieldDefinition( name=name, typ=typ, default=default, @@ -148,7 +149,7 @@ def field_list_from_callable( ): default = _get_dataclass_field_default(dc_field, default_instance) field_list.append( - Field( + FieldDefinition( name=dc_field.name, typ=dc_field.type, default=default, @@ -198,7 +199,7 @@ def field_list_from_callable( ) field_list.append( - Field( + FieldDefinition( name=param.name, # Note that param.annotation does not resolve forward references. typ=hints[param.name], diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index d3c1b69b4..37627703f 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -57,8 +57,6 @@ import termcolor from typing_extensions import Annotated, Final, Literal, get_args, get_origin -from . import _strings - # Most standard fields: these are converted from strings from the CLI. _StandardInstantiator = Callable[[str], Any] # Sequence fields! This should be used whenever argparse's `nargs` field is set. @@ -75,7 +73,7 @@ @dataclasses.dataclass class InstantiatorMetadata: - nargs: Optional[Union[str, int]] + nargs: Union[None, int, Literal["+"]] # Note: unlike in vanilla argparse, our metavar is always a string. We handle # sequences, multiple arguments, etc, manually. metavar: str @@ -189,7 +187,33 @@ def instantiator(string: str) -> None: elif isinstance(typ, type) and issubclass(typ, enum.Enum): auto_choices = tuple(x.name for x in typ) - return lambda arg: _strings.instance_from_string(typ, arg), InstantiatorMetadata( + def instantiator_base_case(string: str) -> Any: + """Given a type and and a string from the command-line, reconstruct an object. Not + intended to deal with containers. + + This is intended to replace all calls to `type(string)`, which can cause unexpected + behavior. As an example, note that the following argparse code will always print + `True`, because `bool("True") == bool("False") == bool("0") == True`. + ``` + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--flag", type=bool) + + print(parser.parse_args().flag) + ``` + """ + assert len(get_args(typ)) == 0, f"Type {typ} cannot be instantiated." + if typ is bool: + return {"True": True, "False": False}[string] # type: ignore + elif isinstance(typ, type) and issubclass(typ, enum.Enum): + return typ[string] # type: ignore + elif typ is bytes: + return bytes(string, encoding="ascii") # type: ignore + else: + return typ(string) # type: ignore + + return instantiator_base_case, InstantiatorMetadata( nargs=None, metavar=_format_metavar(typ.__name__.upper()) if auto_choices is None @@ -198,6 +222,15 @@ def instantiator(string: str) -> None: ) +@overload +def _instantiator_from_type_inner( + typ: Type, + type_from_typevar: Dict[TypeVar, Type], + allow_sequences: Literal["fixed_length"], +) -> Tuple[Instantiator, InstantiatorMetadata]: + ... + + @overload def _instantiator_from_type_inner( typ: Type, @@ -219,17 +252,21 @@ def _instantiator_from_type_inner( def _instantiator_from_type_inner( typ: Type, type_from_typevar: Dict[TypeVar, Type], - allow_sequences: bool, + allow_sequences: Literal["fixed_length", True, False], ) -> Tuple[Instantiator, InstantiatorMetadata]: """Thin wrapper over instantiator_from_type, with some extra asserts for catching errors.""" out = instantiator_from_type(typ, type_from_typevar) - if ( - not allow_sequences - and out[1].nargs is not None - and get_origin(typ) is not Union - ): - raise UnsupportedTypeAnnotationError("Nested sequence types are not supported!") + if out[1].nargs is not None: + if allow_sequences is False: + # We currently only use allow_sequences=False for options in Literal types, + # which are evaluated using `type()`. It should not be possible to hit this + # condition from polling a runtime type. + assert False + if allow_sequences == "fixed_length" and not isinstance(out[1].nargs, int): + raise UnsupportedTypeAnnotationError( + f"Found an unsupported (variable-length) nested sequence of type {typ}." + ) return out @@ -249,7 +286,7 @@ def _instantiator_from_container_type( return instantiator_from_type(contained_type, type_from_typevar) for make, matched_origins in { - _instantiator_from_list_sequence_or_set: ( + _instantiator_from_sequence: ( collections.abc.Sequence, frozenset, list, @@ -280,55 +317,57 @@ def _instantiator_from_tuple( # Ellipsis: variable argument counts. When an ellipsis is used, tuples must # contain only one type. assert len(typeset_no_ellipsis) == 1 - (contained_type,) = typeset_no_ellipsis - - make, inner_meta = _instantiator_from_type_inner( - contained_type, - type_from_typevar, - allow_sequences=False, - ) - return lambda strings: tuple([make(x) for x in strings]), InstantiatorMetadata( - nargs="+", - metavar=f"{inner_meta.metavar} [{inner_meta.metavar} ...]", - choices=inner_meta.choices, - ) + return _instantiator_from_sequence(typ, type_from_typevar) else: instantiators = [] metas = [] + nargs = 0 for t in types: a, b = _instantiator_from_type_inner( t, type_from_typevar, - allow_sequences=False, + allow_sequences="fixed_length", ) instantiators.append(a) metas.append(b) + assert type(b.nargs) in (int, NoneType) + nargs += 1 if b.nargs is None else b.nargs # type: ignore - if len(set(m.choices for m in metas)) > 1: - raise UnsupportedTypeAnnotationError( - "Due to constraints in argparse, all choices in fixed-length tuples" - " must match. This restricts mixing enums & literals with other" - " types." - ) - - def tuple_instantiator(strings: List[str]) -> Tuple[Any, ...]: - if len(strings) != len(instantiators): + def fixed_length_tuple_instantiator(strings: List[str]) -> Any: + # Validate nargs. + if len(strings) != nargs: raise ValueError( - f"expected {len(instantiators)} arguments, but got {strings}" + f"input {strings} is length {len(strings)}, but expected {nargs}." ) - out = tuple(make(x) for make, x in zip(instantiators, strings)) - return out - return tuple_instantiator, InstantiatorMetadata( - nargs=len(types), - metavar=" ".join((_format_metavar(cast(str, m.metavar)) for m in metas)), - choices=metas[0].choices, + # Make tuple. + out = [] + i = 0 + for make, meta in zip(instantiators, metas): + inner_nargs = cast(Optional[int], meta.nargs) + if inner_nargs is None: + if meta.choices is not None and strings[i] not in meta.choices: + raise ValueError( + f" {strings[i]} does not match choices {meta.choices}" + ) + out.append(make(strings[i])) # type: ignore + i += 1 + else: + assert meta.choices is None + out.append(make(strings[i : i + inner_nargs])) # type: ignore + i += inner_nargs + return tuple(out) + + return fixed_length_tuple_instantiator, InstantiatorMetadata( + nargs=nargs, + metavar=" ".join(m.metavar for m in metas), + choices=None, ) def _join_union_metavars(metavars: Iterable[str]) -> str: - """Metavar generation helper for unions. + """Metavar generation helper for unions. Could be revisited. Examples: None, INT => NONE|INT @@ -338,8 +377,8 @@ def _join_union_metavars(metavars: Iterable[str]) -> str: STR, INT [INT ...] => STR|{INT [INT ...]} STR, INT INT => STR|{INT INT} - The curly brackets are unfortunately overloaded but parentheses, square brackets, - and angle brackets all seem to interfere with some argparse internals. + The curly brackets are unfortunately overloaded but alternatives all interfere with + argparse internals. """ metavars = tuple(metavars) merged_metavars = [metavars[0]] @@ -468,56 +507,98 @@ def _instantiator_from_dict( key_instantiator, key_metadata = _instantiator_from_type_inner( key_type, type_from_typevar, - allow_sequences=False, + allow_sequences="fixed_length", ) val_instantiator, val_metadata = _instantiator_from_type_inner( val_type, type_from_typevar, - allow_sequences=False, + allow_sequences="fixed_length", ) + key_nargs = cast(Optional[int], key_metadata.nargs) + key_nargs_int = 1 if key_nargs is None else key_nargs + val_nargs = cast(Optional[int], val_metadata.nargs) + val_nargs_int = 1 if key_nargs is None else key_nargs + assert type(key_nargs) in (int, NoneType) + assert type(val_nargs) in (int, NoneType) + pair_nargs = key_nargs_int + val_nargs_int + def dict_instantiator(strings: List[str]) -> Any: out = {} - if len(strings) % 2 != 0: + if len(strings) % pair_nargs != 0: raise ValueError("incomplete set of key value pairs!") - for i in range(len(strings) // 2): - k = strings[i * 2] - v = strings[i * 2 + 1] - if key_metadata.choices is not None and k not in key_metadata.choices: - raise ValueError( - f"invalid choice: {k} (choose from {key_metadata.choices}))" - ) - if val_metadata.choices is not None and v not in val_metadata.choices: - raise ValueError( - f"invalid choice: {v} (choose from {val_metadata.choices}))" - ) + + index = 0 + for _ in range(len(strings) // pair_nargs): + k: Union[str, List[str]] = strings[index : index + key_nargs_int] + index += key_nargs_int + v: Union[str, List[str]] = strings[index : index + val_nargs_int] + index += val_nargs_int + + if key_nargs is None: + (k,) = cast(List[str], k) + if key_metadata.choices is not None and k not in key_metadata.choices: + raise ValueError( + f"invalid choice: {k} (choose from {key_metadata.choices}))" + ) + if val_nargs is None: + (v,) = cast(List[str], v) + if val_metadata.choices is not None and v not in val_metadata.choices: + raise ValueError( + f"invalid choice: {v} (choose from {val_metadata.choices}))" + ) out[key_instantiator(k)] = val_instantiator(v) # type: ignore return out + pair_metavar = f"{key_metadata.metavar} {val_metadata.metavar}" return dict_instantiator, InstantiatorMetadata( nargs="+", - metavar=f"{key_metadata.metavar} {val_metadata.metavar}", + metavar=f"{pair_metavar} [{pair_metavar} ...]", choices=None, ) -def _instantiator_from_list_sequence_or_set( +def _instantiator_from_sequence( typ: Type, type_from_typevar: Dict[TypeVar, Type] ) -> Tuple[Instantiator, InstantiatorMetadata]: - (contained_type,) = get_args(typ) + """Instantiator for variable-length sequences: list, sets, Tuple[T, ...], etc.""" container_type = get_origin(typ) - assert container_type is not None if container_type is collections.abc.Sequence: container_type = list + if container_type is tuple: + (contained_type, ell) = get_args(typ) + assert ell == Ellipsis + else: + (contained_type,) = get_args(typ) + make, inner_meta = _instantiator_from_type_inner( contained_type, type_from_typevar, - allow_sequences=False, + allow_sequences="fixed_length", ) - return lambda strings: container_type( - [make(x) for x in strings] - ), InstantiatorMetadata( + + def sequence_instantiator(strings: List[str]) -> Any: + # Validate nargs. + assert type(inner_meta.nargs) in (int, NoneType) + if isinstance(inner_meta.nargs, int) and len(strings) % inner_meta.nargs != 0: + raise ValueError( + f"input {strings} is of length {len(strings)}, which is not divisible" + f" by {inner_meta.nargs}." + ) + + # Make tuple. + out = [] + step = inner_meta.nargs if isinstance(inner_meta.nargs, int) else 1 + for i in range(0, len(strings), step): + if inner_meta.nargs is None: + out.append(make(strings[i])) # type: ignore + else: + out.append(make(strings[i : i + inner_meta.nargs])) # type: ignore + assert container_type is not None + return container_type(out) + + return sequence_instantiator, InstantiatorMetadata( nargs="+", metavar=f"{inner_meta.metavar} [{inner_meta.metavar} ...]", choices=inner_meta.choices, diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 5aa7b5614..87dbdef52 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -1,4 +1,4 @@ -"""Abstractions for creating argparse parsers from a dataclass definition.""" +"""Interface for generating `argparse.ArgumentParser()` definitions from callables.""" from __future__ import annotations @@ -25,7 +25,15 @@ import typing_extensions from typing_extensions import get_args, get_origin, is_typeddict -from . import _arguments, _docstrings, _fields, _instantiators, _resolver, _strings +from . import ( + _argparse_formatter, + _arguments, + _docstrings, + _fields, + _instantiators, + _resolver, + _strings, +) T = TypeVar("T") @@ -44,7 +52,7 @@ def _is_possibly_nested_type(typ: Any) -> bool: """Heuristics for determining whether a type can be treated as a 'nested type', - where a single field has multiple corresponding argumentsi (eg for nested + where a single field has multiple corresponding arguments (eg for nested dataclasses or classes). Examples of when we return False: int, str, List[int], List[str], pathlib.Path, etc. @@ -298,7 +306,10 @@ def format_group_name(nested_field_name: str) -> str: metavar=metavar, ) for name, subparser_def in subparsers.parser_from_name.items(): - subparser = argparse_subparsers.add_parser(name) + subparser = argparse_subparsers.add_parser( + name, + formatter_class=_argparse_formatter.ArgparseHelpFormatter, + ) subparser_def.apply(subparser) def _get_leaf_subparsers( @@ -337,7 +348,7 @@ class SubparsersSpecification: @staticmethod def from_field( - field: _fields.Field, + field: _fields.FieldDefinition, type_from_typevar: Dict[TypeVar, Type], parent_classes: Set[Type], prefix: str, diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 568b93cde..2ef8b399c 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -1,4 +1,4 @@ -"""Utilities for resolving generic types and forward references.""" +"""Utilities for resolving types and forward references.""" import copy import dataclasses @@ -60,7 +60,7 @@ def resolved_fields(cls: Type) -> List[dataclasses.Field]: def is_namedtuple(cls: Type) -> bool: return ( hasattr(cls, "_fields") - # Remove in Python >=3.9. + # `_field_types` was removed in Python >=3.9. # and hasattr(cls, "_field_types") and hasattr(cls, "_field_defaults") ) diff --git a/dcargs/_serialization.py b/dcargs/_serialization.py index 73a431f5d..41c4a05e8 100644 --- a/dcargs/_serialization.py +++ b/dcargs/_serialization.py @@ -1,4 +1,4 @@ -"""Type-safe, human-readable serialization helpers.""" +"""Type-safe, human-readable serialization helpers for dataclasses.""" import dataclasses import enum diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 834c512ab..1fba28915 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -1,11 +1,9 @@ -"""Utilities for working with strings.""" -import enum +"""Utilities and constants for working with strings.""" + import functools import re import textwrap -from typing import Type, TypeVar - -from typing_extensions import get_args +from typing import Type from . import _resolver @@ -44,40 +42,11 @@ def subparser_name_from_type(cls: Type) -> str: ) -T = TypeVar("T") - - -def instance_from_string(typ: Type[T], arg: str) -> T: - """Given a type and and a string from the command-line, reconstruct an object. Not - intended to deal with containers. - - This is intended to replace all calls to `type(string)`, which can cause unexpected - behavior. As an example, note that the following argparse code will always print - `True`, because `bool("True") == bool("False") == bool("0") == True`. - ``` - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--flag", type=bool) - - print(parser.parse_args().flag) - ``` - """ - assert len(get_args(typ)) == 0, f"Type {typ} cannot be instantiated." - if typ is bool: - return {"True": True, "False": False}[arg] # type: ignore - elif isinstance(typ, type) and issubclass(typ, enum.Enum): - return typ[arg] # type: ignore - elif typ is bytes: - return bytes(arg, encoding="ascii") # type: ignore - else: - return typ(arg) # type: ignore - - @functools.lru_cache(maxsize=None) -def _strip_regex() -> re.Pattern: - return re.compile(r"\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))") +def _get_ansi_pattern() -> re.Pattern: + # https://stackoverflow.com/a/14693789 + return re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") -def strip_color_codes(x: str): - return _strip_regex().sub("", x) +def strip_ansi_sequences(x: str): + return _get_ansi_pattern().sub("", x) diff --git a/examples/05_hierarchical_configs.py b/examples/05_hierarchical_configs.py index 4aaf469d1..cc70f856b 100644 --- a/examples/05_hierarchical_configs.py +++ b/examples/05_hierarchical_configs.py @@ -49,7 +49,6 @@ class ExperimentConfig: def train( out_dir: pathlib.Path, - /, config: ExperimentConfig, restore_checkpoint: bool = False, checkpoint_interval: int = 1000, diff --git a/examples/06_base_configs.py b/examples/06_base_configs.py index 7994790b0..46e60e955 100644 --- a/examples/06_base_configs.py +++ b/examples/06_base_configs.py @@ -12,7 +12,9 @@ import sys from dataclasses import dataclass -from typing import Dict, Literal, Tuple, TypeVar, Union +from typing import Callable, Dict, Literal, Tuple, TypeVar, Union + +from torch import nn import dcargs @@ -50,6 +52,9 @@ class ExperimentConfig: # reproducible! seed: int + # Activation to use. Not specifiable via the commandline. + activation: Callable[[], nn.Module] + # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type @@ -65,6 +70,7 @@ class ExperimentConfig: # The dcargs.MISSING sentinel allows us to specify that the seed should have no # default, and needs to be populated from the CLI. seed=dcargs.MISSING, + activation=nn.ReLU, ), "big": ExperimentConfig( dataset="imagenet-50", @@ -74,6 +80,7 @@ class ExperimentConfig: units=256, train_steps=100_000, seed=dcargs.MISSING, + activation=nn.GELU, ), } diff --git a/examples/11_dictionaries.py b/examples/11_dictionaries.py index 529b785c7..4f51ca7ef 100644 --- a/examples/11_dictionaries.py +++ b/examples/11_dictionaries.py @@ -6,26 +6,27 @@ Usage: `python ./11_dictionaries.py --help` -`python ./11_dictionaries.py --standard-dict key1 True key2 False` """ -from typing import Dict, TypedDict +from typing import Dict, Tuple, TypedDict import dcargs class DictionarySchema(TypedDict): - field1: str # A string field. - field2: int # A numeric field. - field3: bool # A boolean field. + learning_rate: float + betas: Tuple[float, float] def main( - standard_dict: Dict[str, bool], + standard_dict: Dict[str, float] = { + "learning_rate": 3e-4, + "beta1": 0.9, + "beta2": 0.999, + }, typed_dict: DictionarySchema = { - "field1": "hey", - "field2": 3, - "field3": False, + "learning_rate": 3e-4, + "betas": (0.9, 0.999), }, ) -> None: assert isinstance(standard_dict, dict) diff --git a/setup.py b/setup.py index 383fe10cf..f5ab11847 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,6 @@ "typing_extensions>=4.3.0", "pyyaml", "termcolor", - "argparse-color-formatter", "backports.cached-property; python_version < '3.8.0'", ], extras_require={ diff --git a/tests/test_collections.py b/tests/test_collections.py index d7b4f3701..c60c47d46 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -312,3 +312,73 @@ def main(a: Union[Tuple[int, int], Tuple[int, int, int]]) -> Tuple[int, ...]: dcargs.cli(main, args=["--a"]) with pytest.raises(SystemExit): dcargs.cli(main, args=[]) + + +def test_choices_in_tuples_0(): + @dataclasses.dataclass + class A: + x: Tuple[bool, bool] + + assert dcargs.cli(A, args=["--x", "True", "False"]) == A((True, False)) + + +def test_choices_in_tuples_1(): + @dataclasses.dataclass + class A: + x: Tuple[bool, Literal["True", "False"]] + + assert dcargs.cli(A, args=["--x", "True", "False"]) == A((True, "False")) + + +def test_choices_in_tuples_2(): + @dataclasses.dataclass + class A: + x: Tuple[bool, Literal["True", "False", "None"]] + + assert dcargs.cli(A, args=["--x", "True", "False"]).x == (True, "False") + assert dcargs.cli(A, args=["--x", "False", "None"]).x == (False, "None") + with pytest.raises(SystemExit): + dcargs.cli(A, args=["--x", "None", "False"]) + + +def test_nested_tuple_types(): + @dataclasses.dataclass + class A: + x: Tuple[Tuple[int, int], Tuple[str, str]] + + assert dcargs.cli(A, args="--x 5 5 5 5".split(" ")).x == ((5, 5), ("5", "5")) + + +def test_variable_nested_tuple(): + def main(x: Tuple[Tuple[int, str], ...]) -> tuple: + return x + + assert dcargs.cli(main, args="--x 1 1 2 2".split(" ")) == ((1, "1"), (2, "2")) + with pytest.raises(SystemExit): + dcargs.cli(main, args="--x 1 1 2".split(" ")) + + +def test_super_nested(): + def main( + x: Optional[ + List[ + Tuple[ + Optional[int], + Literal[3, 4], + Union[Tuple[int, int], Tuple[str, str]], + ] + ] + ] = None + ) -> Any: + return x + + assert dcargs.cli(main, args=[]) is None + assert dcargs.cli(main, args="--x None".split(" ")) is None + assert dcargs.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] + assert dcargs.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] + assert dcargs.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ + (2, 3, ("x", "2")), + (2, 3, (1, 2)), + ] + with pytest.raises(SystemExit): + dcargs.cli(main, args=["--help"]) diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index 862c45a77..3b957bf93 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -1,7 +1,7 @@ import contextlib import io import pathlib -from typing import Any, Dict, Mapping, NamedTuple, cast +from typing import Any, Dict, Mapping, NamedTuple, Tuple, Union, cast import pytest from typing_extensions import Literal, TypedDict @@ -49,6 +49,16 @@ def main(params: Mapping[Literal[1, 3, 5, 7], bool] = {5: False, 1: True}) -> An dcargs.cli(main, args="--params 4 Tru 3 False".split(" ")) +def test_tuple_in_dict(): + def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> dict: + return x + + assert dcargs.cli(main, args="--x 1 1 2 2 3 3 4 4".split(" ")) == { + (1, 1): (2, 2), + (3, 3): (4, 4), + } + + def test_basic_typeddict(): class ManyTypesTypedDict(TypedDict): i: int @@ -102,7 +112,7 @@ class HelptextTypedDict(TypedDict): with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(HelptextTypedDict, default_instance={"z": 3}, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = dcargs._strings.strip_ansi_sequences(f.getvalue()) assert cast(str, HelptextTypedDict.__doc__) in helptext assert "--x INT" in helptext assert "--y INT" in helptext @@ -165,7 +175,7 @@ class HelptextNamedTupleDefault(NamedTuple): with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): dcargs.cli(HelptextNamedTupleDefault, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = dcargs._strings.strip_ansi_sequences(f.getvalue()) assert cast(str, HelptextNamedTupleDefault.__doc__) in helptext assert "--x INT Documentation 1 (required)\n" in helptext assert "--y INT Documentation 2 (required)\n" in helptext @@ -204,7 +214,7 @@ class HelptextNamedTuple(NamedTuple): ), args=["--help"], ) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = dcargs._strings.strip_ansi_sequences(f.getvalue()) assert cast(str, HelptextNamedTuple.__doc__) in helptext assert "Documentation 1 (required)\n" in helptext assert "Documentation 2 (required)\n" in helptext diff --git a/tests/test_errors.py b/tests/test_errors.py index b1edb6162..51c4b4d38 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Generic, List, Tuple, TypeVar +from typing import Generic, List, Tuple, TypeVar, Union import pytest from typing_extensions import Literal @@ -7,41 +7,29 @@ import dcargs -def test_choices_in_tuples(): - """Due to argparse limitations, all parameters of `choices` must match. In the - future, we might avoid this by implementing choice restrictions manually.""" - # OK +def test_ambiguous_collection_0(): @dataclasses.dataclass - class A: # type: ignore - x: Tuple[bool, bool] - - assert dcargs.cli(A, args=["--x", "True", "False"]) == A((True, False)) + class A: + x: Tuple[Tuple[int, ...], ...] - # OK - @dataclasses.dataclass - class A: # type: ignore - x: Tuple[bool, Literal["True", "False"]] + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli(A, args=["--x", "0", "1"]) - assert dcargs.cli(A, args=["--x", "True", "False"]) == A((True, "False")) - # Not OK: same argument, different choices. - @dataclasses.dataclass - class A: - x: Tuple[bool, Literal["True", "False", "None"]] +def test_ambiguous_collection_1(): + def main(x: Tuple[List[str], List[str]]) -> None: + pass with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(A, args=["--x", "True", "False"]) - + dcargs.cli(main, args=["--help"]) -def test_nested_sequence_types(): - """Unclear how to handle nested sequences, so we don't support them.""" - @dataclasses.dataclass - class A: - x: Tuple[Tuple[int, ...], ...] +def test_ambiguous_collection_2(): + def main(x: List[Union[Tuple[int, int], Tuple[int, int, int]]]) -> None: + pass with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(A, args=["--x", "0", "1"]) + dcargs.cli(main, args=["--help"]) def test_unsupported_literal(): diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 6e39fb61c..59287de62 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -3,7 +3,8 @@ import enum import io import pathlib -from typing import Generic, List, Optional, Tuple, TypeVar, Union, cast +from collections.abc import Callable +from typing import Dict, Generic, List, Optional, Tuple, TypeVar, Union, cast import pytest from typing_extensions import Literal @@ -12,6 +13,14 @@ import dcargs._strings +def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: + target = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(target): + dcargs.cli(f, args=args) + return dcargs._strings.strip_ansi_sequences(target.getvalue()) + + def test_helptext(): @dataclasses.dataclass class Helptext: @@ -25,11 +34,7 @@ class Helptext: z: int = 3 """Documentation 3""" - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(Helptext, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(Helptext) assert cast(str, Helptext.__doc__) in helptext assert "x INT" in helptext assert "y INT" in helptext @@ -63,11 +68,7 @@ def some_method(self) -> None: # noqa class ChildClass(UnrelatedParentClass, ActualParentClass): pass - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(ChildClass, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(ChildClass) assert "Documentation 1" in helptext assert "Documentation 2" in helptext @@ -96,20 +97,12 @@ def main_no_docstring(a: Inner) -> None: """main_no_docstring.""" pass - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(main_with_docstring, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(main_with_docstring) assert "Documented in function" in helptext and str(Inner.__doc__) not in helptext assert "Args:" not in helptext assert "Hello world!" in helptext - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(main_no_docstring, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(main_no_docstring) print(helptext) assert "Something" in helptext assert "Args:" not in helptext @@ -127,11 +120,7 @@ class HelptextWithVariousDefaults: x: pathlib.Path = pathlib.Path("/some/path/to/a/file") y: Color = Color.RED - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(HelptextWithVariousDefaults, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(HelptextWithVariousDefaults) assert "show this help message and exit\n --x PATH" in helptext assert "(default: /some/path/to/a/file)\n" in helptext assert "--y {RED,GREEN,BLUE}" in helptext @@ -153,11 +142,7 @@ class HelptextMultiline: """Documentation 3 Next line of documentation 3""" - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(HelptextMultiline, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(HelptextMultiline) assert "Documentation 1 (required)\n" in helptext assert "Documentation 2" in helptext assert "documentation 2" in helptext @@ -173,11 +158,7 @@ class HelptextGrouped: y: int z: int = 3 - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(HelptextGrouped, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(HelptextGrouped) assert "Documentation 1 (required)\n" in helptext assert "Description of both y and z. (required)\n" in helptext assert "Description of both y and z. (default: 3)\n" in helptext @@ -189,11 +170,7 @@ class Config: x: Optional[int] = None """An optional variable.""" - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(Config, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(Config) assert " --x {None}|INT" in helptext assert "An optional variable. (default: None)\n" in helptext @@ -211,11 +188,7 @@ class HelptextHardString: # Note that the percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(HelptextHardString, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(HelptextHardString) assert "--x" in helptext assert "Helptext. 2% milk." in helptext @@ -234,11 +207,7 @@ class Parent: class Child(Parent): pass - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(Child, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(Child) assert "--x STR" in helptext assert "Helptext." in helptext assert "(default: 'This docstring" in helptext @@ -263,11 +232,7 @@ class Child2(Parent2): """Helptext!""" # fmt: on - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(Child2, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(Child2) assert "--x STR" in helptext assert "Helptext! (default: 'This" in helptext @@ -277,11 +242,7 @@ def test_tuple_helptext(): class TupleHelptext: x: Tuple[int, str, float] - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(TupleHelptext, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(TupleHelptext) assert "--x INT STR FLOAT\n" in helptext @@ -290,11 +251,7 @@ def test_tuple_helptext_defaults(): class TupleHelptextDefaults: x: Tuple[int, str, str] = (5, "hello world", "hello") - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(TupleHelptextDefaults, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(TupleHelptextDefaults) assert "--x INT STR STR" in helptext assert "(default: 5 'hello world' hello)\n" in helptext @@ -306,11 +263,7 @@ def test_generic_helptext(): class GenericTupleHelptext(Generic[T]): x: T - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(GenericTupleHelptext[int], args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(GenericTupleHelptext[int]) assert "--x INT\n" in helptext @@ -321,11 +274,7 @@ def test_generic_tuple_helptext(): class GenericTupleHelptext(Generic[T]): x: Tuple[T, T, T] - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(GenericTupleHelptext[int], args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(GenericTupleHelptext[int]) assert "--x INT INT INT\n" in helptext @@ -336,11 +285,7 @@ def test_generic_list_helptext(): class GenericTupleHelptext(Generic[T]): x: List[T] - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(GenericTupleHelptext[int], args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(GenericTupleHelptext[int]) assert "--x INT [INT ...]\n" in helptext @@ -350,11 +295,7 @@ class LiteralHelptext: x: Literal[1, 2, 3] """A number.""" - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(LiteralHelptext, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(LiteralHelptext) assert "--x {1,2,3}" in helptext assert "A number. (required)\n" in helptext @@ -365,11 +306,7 @@ class OptionalLiteralHelptext: x: Optional[Literal[1, 2, 3]] = None """A number.""" - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(OptionalLiteralHelptext, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(OptionalLiteralHelptext) assert "--x {None,1,2,3}" in helptext assert "A number. (default: None)\n" in helptext @@ -398,27 +335,20 @@ class MultipleSubparsers: default_factory=Subcommand3 ) - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(MultipleSubparsers, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(MultipleSubparsers) assert "Field a description." in helptext assert "Field b description." not in helptext assert "Field c description." not in helptext - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli( - MultipleSubparsers, args=["subcommand1", "subcommand1", "--help"] - ) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext( + MultipleSubparsers, args=["subcommand1", "subcommand1", "--help"] + ) assert "Field a description." not in helptext assert "Field b description." not in helptext - assert "Field c description. (default: subcommand3)" in helptext + assert "Field c description." in helptext + assert "(default: subcommand3)" in helptext def test_optional_helptext(): @@ -434,12 +364,93 @@ class OptionalHelptext: z: Optional[int] = 3 """Documentation 3""" - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(OptionalHelptext, args=["--help"]) - helptext = dcargs._strings.strip_color_codes(f.getvalue()) + helptext = _get_helptext(OptionalHelptext) assert cast(str, OptionalHelptext.__doc__) in helptext assert "--x {None}|INT" in helptext assert "--y {None}|INT [{None}|INT ...]\n" in helptext assert "[--z {None}|INT]\n" in helptext + + +def test_metavar_0(): + def main(x: Union[Literal[0, 1, 2, 3], Tuple[int, int]]) -> None: + pass + + helptext = _get_helptext(main) + assert "--x {0,1,2,3}|{INT INT}" in helptext + + +def test_metavar_1(): + def main( + x: Union[ + Literal[0, 1, 2, 3], + Literal["hey,there", "hello"], + List[int], + ] + ) -> None: + pass + + # The comma formatting is unfortunate, but matches argparse's default behavior. + helptext = _get_helptext(main) + assert "--x {0,1,2,3,hey,there,hello}|{INT [INT ...]}" in helptext + + +def test_metavar_2(): + def main( + x: Tuple[ + Literal[0, 1, 2, 3], + Union[int, str], + ] + ) -> None: + pass + + helptext = _get_helptext(main) + assert "--x {0,1,2,3} INT|STR" in helptext + + +def test_metavar_3(): + def main( + x: Union[ + Literal[0, 1, 2, 3], + Union[Tuple[int, int], Tuple[str]], + ] + ) -> None: + pass + + helptext = _get_helptext(main) + assert "--x {0,1,2,3}|{INT INT}|STR" in helptext + + +def test_metavar_4(): + def main( + x: Union[ + Literal[0, 1, 2, 3], + Union[Tuple[int, int], Tuple[str, str, str]], + Literal[True], + ] + ) -> None: + pass + + helptext = _get_helptext(main) + assert "--x {0,1,2,3}|{INT INT}|{STR STR STR}|{True}" in helptext + + +def test_metavar_5(): + def main( + x: List[ + Union[Tuple[int, int], Tuple[str, str]], + ] = [(1, 1), (2, 2)] + ) -> None: + pass + + helptext = _get_helptext(main) + assert "[--x {INT INT}|{STR STR} [{INT INT}|{STR STR} ...]]" in helptext + + +def test_metavar_6(): + def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> dict: + return x + + helptext = _get_helptext(main) + assert ( + "--x {INT INT}|{STR STR} INT INT [{INT INT}|{STR STR} INT INT ...]" in helptext + ) diff --git a/tests/test_unsupported_but_should_work.py b/tests/test_unsupported_but_should_work.py index 2452cc228..8022e7409 100644 --- a/tests/test_unsupported_but_should_work.py +++ b/tests/test_unsupported_but_should_work.py @@ -123,7 +123,7 @@ class Helptext: with contextlib.redirect_stdout(f): dcargs.cli(Helptext, args=["--help"]) helptext = f.getvalue() - assert dcargs._strings.strip_color_codes(cast(str, Helptext.__doc__)) in helptext + assert dcargs._strings.strip_ansi_sequences(cast(str, Helptext.__doc__)) in helptext # Note that required detection seems to be broken here. assert "Documentation 1" in helptext From ae521cbbd5ed86155f4f6c1d3b6bf1aa4ed9fb50 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 18 Jul 2022 02:11:23 -0700 Subject: [PATCH 062/491] Support `total=False` for TypedDict --- README.md | 77 +++++++++++++++++++---------------- dcargs/_arguments.py | 2 + dcargs/_calling.py | 9 ++-- dcargs/_fields.py | 29 ++++++++++--- dcargs/_parsers.py | 8 ++-- examples/11_dictionaries.py | 16 ++++---- tests/test_dict_namedtuple.py | 59 +++++++++++++++++++++------ 7 files changed, 131 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index 7d53c0f2a..1220456b8 100644 --- a/README.md +++ b/README.md @@ -177,8 +177,8 @@ Returns:
-In the simplest case, `dcargs.cli()` can be used to run a function with -arguments populated from the CLI. +In the simplest case, `dcargs.cli()` can be used to run a function with arguments +populated from the CLI. **Code ([link](examples/01_functions.py)):** @@ -238,8 +238,8 @@ hello 10
-Common pattern: use `dcargs.cli()` to instantiate a dataclass. The outputted -instance can be used as a typed alternative for an argparse namespace. +Common pattern: use `dcargs.cli()` to instantiate a dataclass. The outputted instance +can be used as a typed alternative for an argparse namespace. **Code ([link](examples/02_dataclasses.py)):** @@ -299,8 +299,8 @@ Args(field1='hello', field2=5)
-We can generate argument parsers from more advanced type annotations, like enums -and tuple types. +We can generate argument parsers from more advanced type annotations, like enums and +tuple types. **Code ([link](examples/03_enums_and_containers.py)):** @@ -389,8 +389,8 @@ TrainConfig(dataset_sources=(PosixPath('data'),), image_dimensions=(32
-Booleans can either be expected to be explicitly passed in, or, if given a -default value, automatically converted to flags. +Booleans can either be expected to be explicitly passed in, or, if given a default +value, automatically converted to flags. **Code ([link](examples/04_flags.py)):** @@ -629,9 +629,9 @@ usage: 05_hierarchical_configs.py [-h] --out-dir PATH
-We can integrate `dcargs.cli()` into common configuration patterns: here, we -select one of multiple possible base configurations, and then use the CLI to -either override (existing) or fill in (missing) values. +We can integrate `dcargs.cli()` into common configuration patterns: here, we select +one of multiple possible base configurations, and then use the CLI to either override +(existing) or fill in (missing) values. **Code ([link](examples/06_base_configs.py)):** @@ -775,7 +775,7 @@ arguments: (required) --activation {<class 'torch.nn.modules.activation.ReLU'>} Activation to use. Not specifiable via the - commandline. (not parsable) + commandline. (fixed) optimizer arguments: Optimizer parameters. @@ -814,7 +814,7 @@ arguments: (required) --activation {<class 'torch.nn.modules.activation.GELU'>} Activation to use. Not specifiable via the - commandline. (not parsable) + commandline. (fixed) optimizer arguments: Optimizer parameters. @@ -839,9 +839,8 @@ ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learni
-`typing.Literal[]` can be used to restrict inputs to a fixed set of literal -choices; `typing.Union[]` can be used to restrict inputs to a fixed set of -types. +`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices; +`typing.Union[]` can be used to restrict inputs to a fixed set of types. **Code ([link](examples/07_literals_and_unions.py)):** @@ -932,8 +931,7 @@ arguments:
-Positional-only arguments in functions are converted to positional CLI -arguments. +Positional-only arguments in functions are converted to positional CLI arguments. **Code ([link](examples/08_positional_args.py)):** @@ -1053,8 +1051,7 @@ background_rgb=(1.0, 0.0, 0.0)
-Unions over nested types (classes or dataclasses) are populated using -subparsers. +Unions over nested types (classes or dataclasses) are populated using subparsers. **Code ([link](examples/09_subparsers.py)):** @@ -1275,11 +1272,8 @@ AdamOptimizer(learning_rate=0.0003, betas=(0.9, 0.999))
-Dictionary inputs can be specified using either a standard `Dict[K, V]` -annotation, or a `TypedDict` type. - -Note that setting `total=False` for `TypedDict` is currently not (but reasonably -could be) supported. +Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation, +or a `TypedDict` type. **Code ([link](examples/11_dictionaries.py)):** @@ -1289,21 +1283,22 @@ from typing import Dict, Tuple, TypedDict import dcargs -class DictionarySchema(TypedDict): +class DictionarySchema( + TypedDict, + # Setting `total=False` specifies that not all keys need to exist. + total=False, +): learning_rate: float betas: Tuple[float, float] def main( + typed_dict: DictionarySchema, standard_dict: Dict[str, float] = { "learning_rate": 3e-4, "beta1": 0.9, "beta2": 0.999, }, - typed_dict: DictionarySchema = { - "learning_rate": 3e-4, - "betas": (0.9, 0.999), - }, ) -> None: assert isinstance(standard_dict, dict) assert isinstance(typed_dict, dict) @@ -1321,9 +1316,9 @@ if __name__ == "__main__": 
 $ python ./11_dictionaries.py --help
-usage: 11_dictionaries.py [-h] [--standard-dict STR FLOAT [STR FLOAT ...]]
-                          [--typed-dict.learning-rate FLOAT]
+usage: 11_dictionaries.py [-h] [--typed-dict.learning-rate FLOAT]
                           [--typed-dict.betas FLOAT FLOAT]
+                          [--standard-dict STR FLOAT [STR FLOAT ...]]
 
 arguments:
   -h, --help            show this help message and exit
@@ -1334,9 +1329,23 @@ arguments:
 typed_dict arguments:
 
   --typed-dict.learning-rate FLOAT
-                        (default: 0.0003)
+                        Setting `total=False` specifies that not all keys need
+                        to exist. (unset by default)
   --typed-dict.betas FLOAT FLOAT
-                        (default: 0.9 0.999)
+                        Setting `total=False` specifies that not all keys need
+                        to exist. (unset by default)
+
+ +
+$ python ./11_dictionaries.py --typed-dict.learning-rate 3e-4
+Standard dict: {'learning_rate': 0.0003, 'beta1': 0.9, 'beta2': 0.999}
+Typed dict: {'learning_rate': 0.0003}
+
+ +
+$ python ./11_dictionaries.py --typed-dict.betas 0.9 0.999
+Standard dict: {'learning_rate': 0.0003, 'beta1': 0.9, 'beta2': 0.999}
+Typed dict: {'betas': (0.9, 0.999)}
diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index efc216d97..36663042b 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -263,6 +263,8 @@ def _rule_generate_helptext( default_text = f"(sets: {arg.field.name}=True)" elif lowered.action == "store_false": default_text = f"(sets: {arg.field.name}=False)" + elif arg.field.default is _fields.EXCLUDE_FROM_CALL: + default_text = "(unset by default)" elif lowered.nargs is not None and hasattr(default, "__iter__"): # For tuple types, we might have default as (0, 1, 2, 3). # For list types, we might have default as [0, 1, 2, 3]. diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 580bd51e3..3d7aaa966 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -155,9 +155,10 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: ) consumed_keywords |= consumed_keywords_child - if field.positional: - args.append(value) - else: - kwargs[field.name] = value + if value is not _fields.EXCLUDE_FROM_CALL: + if field.positional: + args.append(value) + else: + kwargs[field.name] = value return f(*args, **kwargs), consumed_keywords # type: ignore diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 1c905587c..4cdc890f5 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -9,7 +9,7 @@ import docstring_parser from typing_extensions import get_type_hints, is_typeddict -from . import _docstrings, _resolver +from . import _docstrings, _instantiators, _parsers, _resolver @dataclasses.dataclass(frozen=True) @@ -44,11 +44,16 @@ class NonpropagatingMissingType(_Singleton): pass +class ExcludeFromKwargsType(_Singleton): + pass + + # We have two types of missing sentinels: a propagating missing value, which when set as # a default will set all child values of nested structures as missing as well, and a # nonpropagating missing sentinel, which does not override child defaults. MISSING_PROP = PropagatingMissingType() MISSING_NONPROP = NonpropagatingMissingType() +EXCLUDE_FROM_CALL = ExcludeFromKwargsType() # Note that our "public" missing API will always be the propagating missing sentinel. MISSING_PUBLIC: Any = MISSING_PROP @@ -98,16 +103,28 @@ def field_list_from_callable( if cls is not None and is_typeddict(cls): # Handle typed dictionaries. field_list = [] - no_default_instance = default_instance in MISSING_SINGLETONS - assert no_default_instance or isinstance(default_instance, dict) + valid_default_instance = ( + default_instance not in MISSING_SINGLETONS + and default_instance is not EXCLUDE_FROM_CALL + ) + assert not valid_default_instance or isinstance(default_instance, dict) for name, typ in get_type_hints(cls).items(): + if valid_default_instance: + default = default_instance.get(name, MISSING_PROP) # type: ignore + elif getattr(cls, "__total__") is False: + default = EXCLUDE_FROM_CALL + if _parsers.is_possibly_nested_type(typ): + raise _instantiators.UnsupportedTypeAnnotationError( + "`total=False` not supported for nested structures." + ) + else: + default = MISSING_PROP + field_list.append( FieldDefinition( name=name, typ=typ, - default=MISSING_PROP - if no_default_instance - else default_instance.get(name, MISSING_PROP), # type: ignore + default=default, helptext=_docstrings.get_field_docstring(cls, name), positional=False, ) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 87dbdef52..aff0fce1f 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -50,7 +50,7 @@ ) -def _is_possibly_nested_type(typ: Any) -> bool: +def is_possibly_nested_type(typ: Any) -> bool: """Heuristics for determining whether a type can be treated as a 'nested type', where a single field has multiple corresponding arguments (eg for nested dataclasses or classes). @@ -189,10 +189,10 @@ def from_callable( continue else: field = dataclasses.replace(field, typ=type(field.default)) - assert _is_possibly_nested_type(field.typ) + assert is_possibly_nested_type(field.typ) # (2) Handle nested callables. - if _is_possibly_nested_type(field.typ): + if is_possibly_nested_type(field.typ): nested_parser = ParserSpecification.from_callable( field.typ, description=None, @@ -361,7 +361,7 @@ def from_field( # We don't use sets here to retain order of subcommands. options = [type_from_typevar.get(typ, typ) for typ in get_args(field.typ)] options_no_none = [o for o in options if o != type(None)] # noqa - if not all(map(_is_possibly_nested_type, options_no_none)): + if not all(map(is_possibly_nested_type, options_no_none)): return None parser_from_name: Dict[str, ParserSpecification] = {} diff --git a/examples/11_dictionaries.py b/examples/11_dictionaries.py index 4f51ca7ef..ef788b71f 100644 --- a/examples/11_dictionaries.py +++ b/examples/11_dictionaries.py @@ -1,11 +1,10 @@ """Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation, or a `TypedDict` type. -Note that setting `total=False` for `TypedDict` is currently not (but reasonably could be) -supported. - Usage: `python ./11_dictionaries.py --help` +`python ./11_dictionaries.py --typed-dict.learning-rate 3e-4` +`python ./11_dictionaries.py --typed-dict.betas 0.9 0.999` """ from typing import Dict, Tuple, TypedDict @@ -13,21 +12,22 @@ import dcargs -class DictionarySchema(TypedDict): +class DictionarySchema( + TypedDict, + # Setting `total=False` specifies that not all keys need to exist. + total=False, +): learning_rate: float betas: Tuple[float, float] def main( + typed_dict: DictionarySchema, standard_dict: Dict[str, float] = { "learning_rate": 3e-4, "beta1": 0.9, "beta2": 0.999, }, - typed_dict: DictionarySchema = { - "learning_rate": 3e-4, - "betas": (0.9, 0.999), - }, ) -> None: assert isinstance(standard_dict, dict) assert isinstance(typed_dict, dict) diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index 3b957bf93..cfe791b06 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -63,22 +63,55 @@ def test_basic_typeddict(): class ManyTypesTypedDict(TypedDict): i: int s: str - f: float - p: pathlib.Path assert dcargs.cli( ManyTypesTypedDict, - args=[ - "--i", - "5", - "--s", - "5", - "--f", - "5", - "--p", - "~", - ], - ) == dict(i=5, s="5", f=5.0, p=pathlib.Path("~")) + args="--i 5 --s 5".split(" "), + ) == dict(i=5, s="5") + + with pytest.raises(SystemExit): + dcargs.cli(ManyTypesTypedDict, args="--i 5".split(" ")) + + with pytest.raises(SystemExit): + dcargs.cli(ManyTypesTypedDict, args="--s 5".split(" ")) + + +def test_total_false_typeddict(): + class ManyTypesTypedDict(TypedDict, total=False): + i: int + s: str + + assert dcargs.cli( + ManyTypesTypedDict, + args="--i 5 --s 5".split(" "), + ) == dict(i=5, s="5") + + assert dcargs.cli(ManyTypesTypedDict, args="--i 5".split(" ")) == dict(i=5) + assert dcargs.cli(ManyTypesTypedDict, args="--s 5".split(" ")) == dict(s="5") + + +def test_total_false_nested_typeddict(): + class ChildTypedDict(TypedDict, total=False): + i: int + s: str + + class ParentTypedDict(TypedDict, total=False): + child: ChildTypedDict + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli( + ParentTypedDict, + args="--child.i 5 --child.s 5".split(" "), + ) + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + assert ( + dcargs.cli( + ParentTypedDict, + args=[""], + ) + == {} + ) def test_nested_typeddict(): From 362afe54c9babe49775946d81231be56e8338524 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 18 Jul 2022 13:39:10 -0700 Subject: [PATCH 063/491] Minor --- README.md | 67 +++++++++++++++++++++---------------- dcargs/_arguments.py | 1 - examples/06_base_configs.py | 10 +++--- 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 1220456b8..8d63b6509 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,16 @@ the simplest case, this can be used as a drop-in replacement for `argparse`: import argparse parser = argparse.ArgumentParser() -parser.add_argument("--a", type=int, required=True) -parser.add_argument("--b", type=int, default=1) +parser.add_argument( + "--a", + type=int, + required=True, +) +parser.add_argument( + "--b", + type=int, + default=3, +) args = parser.parse_args() print(args.a + args.b) @@ -57,16 +65,16 @@ dcargs.cli(main)
-The broader goal is a replacement for tools like `argparse`, `hydra`, and +The broader goal is also a replacement for tools like `hydra`, `gin-config`, and `ml_collections` that's: - **Low effort.** Standard Python type annotations, docstrings, and default - values are parsed to automatically generate argument parsers with informative - helptext. + values are parsed to automatically generate command-line interfaces with + informative helptext. - **Expressive.** `dcargs.cli()` understands functions, classes, dataclasses, and _nested_ classes and dataclasses, as well as frequently used annotations - like unions, literals, collections, and generics, which can be composed into + like unions, literals, and collections, which can be composed into hierarchical configuration objects built on standard Python features. - **Typed.** Unlike dynamic configuration namespaces produced by libraries like @@ -177,8 +185,8 @@ Returns:
-In the simplest case, `dcargs.cli()` can be used to run a function with arguments -populated from the CLI. +In the simplest case, `dcargs.cli()` can be used to run a function with +arguments populated from the CLI. **Code ([link](examples/01_functions.py)):** @@ -238,8 +246,8 @@ hello 10
-Common pattern: use `dcargs.cli()` to instantiate a dataclass. The outputted instance -can be used as a typed alternative for an argparse namespace. +Common pattern: use `dcargs.cli()` to instantiate a dataclass. The outputted +instance can be used as a typed alternative for an argparse namespace. **Code ([link](examples/02_dataclasses.py)):** @@ -299,8 +307,8 @@ Args(field1='hello', field2=5)
-We can generate argument parsers from more advanced type annotations, like enums and -tuple types. +We can generate argument parsers from more advanced type annotations, like enums +and tuple types. **Code ([link](examples/03_enums_and_containers.py)):** @@ -389,8 +397,8 @@ TrainConfig(dataset_sources=(PosixPath('data'),), image_dimensions=(32
-Booleans can either be expected to be explicitly passed in, or, if given a default -value, automatically converted to flags. +Booleans can either be expected to be explicitly passed in, or, if given a +default value, automatically converted to flags. **Code ([link](examples/04_flags.py)):** @@ -629,9 +637,9 @@ usage: 05_hierarchical_configs.py [-h] --out-dir PATH
-We can integrate `dcargs.cli()` into common configuration patterns: here, we select -one of multiple possible base configurations, and then use the CLI to either override -(existing) or fill in (missing) values. +We can integrate `dcargs.cli()` into common configuration patterns: here, we +select one of multiple possible base configurations, and then use the CLI to +either override (existing) or fill in (missing) values. **Code ([link](examples/06_base_configs.py)):** @@ -746,14 +754,14 @@ if __name__ == "__main__": **Example usage:** 
-$ python ./06_base_configs_argv.py
+$ python ./06_base_configs.py
 usage:
   examples/06_base_configs.py small --help
   examples/06_base_configs.py big --help
 
-$ python ./06_base_configs_argv.py small --help
+$ python ./06_base_configs.py small --help
 usage: examples/06_base_configs.py small [-h] [--dataset {mnist,imagenet-50}]
                                          [--optimizer.learning-rate FLOAT]
                                          [--num-layers INT] [--units INT]
@@ -785,12 +793,12 @@ optimizer arguments:
 
-$ python ./06_base_configs_argv.py small --seed 94720
+$ python ./06_base_configs.py small --seed 94720
 ExperimentConfig(dataset='mnist', optimizer=SgdOptimizer(learning_rate=0.0003), num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720, activation=<class 'torch.nn.modules.activation.ReLU'>)
 
-$ python ./06_base_configs_argv.py big --help
+$ python ./06_base_configs.py big --help
 usage: examples/06_base_configs.py big [-h] [--dataset {mnist,imagenet-50}]
                                        [--optimizer.learning-rate FLOAT]
                                        [--optimizer.betas FLOAT FLOAT]
@@ -826,7 +834,7 @@ optimizer arguments:
 
-$ python ./06_base_configs_argv.py big --seed 94720
+$ python ./06_base_configs.py big --seed 94720
 ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learning_rate=0.001, betas=(0.9, 0.999)), num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720, activation=<class 'torch.nn.modules.activation.GELU'>)
@@ -839,8 +847,9 @@ ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learni
-`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices; -`typing.Union[]` can be used to restrict inputs to a fixed set of types. +`typing.Literal[]` can be used to restrict inputs to a fixed set of literal +choices; `typing.Union[]` can be used to restrict inputs to a fixed set of +types. **Code ([link](examples/07_literals_and_unions.py)):** @@ -931,7 +940,8 @@ arguments:
-Positional-only arguments in functions are converted to positional CLI arguments. +Positional-only arguments in functions are converted to positional CLI +arguments. **Code ([link](examples/08_positional_args.py)):** @@ -1051,7 +1061,8 @@ background_rgb=(1.0, 0.0, 0.0)
-Unions over nested types (classes or dataclasses) are populated using subparsers. +Unions over nested types (classes or dataclasses) are populated using +subparsers. **Code ([link](examples/09_subparsers.py)):** @@ -1272,8 +1283,8 @@ AdamOptimizer(learning_rate=0.0003, betas=(0.9, 0.999))
-Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation, -or a `TypedDict` type. +Dictionary inputs can be specified using either a standard `Dict[K, V]` +annotation, or a `TypedDict` type. **Code ([link](examples/11_dictionaries.py)):** diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 36663042b..571c87d95 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -73,7 +73,6 @@ def lowered(self) -> LoweredArgumentDefinition: _rule_generate_helptext, _rule_set_name_or_flag, _rule_positional_special_handling, - # _rule_bold_metavar, ) return functools.reduce( lambda lowered, rule: rule(self, lowered), diff --git a/examples/06_base_configs.py b/examples/06_base_configs.py index 46e60e955..702c88798 100644 --- a/examples/06_base_configs.py +++ b/examples/06_base_configs.py @@ -3,11 +3,11 @@ (existing) or fill in (missing) values. Usage: -`python ./06_base_configs_argv.py` -`python ./06_base_configs_argv.py small --help` -`python ./06_base_configs_argv.py small --seed 94720` -`python ./06_base_configs_argv.py big --help` -`python ./06_base_configs_argv.py big --seed 94720` +`python ./06_base_configs.py` +`python ./06_base_configs.py small --help` +`python ./06_base_configs.py small --seed 94720` +`python ./06_base_configs.py big --help` +`python ./06_base_configs.py big --seed 94720` """ import sys From 47875b0d6d342fb81bbc81a04ab57fb656a7658c Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 18 Jul 2022 15:22:49 -0700 Subject: [PATCH 064/491] Support directly passing builtins into dcargs.cli --- dcargs/_calling.py | 2 +- dcargs/_fields.py | 133 +++++++++++++++++++++++++++++-------------- setup.py | 2 +- tests/test_dcargs.py | 21 ++++++- tests/test_errors.py | 12 +++- 5 files changed, 121 insertions(+), 49 deletions(-) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 3d7aaa966..6c57551cb 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -161,4 +161,4 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: else: kwargs[field.name] = value - return f(*args, **kwargs), consumed_keywords # type: ignore + return _resolver.unwrap_origin(f)(*args, **kwargs), consumed_keywords # type: ignore diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 4cdc890f5..a6b852864 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -4,7 +4,7 @@ import dataclasses import inspect import warnings -from typing import Any, Callable, List, Optional, Type, TypeVar, Union +from typing import Any, Callable, List, Optional, Type, TypeVar, Union, cast import docstring_parser from typing_extensions import get_type_hints, is_typeddict @@ -89,7 +89,7 @@ def field_list_from_callable( `f` can be from a dataclass type, regular class type, or function.""" # Unwrap generics. - f, _unused_type_from_typevar = _resolver.resolve_generic_types(f) + f, type_from_typevar = _resolver.resolve_generic_types(f) # If `f` is a type: # 1. Set cls to the type. @@ -98,7 +98,6 @@ def field_list_from_callable( if isinstance(f, type): cls = f f = cls.__init__ # type: ignore - ignore_self = True if cls is not None and is_typeddict(cls): # Handle typed dictionaries. @@ -181,51 +180,97 @@ def field_list_from_callable( default_instance in MISSING_SINGLETONS ), "`default_instance` is only supported for dataclass and TypedDict types." - # Get type annotations, docstrings. - hints = get_type_hints(f) - docstring = inspect.getdoc(f) - docstring_from_arg_name = {} - if docstring is not None: - for param_doc in docstring_parser.parse(docstring).params: - docstring_from_arg_name[param_doc.arg_name] = param_doc.description - del docstring - # Generate field list from function signature. field_list = [] - ignore_self = cls is not None - params = inspect.signature(f).parameters.values() - for param in params: - # For `__init__`, skip self parameter. - if ignore_self: - ignore_self = False - continue - - # Get default value. - default = param.default - - # Get helptext from docstring. - helptext = docstring_from_arg_name.get(param.name) - if helptext is None and cls is not None: - helptext = _docstrings.get_field_docstring(cls, param.name) - - if param.name not in hints: - raise TypeError( - f"Expected fully type-annotated callable, but {f} with arguments" - f" {tuple(map(lambda p: p.name, params))} has no annotation for" - f" '{param.name}'." - ) + params = list(inspect.signature(f).parameters.values()) + if cls is not None: + # Ignore self parameter. + params = params[1:] + + try: + return _field_list_from_params(f, cls, params) + except TypeError as e: + # Try to support passing things like int, str, Dict[K,V], torch.device + # directly into dcargs.cli(). These aren't "type-annotated callables" but + # this a nice-to-have. + param_count = 0 + has_kw_only = False + has_var_positional = False + for param in params: + if ( + param.kind + in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + ) + and param.default is inspect.Parameter.empty + ): + param_count += 1 + elif param.kind is inspect.Parameter.KEYWORD_ONLY: + has_kw_only = True + elif param.kind is inspect.Parameter.VAR_POSITIONAL: + has_var_positional = True + + if not has_kw_only and ( + param_count == 1 or (param_count == 0 and has_var_positional) + ): + # Things look ok! + if cls is not None: + f = cls + return [ + FieldDefinition( + name=_resolver.unwrap_origin(f).__name__, + typ=cast(Type, f), + default=MISSING_NONPROP, + helptext=None, + positional=True, + ) + ] + else: + raise e - field_list.append( - FieldDefinition( - name=param.name, - # Note that param.annotation does not resolve forward references. - typ=hints[param.name], - default=default, - helptext=helptext, - positional=param.kind is inspect.Parameter.POSITIONAL_ONLY, - ) + +def _field_list_from_params( + f: Callable, cls: Optional[Type], params: List[inspect.Parameter] +) -> List[FieldDefinition]: + # Get type annotations, docstrings. + docstring = inspect.getdoc(f) + docstring_from_arg_name = {} + if docstring is not None: + for param_doc in docstring_parser.parse(docstring).params: + docstring_from_arg_name[param_doc.arg_name] = param_doc.description + del docstring + hints = get_type_hints(f) + + field_list = [] + for param in params: + # Get default value. + default = param.default + + # Get helptext from docstring. + helptext = docstring_from_arg_name.get(param.name) + if helptext is None and cls is not None: + helptext = _docstrings.get_field_docstring(cls, param.name) + + if param.name not in hints: + raise TypeError( + f"Expected fully type-annotated callable, but {f} with arguments" + f" {tuple(map(lambda p: p.name, params))} has no annotation for" + f" '{param.name}'." ) - return field_list + + field_list.append( + FieldDefinition( + name=param.name, + # Note that param.annotation does not resolve forward references. + typ=hints[param.name], + default=default, + helptext=helptext, + positional=param.kind is inspect.Parameter.POSITIONAL_ONLY, + ) + ) + + return field_list def _ensure_dataclass_instance_used_as_default_is_frozen( diff --git a/setup.py b/setup.py index f5ab11847..07998abcb 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.1.7", + version="0.1.8", description="Strongly typed, zero-effort CLIs", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 86b9c3221..adb2aecc0 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -2,7 +2,7 @@ import dataclasses import enum import pathlib -from typing import Any, AnyStr, Callable, ClassVar, List, Optional, TypeVar, Union +from typing import Any, AnyStr, Callable, ClassVar, Dict, List, Optional, TypeVar, Union import pytest import torch @@ -453,3 +453,22 @@ def main(device: torch.device) -> torch.device: return device assert dcargs.cli(main, args=["--device", "cpu"]) == torch.device("cpu") + + +def test_torch_device_2(): + assert dcargs.cli(torch.device, args=["cpu"]) == torch.device("cpu") + + +def test_just_int(): + assert dcargs.cli(int, args=["123"]) == 123 + + +def test_just_dict(): + assert dcargs.cli(Dict[str, str], args="key value key2 value2".split(" ")) == { + "key": "value", + "key2": "value2", + } + + +def test_just_list(): + assert dcargs.cli(List[int], args="1 2 3 4".split(" ")) == [1, 2, 3, 4] diff --git a/tests/test_errors.py b/tests/test_errors.py index 51c4b4d38..240370292 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -123,8 +123,16 @@ class ChildClass(UnrelatedParentClass, ActualParentClass[int]): dcargs.cli(ChildClass, args=["--x", "1", "--y", "2", "--z", "3"]) -def test_missing_annotation(): - def main(a) -> None: +def test_missing_annotation_1(): + def main(a, b) -> None: + pass + + with pytest.raises(TypeError): + dcargs.cli(main, args=["--help"]) + + +def test_missing_annotation_2(): + def main(*, a) -> None: pass with pytest.raises(TypeError): From fffe666e6530f376341cbf9b0349362f612a5c7d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 19 Jul 2022 05:50:20 -0700 Subject: [PATCH 065/491] Refactor docs, move serialization utils to .extras --- .github/workflows/docs.yml | 31 + README.md | 1604 +--------------------- _update_readme.py | 197 --- dcargs/__init__.py | 13 +- dcargs/_cli.py | 71 +- dcargs/_deprecated.py | 2 + dcargs/_fields.py | 4 +- dcargs/extras/__init__.py | 3 + dcargs/{ => extras}/_serialization.py | 15 +- docs/.gitignore | 1 + docs/Makefile | 20 + docs/requirements.txt | 9 + docs/source/alternatives.md | 24 + docs/source/conf.py | 382 ++++++ docs/source/example_01.rst | 46 + docs/source/example_02.rst | 45 + docs/source/example_03.rst | 64 + docs/source/example_04.rst | 54 + docs/source/example_05.rst | 89 ++ docs/source/example_06.rst | 136 ++ docs/source/example_07.rst | 63 + docs/source/example_08.rst | 77 ++ docs/source/example_09.rst | 60 + docs/source/example_10.rst | 87 ++ docs/source/example_11.rst | 58 + docs/source/example_12.rst | 43 + docs/source/example_13.rst | 49 + docs/source/example_14.rst | 57 + docs/source/index.rst | 115 ++ docs/source/serialization.rst | 22 + docs/update_example_docs.py | 119 ++ examples/04_flags.py | 2 +- examples/05_hierarchical_configs.py | 4 +- setup.py | 2 +- tests/test_generics_and_serialization.py | 8 +- 35 files changed, 1724 insertions(+), 1852 deletions(-) create mode 100644 .github/workflows/docs.yml delete mode 100644 _update_readme.py create mode 100644 dcargs/_deprecated.py create mode 100644 dcargs/extras/__init__.py rename dcargs/{ => extras}/_serialization.py (96%) create mode 100644 docs/.gitignore create mode 100644 docs/Makefile create mode 100644 docs/requirements.txt create mode 100644 docs/source/alternatives.md create mode 100644 docs/source/conf.py create mode 100644 docs/source/example_01.rst create mode 100644 docs/source/example_02.rst create mode 100644 docs/source/example_03.rst create mode 100644 docs/source/example_04.rst create mode 100644 docs/source/example_05.rst create mode 100644 docs/source/example_06.rst create mode 100644 docs/source/example_07.rst create mode 100644 docs/source/example_08.rst create mode 100644 docs/source/example_09.rst create mode 100644 docs/source/example_10.rst create mode 100644 docs/source/example_11.rst create mode 100644 docs/source/example_12.rst create mode 100644 docs/source/example_13.rst create mode 100644 docs/source/example_14.rst create mode 100644 docs/source/index.rst create mode 100644 docs/source/serialization.rst create mode 100644 docs/update_example_docs.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..63108f0ab --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,31 @@ +name: docs + +on: + push: + branches: [master] + +jobs: + docs: + runs-on: ubuntu-latest + container: + image: python:3.8 + steps: + + # Check out source + - uses: actions/checkout@v2 + + # Build documentation + - name: Building documentation + run: | + apt-get update + apt-get install -y graphviz + pip install -e ".[testing]" + pip install -r docs/requirements.txt + sphinx-build docs/source docs/build -b dirhtml + + # Deploy + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./docs/build diff --git a/README.md b/README.md index 8d63b6509..e80533399 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,21 @@ # dcargs -```bash -pip install dcargs -``` - ![build](https://github.com/brentyi/dcargs/workflows/build/badge.svg) ![mypy](https://github.com/brentyi/dcargs/workflows/mypy/badge.svg?branch=master) ![lint](https://github.com/brentyi/dcargs/workflows/lint/badge.svg) [![codecov](https://codecov.io/gh/brentyi/dcargs/branch/master/graph/badge.svg)](https://codecov.io/gh/brentyi/dcargs) +[![PyPI Python Version][pypi-versions-badge]][pypi] -- [Overview](#overview) -- [API](#api) -- [Examples](#examples) -- [Serialization](#serialization) -- [Alternative tools](#alternative-tools) - -## Overview +[pypi-versions-badge]: https://img.shields.io/pypi/pyversions/dcargs +[pypi-badge]: https://badge.fury.io/py/dcargs.svg +[pypi]: https://pypi.org/project/dcargs/ **`dcargs`** is a library for typed CLI interfaces and configuration objects. +``` +pip install dcargs +``` + Our core interface generates argument parsers from type-annotated callables. In the simplest case, this can be used as a drop-in replacement for `argparse`: @@ -65,1584 +62,9 @@ dcargs.cli(main)
-The broader goal is also a replacement for tools like `hydra`, `gin-config`, and -`ml_collections` that's: - -- **Low effort.** Standard Python type annotations, docstrings, and default - values are parsed to automatically generate command-line interfaces with - informative helptext. - -- **Expressive.** `dcargs.cli()` understands functions, classes, dataclasses, - and _nested_ classes and dataclasses, as well as frequently used annotations - like unions, literals, and collections, which can be composed into - hierarchical configuration objects built on standard Python features. - -- **Typed.** Unlike dynamic configuration namespaces produced by libraries like - `argparse`, `YACS`, `abseil`, `hydra`, or `ml_collections`, typed outputs mean - that IDE-assisted autocomplete, rename, refactor, and go-to-definition - operations work out-of-the-box, as well as static checking tools like `mypy` - and `pyright`. - -- **Modular.** Most approaches to configuration objects require a centralized - definition of all configurable fields. Hierarchically nesting configuration - structures, however, makes it easy to distribute definitions, defaults, and - documentation of configurable fields across modules or source files. A model - configuration dataclass, for example, can be co-located in its entirety with - the model implementation and dropped into any experiment configuration with an - import — this eliminates redundancy and makes entire modules easy to port - across codebases. - -## API - - - -```python -def cli( - f: Callable[..., T], - *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default_instance: Optional[T] = None, - avoid_subparsers: bool = False, -) -> T -``` - - - -
-Docstring - - - -``` -Call `f(...)`, with arguments populated from an automatically generated CLI -interface. - -`f` should have type-annotated inputs, and can be a function or class. Note that if -`f` is a class, `dcargs.cli()` returns an instance. - -The parser is generated by populating helptext from docstrings and types from -annotations; a broad range of core type annotations are supported... - - Types natively accepted by `argparse`: str, int, float, pathlib.Path, etc. - - Default values for optional parameters. - - Booleans, which are automatically converted to flags when provided a default - value. - - Enums (via `enum.Enum`). - - Various annotations from the standard typing library. Some examples: - - `typing.ClassVar[T]`. - - `typing.Optional[T]`. - - `typing.Literal[T]`. - - `typing.Sequence[T]`. - - `typing.List[T]`. - - `typing.Dict[K, V]`. - - `typing.Tuple`, such as `typing.Tuple[T1, T2, T3]` or - `typing.Tuple[T, ...]`. - - `typing.Set[T]`. - - `typing.Final[T]` and `typing.Annotated[T]`. - - `typing.Union[T1, T2]`. - - Various nested combinations of the above: `Optional[Literal[T]]`, - `Final[Optional[Sequence[T]]]`, etc. - - Hierarchical structures via nested dataclasses, TypedDict, NamedTuple, - classes. - - Simple nesting. - - Unions over nested structures (subparsers). - - Optional unions over nested structures (optional subparsers). - - Generics (including nested generics). - -Args: - f: Callable. - -Keyword Args: - prog: The name of the program printed in helptext. Mirrors argument from - `argparse.ArgumentParser()`. - description: Description text for the parser, displayed when the --help flag is - passed in. If not specified, `f`'s docstring is used. Mirrors argument from - `argparse.ArgumentParser()`. - args: If set, parse arguments from a sequence of strings instead of the - commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. - default_instance: An instance of `T` to use for default values; only supported - if `T` is a dataclass, TypedDict, or NamedTuple. Helpful for merging CLI - arguments with values loaded from elsewhere. (for example, a config object - loaded from a yaml file) - avoid_subparsers: Avoid creating a subparser when defaults are provided for - unions over nested types. Generates cleaner but less expressive CLIs. - -Returns: - The output of `f(...)`. -``` - - - -
- -## Examples - - -
- -1. Functions - -
- -In the simplest case, `dcargs.cli()` can be used to run a function with -arguments populated from the CLI. - -**Code ([link](examples/01_functions.py)):** - -```python -import dcargs - - -def main( - field1: str, - field2: int = 3, -) -> None: - """Function, whose arguments will be populated from a CLI interface. - - Args: - field1: A string field. - field2: A numeric field, with a default value. - """ - print(field1, field2) - - -if __name__ == "__main__": - dcargs.cli(main) -``` - -
- -**Example usage:** - -
-$ python ./01_functions.py --help
-usage: 01_functions.py [-h] --field1 STR [--field2 INT]
-
-Function, whose arguments will be populated from a CLI interface.
-
-arguments:
-  -h, --help    show this help message and exit
-  --field1 STR  A string field. (required)
-  --field2 INT  A numeric field, with a default value. (default: 3)
-
- -
-$ python ./01_functions.py --field1 hello
-hello 3
-
- -
-$ python ./01_functions.py --field1 hello --field2 10
-hello 10
-
- -
-
- -
- -2. Dataclasses - -
- -Common pattern: use `dcargs.cli()` to instantiate a dataclass. The outputted -instance can be used as a typed alternative for an argparse namespace. - -**Code ([link](examples/02_dataclasses.py)):** - -```python -import dataclasses - -import dcargs - - -@dataclasses.dataclass -class Args: - """Description. - This should show up in the helptext!""" - - field1: str # A string field. - field2: int = 3 # A numeric field, with a default value. - - -if __name__ == "__main__": - args = dcargs.cli(Args) - print(args) -``` - -
- -**Example usage:** - -
-$ python ./02_dataclasses.py --help
-usage: 02_dataclasses.py [-h] --field1 STR [--field2 INT]
-
-Description.
-This should show up in the helptext!
-
-arguments:
-  -h, --help    show this help message and exit
-  --field1 STR  A string field. (required)
-  --field2 INT  A numeric field, with a default value. (default: 3)
-
- -
-$ python ./02_dataclasses.py --field1 hello
-Args(field1='hello', field2=3)
-
- -
-$ python ./02_dataclasses.py --field1 hello --field2 5
-Args(field1='hello', field2=5)
-
- -
-
- -
- -3. Enums And Containers - -
- -We can generate argument parsers from more advanced type annotations, like enums -and tuple types. - -**Code ([link](examples/03_enums_and_containers.py)):** - -```python -import dataclasses -import enum -import pathlib -from typing import Optional, Tuple - -import dcargs - - -class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - -@dataclasses.dataclass(frozen=True) -class TrainConfig: - # Example of a variable-length tuple. `typing.List`, `typing.Sequence`, - # `typing.Set`, `typing.Dict`, etc are all supported as well. - dataset_sources: Tuple[pathlib.Path, ...] - """Paths to load training data from. This can be multiple!""" - - # Fixed-length tuples are also okay. - image_dimensions: Tuple[int, int] = (32, 32) - """Height and width of some image data.""" - - # Enums are handled seamlessly. - optimizer_type: OptimizerType = OptimizerType.ADAM - """Gradient-based optimizer to use.""" - - # We can also explicitly mark arguments as optional. - checkpoint_interval: Optional[int] = None - """Interval to save checkpoints at.""" - - -if __name__ == "__main__": - config = dcargs.cli(TrainConfig) - print(config) -``` - -
- -**Example usage:** - -
-$ python ./03_enums_and_containers.py --help
-usage: 03_enums_and_containers.py [-h] --dataset-sources PATH [PATH ...]
-                                  [--image-dimensions INT INT]
-                                  [--optimizer-type {ADAM,SGD}]
-                                  [--checkpoint-interval {None}|INT]
-
-arguments:
-  -h, --help            show this help message and exit
-  --dataset-sources PATH [PATH ...]
-                        Paths to load training data from. This can be
-                        multiple! (required)
-  --image-dimensions INT INT
-                        Height and width of some image data. (default: 32
-                        32)
-  --optimizer-type {ADAM,SGD}
-                        Gradient-based optimizer to use. (default:
-                        ADAM)
-  --checkpoint-interval {None}|INT
-                        Interval to save checkpoints at. (default:
-                        None)
-
- -
-$ python ./03_enums_and_containers.py --dataset-sources ./data --image-dimensions 16 16
-TrainConfig(dataset_sources=(PosixPath('data'),), image_dimensions=(16, 16), optimizer_type=<OptimizerType.ADAM: 1>, checkpoint_interval=None)
-
- -
-$ python ./03_enums_and_containers.py --dataset-sources ./data --optimizer-type SGD
-TrainConfig(dataset_sources=(PosixPath('data'),), image_dimensions=(32, 32), optimizer_type=<OptimizerType.SGD: 2>, checkpoint_interval=None)
-
- -
-
- -
- -4. Flags - -
- -Booleans can either be expected to be explicitly passed in, or, if given a -default value, automatically converted to flags. - -**Code ([link](examples/04_flags.py)):** - -```python -import dataclasses -from typing import Optional - -import dcargs - - -@dataclasses.dataclass -class Args: - # Boolean. This expects an explicit "True" or "False". - boolean: bool - - # Optional boolean. Same as above, but can be omitted. - optional_boolean: Optional[bool] - - # Pass --flag-a in to set this value to True. - flag_a: bool = False - - # Pass --no-flag-b in to set this value to False. - flag_b: bool = True - - -if __name__ == "__main__": - args = dcargs.cli(Args) - print(args) -``` - -
- -**Example usage:** - -
-$ python ./04_flags.py --help
-usage: 04_flags.py [-h] --boolean {True,False} --optional-boolean
-                   {None,True,False} [--flag-a] [--no-flag-b]
-
-arguments:
-  -h, --help            show this help message and exit
-  --boolean {True,False}
-                        Boolean. This expects an explicit "True" or "False".
-                        (required)
-  --optional-boolean {None,True,False}
-                        Optional boolean. Same as above, but can be omitted.
-                        (required)
-  --flag-a              Pass --flag-a in to set this value to True. (sets:
-                        flag_a=True)
-  --no-flag-b           Pass --no-flag-b in to set this value to False.
-                        (sets: flag_b=False)
-
- -
-$ python ./04_flags.py --boolean True
-usage: 04_flags.py [-h] --boolean {True,False} --optional-boolean
-                   {None,True,False} [--flag-a] [--no-flag-b]
-04_flags.py: error: the following arguments are required: --optional-boolean
-
- -
-$ python ./04_flags.py --boolean False --flag-a
-usage: 04_flags.py [-h] --boolean {True,False} --optional-boolean
-                   {None,True,False} [--flag-a] [--no-flag-b]
-04_flags.py: error: the following arguments are required: --optional-boolean
-
- -
-$ python ./04_flags.py --boolean False --no-flag-b
-usage: 04_flags.py [-h] --boolean {True,False} --optional-boolean
-                   {None,True,False} [--flag-a] [--no-flag-b]
-04_flags.py: error: the following arguments are required: --optional-boolean
-
- -
-
- -
- -5. Hierarchical Configs - -
- -Parsing of nested types (in this case nested dataclasses) enables hierarchical -configuration objects that are both modular and highly expressive. - -**Code ([link](examples/05_hierarchical_configs.py)):** - -```python -import dataclasses -import enum -import pathlib - -import dcargs - - -class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - -@dataclasses.dataclass(frozen=True) -class OptimizerConfig: - # Gradient-based optimizer to use. - algorithm: OptimizerType = OptimizerType.ADAM - - # Learning rate to use. - learning_rate: float = 3e-4 - - # Coefficient for L2 regularization. - weight_decay: float = 1e-2 - - -@dataclasses.dataclass(frozen=True) -class ExperimentConfig: - # Various configurable options for our optimizer. - optimizer: OptimizerConfig - - # Batch size. - batch_size: int = 32 - - # Total number of training steps. - train_steps: int = 100_000 - - # Random seed. This is helpful for making sure that our experiments are all - # reproducible! - seed: int = 0 - - -def train( - out_dir: pathlib.Path, - config: ExperimentConfig, - restore_checkpoint: bool = False, - checkpoint_interval: int = 1000, -) -> None: - """Train a model. - - Args: - out_dir: Where to save logs and checkpoints. - config: Experiment configuration. - restore_checkpoint: Set to restore an existing checkpoint. - checkpoint_interval: Training steps between each checkpoint save. - """ - print(f"{out_dir=}, {restore_checkpoint=}, {checkpoint_interval=}") - print(f"{config=}") - print(dcargs.to_yaml(config)) - - -if __name__ == "__main__": - dcargs.cli(train) -``` - -
- -**Example usage:** - -
-$ python ./05_hierarchical_configs.py --help
-usage: 05_hierarchical_configs.py [-h] --out-dir PATH
-                                  [--config.optimizer.algorithm {ADAM,SGD}]
-                                  [--config.optimizer.learning-rate FLOAT]
-                                  [--config.optimizer.weight-decay FLOAT]
-                                  [--config.batch-size INT]
-                                  [--config.train-steps INT]
-                                  [--config.seed INT] [--restore-checkpoint]
-                                  [--checkpoint-interval INT]
-
-Train a model.
-
-arguments:
-  -h, --help            show this help message and exit
-  --out-dir PATH  Where to save logs and checkpoints.
-                        (required)
-  --restore-checkpoint  Set to restore an existing checkpoint. (sets:
-                        restore_checkpoint=True)
-  --checkpoint-interval INT
-                        Training steps between each checkpoint save.
-                        (default: 1000)
-
-config.optimizer arguments:
-  Various configurable options for our optimizer.
-
-  --config.optimizer.algorithm {ADAM,SGD}
-                        Gradient-based optimizer to use. (default:
-                        ADAM)
-  --config.optimizer.learning-rate FLOAT
-                        Learning rate to use. (default: 0.0003)
-  --config.optimizer.weight-decay FLOAT
-                        Coefficient for L2 regularization. (default:
-                        0.01)
-
-config arguments:
-  Experiment configuration.
-
-  --config.batch-size INT
-                        Batch size. (default: 32)
-  --config.train-steps INT
-                        Total number of training steps. (default:
-                        100000)
-  --config.seed INT  Random seed. This is helpful for making sure that our
-                        experiments are all reproducible! (default: 0)
-
- -
-$ python ./05_hierarchical_configs.py . --config.optimizer.algorithm SGD
-usage: 05_hierarchical_configs.py [-h] --out-dir PATH
-                                  [--config.optimizer.algorithm {ADAM,SGD}]
-                                  [--config.optimizer.learning-rate FLOAT]
-                                  [--config.optimizer.weight-decay FLOAT]
-                                  [--config.batch-size INT]
-                                  [--config.train-steps INT]
-                                  [--config.seed INT] [--restore-checkpoint]
-                                  [--checkpoint-interval INT]
-05_hierarchical_configs.py: error: the following arguments are required: --out-dir
-
- -
-$ python ./05_hierarchical_configs.py . --restore-checkpoint
-usage: 05_hierarchical_configs.py [-h] --out-dir PATH
-                                  [--config.optimizer.algorithm {ADAM,SGD}]
-                                  [--config.optimizer.learning-rate FLOAT]
-                                  [--config.optimizer.weight-decay FLOAT]
-                                  [--config.batch-size INT]
-                                  [--config.train-steps INT]
-                                  [--config.seed INT] [--restore-checkpoint]
-                                  [--checkpoint-interval INT]
-05_hierarchical_configs.py: error: the following arguments are required: --out-dir
-
- -
-
- -
- -6. Base Configs - -
- -We can integrate `dcargs.cli()` into common configuration patterns: here, we -select one of multiple possible base configurations, and then use the CLI to -either override (existing) or fill in (missing) values. - -**Code ([link](examples/06_base_configs.py)):** - -```python -import sys -from dataclasses import dataclass -from typing import Callable, Dict, Literal, Tuple, TypeVar, Union - -from torch import nn - -import dcargs - - -@dataclass(frozen=True) -class AdamOptimizer: - learning_rate: float = 1e-3 - betas: Tuple[float, float] = (0.9, 0.999) - - -@dataclass(frozen=True) -class SgdOptimizer: - learning_rate: float = 3e-4 - - -@dataclass(frozen=True) -class ExperimentConfig: - # Dataset to run experiment on. - dataset: Literal["mnist", "imagenet-50"] - - # Optimizer parameters. - optimizer: Union[AdamOptimizer, SgdOptimizer] - - # Model size. - num_layers: int - units: int - - # Batch size. - batch_size: int - - # Total number of training steps. - train_steps: int - - # Random seed. This is helpful for making sure that our experiments are all - # reproducible! - seed: int - - # Activation to use. Not specifiable via the commandline. - activation: Callable[[], nn.Module] - - -# Note that we could also define this library using separate YAML files (similar to -# `config_path`/`config_name` in Hydra), but staying in Python enables seamless type -# checking + IDE support. -base_configs = { - "small": ExperimentConfig( - dataset="mnist", - optimizer=SgdOptimizer(), - batch_size=2048, - num_layers=4, - units=64, - train_steps=30_000, - # The dcargs.MISSING sentinel allows us to specify that the seed should have no - # default, and needs to be populated from the CLI. - seed=dcargs.MISSING, - activation=nn.ReLU, - ), - "big": ExperimentConfig( - dataset="imagenet-50", - optimizer=AdamOptimizer(), - batch_size=32, - num_layers=8, - units=256, - train_steps=100_000, - seed=dcargs.MISSING, - activation=nn.GELU, - ), -} - - -T = TypeVar("T") - - -def cli_from_base_configs(base_library: Dict[str, T]) -> T: - """Populate an instance of `cls`, where the first positional argument is used to - select from a library of named base configs.""" - # Get base configuration name from the first positional argument. - if len(sys.argv) < 2 or sys.argv[1] not in base_library: - valid_usages = map(lambda k: f"{sys.argv[0]} {k} --help", base_library.keys()) - raise SystemExit("usage:\n " + "\n ".join(valid_usages)) - - # Get base configuration from our library, and use it for default CLI parameters. - default_instance = base_library[sys.argv[1]] - return dcargs.cli( - type(default_instance), - prog=" ".join(sys.argv[:2]), - args=sys.argv[2:], - default_instance=default_instance, - # `avoid_subparsers` will avoid making a subparser for unions when a default is - # provided; in this case, it simplifies our CLI but makes it less expressive - # (cannot switch away from the base optimizer types). - avoid_subparsers=True, - ) - - -if __name__ == "__main__": - config = cli_from_base_configs(base_configs) - print(config) -``` - -
- -**Example usage:** - -
-$ python ./06_base_configs.py
-usage:
-  examples/06_base_configs.py small --help
-  examples/06_base_configs.py big --help
-
- -
-$ python ./06_base_configs.py small --help
-usage: examples/06_base_configs.py small [-h] [--dataset {mnist,imagenet-50}]
-                                         [--optimizer.learning-rate FLOAT]
-                                         [--num-layers INT] [--units INT]
-                                         [--batch-size INT]
-                                         [--train-steps INT] --seed INT
-                                         [--activation {<class 'torch.nn.modules.activation.ReLU'>}]
-
-arguments:
-  -h, --help            show this help message and exit
-  --dataset {mnist,imagenet-50}
-                        Dataset to run experiment on. (default: mnist)
-  --num-layers INT  Model size. (default: 4)
-  --units INT   Model size. (default: 64)
-  --batch-size INT  Batch size. (default: 2048)
-  --train-steps INT  Total number of training steps. (default:
-                        30000)
-  --seed INT    Random seed. This is helpful for making sure that our
-                        experiments are all reproducible!
-                        (required)
-  --activation {<class 'torch.nn.modules.activation.ReLU'>}
-                        Activation to use. Not specifiable via the
-                        commandline. (fixed)
-
-optimizer arguments:
-  Optimizer parameters.
-
-  --optimizer.learning-rate FLOAT
-                        (default: 0.0003)
-
- -
-$ python ./06_base_configs.py small --seed 94720
-ExperimentConfig(dataset='mnist', optimizer=SgdOptimizer(learning_rate=0.0003), num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720, activation=<class 'torch.nn.modules.activation.ReLU'>)
-
- -
-$ python ./06_base_configs.py big --help
-usage: examples/06_base_configs.py big [-h] [--dataset {mnist,imagenet-50}]
-                                       [--optimizer.learning-rate FLOAT]
-                                       [--optimizer.betas FLOAT FLOAT]
-                                       [--num-layers INT] [--units INT]
-                                       [--batch-size INT] [--train-steps INT]
-                                       --seed INT
-                                       [--activation {<class 'torch.nn.modules.activation.GELU'>}]
-
-arguments:
-  -h, --help            show this help message and exit
-  --dataset {mnist,imagenet-50}
-                        Dataset to run experiment on. (default:
-                        imagenet-50)
-  --num-layers INT  Model size. (default: 8)
-  --units INT   Model size. (default: 256)
-  --batch-size INT  Batch size. (default: 32)
-  --train-steps INT  Total number of training steps. (default:
-                        100000)
-  --seed INT    Random seed. This is helpful for making sure that our
-                        experiments are all reproducible!
-                        (required)
-  --activation {<class 'torch.nn.modules.activation.GELU'>}
-                        Activation to use. Not specifiable via the
-                        commandline. (fixed)
-
-optimizer arguments:
-  Optimizer parameters.
-
-  --optimizer.learning-rate FLOAT
-                        (default: 0.001)
-  --optimizer.betas FLOAT FLOAT
-                        (default: 0.9 0.999)
-
- -
-$ python ./06_base_configs.py big --seed 94720
-ExperimentConfig(dataset='imagenet-50', optimizer=AdamOptimizer(learning_rate=0.001, betas=(0.9, 0.999)), num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720, activation=<class 'torch.nn.modules.activation.GELU'>)
-
- -
-
- -
- -7. Literals And Unions - -
- -`typing.Literal[]` can be used to restrict inputs to a fixed set of literal -choices; `typing.Union[]` can be used to restrict inputs to a fixed set of -types. - -**Code ([link](examples/07_literals_and_unions.py)):** - -```python -import dataclasses -import enum -from typing import Literal, Optional, Tuple, Union - -import dcargs - - -class Color(enum.Enum): - RED = enum.auto() - GREEN = enum.auto() - BLUE = enum.auto() - - -@dataclasses.dataclass(frozen=True) -class Args: - # We can use Literal[] to restrict the set of allowable inputs, for example, over - # enums. - restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED - - # Literals can also be marked Optional. - integer: Optional[Literal[0, 1, 2, 3]] = None - - # Unions can be used to specify multiple allowable types. - union_over_types: Union[int, str] = 0 - string_or_enum: Union[Literal["red", "green"], Color] = "red" - - # Unions also work over more complex nested types. - union_over_tuples: Union[Tuple[int, int], Tuple[str]] = ("1",) - - # And can be nested in other types. - tuple_of_string_or_enum: Tuple[Union[Literal["red", "green"], Color], ...] = ( - "red", - Color.RED, - ) - - -if __name__ == "__main__": - args = dcargs.cli(Args) - print(args) -``` - -
- -**Example usage:** - -
-$ python ./07_literals_and_unions.py --help
-usage: 07_literals_and_unions.py [-h] [--restricted-enum {RED,GREEN}]
-                                 [--integer {None,0,1,2,3}]
-                                 [--union-over-types INT|STR]
-                                 [--string-or-enum {red,green,RED,GREEN,BLUE}]
-                                 [--union-over-tuples {INT INT}|STR]
-                                 [--tuple-of-string-or-enum {red,green,RED,GREEN,BLUE} [{red,green,RED,GREEN,BLUE} ...]]
-
-arguments:
-  -h, --help            show this help message and exit
-  --restricted-enum {RED,GREEN}
-                        We can use Literal[] to restrict the set of allowable
-                        inputs, for example, over enums. (default:
-                        RED)
-  --integer {None,0,1,2,3}
-                        Literals can also be marked Optional. (default:
-                        None)
-  --union-over-types INT|STR
-                        Unions can be used to specify multiple allowable
-                        types. (default: 0)
-  --string-or-enum {red,green,RED,GREEN,BLUE}
-                        Unions can be used to specify multiple allowable
-                        types. (default: red)
-  --union-over-tuples {INT INT}|STR
-                        Unions also work over more complex nested types.
-                        (default: 1)
-  --tuple-of-string-or-enum {red,green,RED,GREEN,BLUE} [{red,green,RED,GREEN,BLUE} ...]
-                        And can be nested in other types. (default: red
-                        RED)
-
- -
-
- -
- -8. Positional Args - -
- -Positional-only arguments in functions are converted to positional CLI -arguments. - -**Code ([link](examples/08_positional_args.py)):** - -```python -from __future__ import annotations - -import dataclasses -import enum -import pathlib -from typing import Tuple - -import dcargs - - -def main( - source: pathlib.Path, - dest: pathlib.Path, - /, # Mark the end of positional arguments. - optimizer: OptimizerConfig, - force: bool = False, - verbose: bool = False, - background_rgb: Tuple[float, float, float] = (1.0, 0.0, 0.0), -) -> None: - """Command-line interface defined using a function signature. Note that this - docstring is parsed to generate helptext. - - Args: - source: Source path. - dest: Destination path. - optimizer: Configuration for our optimizer object. - force: Do not prompt before overwriting. - verbose: Explain what is being done. - background_rgb: Background color. Red by default. - """ - print(f"{source=}\n{dest=}\n{optimizer=}\n{force=}\n{verbose=}\n{background_rgb=}") - - -class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - -@dataclasses.dataclass(frozen=True) -class OptimizerConfig: - algorithm: OptimizerType = OptimizerType.ADAM - """Gradient-based optimizer to use.""" - - learning_rate: float = 3e-4 - """Learning rate to use.""" - - weight_decay: float = 1e-2 - """Coefficient for L2 regularization.""" - - -if __name__ == "__main__": - dcargs.cli(main) -``` - -
- -**Example usage:** - -
-$ python ./08_positional_args.py --help
-usage: 08_positional_args.py [-h] [--optimizer.algorithm {ADAM,SGD}]
-                             [--optimizer.learning-rate FLOAT]
-                             [--optimizer.weight-decay FLOAT] [--force]
-                             [--verbose] [--background-rgb FLOAT FLOAT FLOAT]
-                             SOURCE DEST
-
-Command-line interface defined using a function signature. Note that this
-docstring is parsed to generate helptext.
-
-positional arguments:
-  SOURCE                Source path. (required)
-  DEST                  Destination path. (required)
-
-arguments:
-  -h, --help            show this help message and exit
-  --force               Do not prompt before overwriting. (sets:
-                        force=True)
-  --verbose             Explain what is being done. (sets:
-                        verbose=True)
-  --background-rgb FLOAT FLOAT FLOAT
-                        Background color. Red by default. (default: 1.0
-                        0.0 0.0)
-
-optimizer arguments:
-  Configuration for our optimizer object.
-
-  --optimizer.algorithm {ADAM,SGD}
-                        Gradient-based optimizer to use. (default:
-                        ADAM)
-  --optimizer.learning-rate FLOAT
-                        Learning rate to use. (default: 0.0003)
-  --optimizer.weight-decay FLOAT
-                        Coefficient for L2 regularization. (default:
-                        0.01)
-
- -
-$ python ./08_positional_args.py ./a ./b --optimizer.learning-rate 1e-5
-source=PosixPath('a')
-dest=PosixPath('b')
-optimizer=OptimizerConfig(algorithm=<OptimizerType.ADAM: 1>, learning_rate=1e-05, weight_decay=0.01)
-force=False
-verbose=False
-background_rgb=(1.0, 0.0, 0.0)
-
- -
-
- -
- -9. Subparsers - -
- -Unions over nested types (classes or dataclasses) are populated using -subparsers. - -**Code ([link](examples/09_subparsers.py)):** - -```python -from __future__ import annotations - -import dataclasses -from typing import Union - -import dcargs - - -@dataclasses.dataclass(frozen=True) -class Checkout: - """Checkout a branch.""" - - branch: str - - -@dataclasses.dataclass(frozen=True) -class Commit: - """Commit changes.""" - - message: str - all: bool = False - - -def main(cmd: Union[Checkout, Commit]) -> None: - print(cmd) - - -if __name__ == "__main__": - dcargs.cli(main) -``` - -
- -**Example usage:** - -
-$ python ./09_subparsers.py --help
-usage: 09_subparsers.py [-h] {checkout,commit}
-
-arguments:
-  -h, --help         show this help message and exit
-
-subcommands:
-
-  {checkout,commit}
-
- -
-$ python ./09_subparsers.py commit --help
-usage: 09_subparsers.py commit [-h] --cmd.message STR [--cmd.all]
-
-Commit changes.
-
-arguments:
-  -h, --help         show this help message and exit
-
-cmd arguments:
-  --cmd.message STR  (required)
-  --cmd.all          (sets: all=True)
-
- -
-$ python ./09_subparsers.py commit --cmd.message hello --cmd.all
-Commit(message='hello', all=True)
-
- -
-$ python ./09_subparsers.py checkout --help
-usage: 09_subparsers.py checkout [-h] --cmd.branch STR
-
-Checkout a branch.
-
-arguments:
-  -h, --help        show this help message and exit
-
-cmd arguments:
-  --cmd.branch STR  (required)
-
- -
-$ python ./09_subparsers.py checkout --cmd.branch main
-Checkout(branch='main')
-
- -
-
- -
- -10. Multiple Subparsers - -
- -Multiple unions over nested types are populated using a series of subparsers. - -**Code ([link](examples/10_multiple_subparsers.py)):** - -```python -from __future__ import annotations - -import dataclasses -from typing import Literal, Tuple, Union - -import dcargs - -# Possible dataset configurations. - - -@dataclasses.dataclass -class MnistDataset: - binary: bool = False - """Set to load binary version of MNIST dataset.""" - - -@dataclasses.dataclass -class ImageNetDataset: - subset: Literal[50, 100, 1000] - """Choose between ImageNet-50, ImageNet-100, ImageNet-1000, etc.""" - - -# Possible optimizer configurations. - - -@dataclasses.dataclass -class AdamOptimizer: - learning_rate: float = 1e-3 - betas: Tuple[float, float] = (0.9, 0.999) - - -@dataclasses.dataclass -class SgdOptimizer: - learning_rate: float = 3e-4 - - -# Train script. - - -def train( - dataset: Union[MnistDataset, ImageNetDataset] = MnistDataset(), - optimizer: Union[AdamOptimizer, SgdOptimizer] = AdamOptimizer(), -) -> None: - """Example training script. - - Args: - dataset: Dataset to train on. - optimizer: Optimizer to train with. - - Returns: - None: - """ - print(dataset) - print(optimizer) - - -if __name__ == "__main__": - dcargs.cli(train) -``` - -
- -**Example usage:** - -
-$ python ./10_multiple_subparsers.py
-MnistDataset(binary=False)
-AdamOptimizer(learning_rate=0.001, betas=(0.9, 0.999))
-
- -
-$ python ./10_multiple_subparsers.py --help
-usage: 10_multiple_subparsers.py [-h] [{mnist-dataset,image-net-dataset}]
-
-Example training script.
-
-arguments:
-  -h, --help            show this help message and exit
-
-optional subcommands:
-  Dataset to train on.  (default: mnist-dataset)
-
-  [{mnist-dataset,image-net-dataset}]
-
- -
-$ python ./10_multiple_subparsers.py mnist-dataset --help
-usage: 10_multiple_subparsers.py mnist-dataset [-h] [--dataset.binary]
-                                               [{adam-optimizer,sgd-optimizer}]
-
-arguments:
-  -h, --help            show this help message and exit
-
-dataset arguments:
-  --dataset.binary      Set to load binary version of MNIST dataset.
-                        (sets: binary=True)
-
-optional subcommands:
-  Optimizer to train with.  (default: adam-optimizer)
-
-  [{adam-optimizer,sgd-optimizer}]
-
- -
-$ python ./10_multiple_subparsers.py mnist-dataset adam-optimizer --optimizer.learning-rate 3e-4
-MnistDataset(binary=False)
-AdamOptimizer(learning_rate=0.0003, betas=(0.9, 0.999))
-
- -
-
- -
- -11. Dictionaries - -
- -Dictionary inputs can be specified using either a standard `Dict[K, V]` -annotation, or a `TypedDict` type. - -**Code ([link](examples/11_dictionaries.py)):** - -```python -from typing import Dict, Tuple, TypedDict - -import dcargs - - -class DictionarySchema( - TypedDict, - # Setting `total=False` specifies that not all keys need to exist. - total=False, -): - learning_rate: float - betas: Tuple[float, float] - - -def main( - typed_dict: DictionarySchema, - standard_dict: Dict[str, float] = { - "learning_rate": 3e-4, - "beta1": 0.9, - "beta2": 0.999, - }, -) -> None: - assert isinstance(standard_dict, dict) - assert isinstance(typed_dict, dict) - print("Standard dict:", standard_dict) - print("Typed dict:", typed_dict) - - -if __name__ == "__main__": - dcargs.cli(main) -``` - -
- -**Example usage:** - -
-$ python ./11_dictionaries.py --help
-usage: 11_dictionaries.py [-h] [--typed-dict.learning-rate FLOAT]
-                          [--typed-dict.betas FLOAT FLOAT]
-                          [--standard-dict STR FLOAT [STR FLOAT ...]]
-
-arguments:
-  -h, --help            show this help message and exit
-  --standard-dict STR FLOAT [STR FLOAT ...]
-                        (default: learning_rate 0.0003 beta1 0.9 beta2
-                        0.999)
-
-typed_dict arguments:
-
-  --typed-dict.learning-rate FLOAT
-                        Setting `total=False` specifies that not all keys need
-                        to exist. (unset by default)
-  --typed-dict.betas FLOAT FLOAT
-                        Setting `total=False` specifies that not all keys need
-                        to exist. (unset by default)
-
- -
-$ python ./11_dictionaries.py --typed-dict.learning-rate 3e-4
-Standard dict: {'learning_rate': 0.0003, 'beta1': 0.9, 'beta2': 0.999}
-Typed dict: {'learning_rate': 0.0003}
-
- -
-$ python ./11_dictionaries.py --typed-dict.betas 0.9 0.999
-Standard dict: {'learning_rate': 0.0003, 'beta1': 0.9, 'beta2': 0.999}
-Typed dict: {'betas': (0.9, 0.999)}
-
- -
-
- -
- -12. Named Tuples - -
- -Example using `dcargs.cli()` to instantiate a named tuple. - -**Code ([link](examples/12_named_tuples.py)):** - -```python -from typing import NamedTuple - -import dcargs - - -class TupleType(NamedTuple): - """Description. - This should show up in the helptext!""" - - field1: str # A string field. - field2: int = 3 # A numeric field, with a default value. - flag: bool = False # A boolean flag. - - -if __name__ == "__main__": - x = dcargs.cli(TupleType) - assert isinstance(x, tuple) - print(x) -``` - -
- -**Example usage:** - -
-$ python ./12_named_tuples.py --help
-usage: 12_named_tuples.py [-h] --field1 STR [--field2 INT] [--flag]
-
-Description.
-This should show up in the helptext!
-
-arguments:
-  -h, --help    show this help message and exit
-  --field1 STR  A string field. (required)
-  --field2 INT  A numeric field, with a default value. (default: 3)
-  --flag        A boolean flag. (sets: flag=True)
-
- -
-$ python ./12_named_tuples.py --field1 hello
-TupleType(field1='hello', field2=3, flag=False)
-
- -
-
- -
- -13. Standard Classes - -
- -In addition to functions and dataclasses, we can also generate CLIs from (the -constructors of) standard Python classes. - -**Code ([link](examples/13_standard_classes.py)):** - -```python -import dcargs - - -class Args: - def __init__( - self, - field1: str, - field2: int, - flag: bool = False, - ): - """Arguments. - - Args: - field1: A string field. - field2: A numeric field. - flag: A boolean flag. - """ - self.data = [field1, field2, flag] - - -if __name__ == "__main__": - args = dcargs.cli(Args) - print(args.data) -``` - -
- -**Example usage:** - -
-$ python ./13_standard_classes.py --help
-usage: 13_standard_classes.py [-h] --field1 STR --field2 INT [--flag]
-
-Arguments.
-
-arguments:
-  -h, --help    show this help message and exit
-  --field1 STR  A string field. (required)
-  --field2 INT  A numeric field. (required)
-  --flag        A boolean flag. (sets: flag=True)
-
- -
-$ python ./13_standard_classes.py --field1 hello --field2 7
-['hello', 7, False]
-
- -
-
- -
- -14. Generics - -
- -Example of parsing for generic dataclasses. - -**Code ([link](examples/14_generics.py)):** - -```python -import dataclasses -from typing import Generic, TypeVar - -import dcargs - -ScalarType = TypeVar("ScalarType") -ShapeType = TypeVar("ShapeType") - - -@dataclasses.dataclass(frozen=True) -class Point3(Generic[ScalarType]): - x: ScalarType - y: ScalarType - z: ScalarType - frame_id: str - - -@dataclasses.dataclass(frozen=True) -class Triangle: - a: Point3[float] - b: Point3[float] - c: Point3[float] - - -@dataclasses.dataclass(frozen=True) -class Args(Generic[ShapeType]): - point_continuous: Point3[float] - point_discrete: Point3[int] - shape: ShapeType - - -if __name__ == "__main__": - args = dcargs.cli(Args[Triangle]) - print(args) -``` - -
- -**Example usage:** - -
-$ python ./14_generics.py --help
-usage: 14_generics.py [-h] --point-continuous.x FLOAT --point-continuous.y
-                      FLOAT --point-continuous.z FLOAT
-                      --point-continuous.frame-id STR --point-discrete.x INT
-                      --point-discrete.y INT --point-discrete.z INT
-                      --point-discrete.frame-id STR --shape.a.x FLOAT
-                      --shape.a.y FLOAT --shape.a.z FLOAT --shape.a.frame-id
-                      STR --shape.b.x FLOAT --shape.b.y FLOAT --shape.b.z
-                      FLOAT --shape.b.frame-id STR --shape.c.x FLOAT
-                      --shape.c.y FLOAT --shape.c.z FLOAT --shape.c.frame-id
-                      STR
-
-arguments:
-  -h, --help            show this help message and exit
-
-point_continuous arguments:
-
-  --point-continuous.x FLOAT
-                        (required)
-  --point-continuous.y FLOAT
-                        (required)
-  --point-continuous.z FLOAT
-                        (required)
-  --point-continuous.frame-id STR
-                        (required)
-
-point_discrete arguments:
-
-  --point-discrete.x INT
-                        (required)
-  --point-discrete.y INT
-                        (required)
-  --point-discrete.z INT
-                        (required)
-  --point-discrete.frame-id STR
-                        (required)
-
-shape.a arguments:
-
-  --shape.a.x FLOAT  (required)
-  --shape.a.y FLOAT  (required)
-  --shape.a.z FLOAT  (required)
-  --shape.a.frame-id STR
-                        (required)
-
-shape.b arguments:
-
-  --shape.b.x FLOAT  (required)
-  --shape.b.y FLOAT  (required)
-  --shape.b.z FLOAT  (required)
-  --shape.b.frame-id STR
-                        (required)
-
-shape.c arguments:
-
-  --shape.c.x FLOAT  (required)
-  --shape.c.y FLOAT  (required)
-  --shape.c.z FLOAT  (required)
-  --shape.c.frame-id STR
-                        (required)
-
- -
-
- -## Serialization - -As a secondary feature aimed at enabling the use of `dcargs.cli()` for general -configuration use cases, we also introduce functions for human-readable -dataclass serialization: - -- dcargs.from_yaml(cls: Type[T], stream: Union[str, - IO[str], bytes, IO[bytes]]) -> T and - dcargs.to_yaml(instance: T) -> str convert - between YAML-style strings and dataclass instances. - -The functions attempt to strike a balance between flexibility and robustness — -in contrast to naively dumping or loading dataclass instances (via pickle, -PyYAML, etc), explicit type references enable custom tags that are robust -against code reorganization and refactor, while a PyYAML backend enables -serialization of arbitrary Python objects. - -Note that we generally prefer to use YAML purely for serialization, as opposed -to a configuration interface that humans are expected to manually write or -modify. Specifying things like loadable base configurations can be done directly -in Python, which enables all of the usual autocompletion and type checking -features. - -## Alternative tools - -The core functionality of `dcargs` — generating argument parsers from type -annotations — can be found as a subset of the features offered by many other -libraries. A summary of some distinguishing features: - -| | Choices from literals | Generics | Docstrings as helptext | Nesting | Subparsers | Containers | -| ------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------- | -------- | ---------------------- | ------- | ---------- | ---------- | -| **dcargs** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| **[datargs](https://github.com/roee30/datargs)** | ✓ | | | | ✓ | ✓ | -| **[tap](https://github.com/swansonk14/typed-argument-parser)** | ✓ | | ✓ | | ✓ | ✓ | -| **[simple-parsing](https://github.com/lebrice/SimpleParsing)** | [soon](https://github.com/lebrice/SimpleParsing/pull/86) | | ✓ | ✓ | ✓ | ✓ | -| **[argparse-dataclass](https://pypi.org/project/argparse-dataclass/)** | | | | | | | -| **[argparse-dataclasses](https://pypi.org/project/argparse-dataclasses/)** | | | | | | | -| **[dataclass-cli](https://github.com/malte-soe/dataclass-cli)** | | | | | | | -| **[clout](https://pypi.org/project/clout/)** | | | | ✓ | | | -| **[hf_argparser](https://github.com/huggingface/transformers/blob/master/src/transformers/hf_argparser.py)** | | | | | | ✓ | -| **[pyrallis](https://github.com/eladrich/pyrallis/)** | | | ✓ | ✓ | | ✓ | +The broader goal is to enable replacing configuration frameworks like `hydra`, +`gin-config`, and `ml_collections` with hierarchical structures built using +standard Python dataclasses and type annotations. -Note that most of these other libraries are generally aimed specifically at -_dataclasses_ rather than general typed callables, but offer other features that -you might find useful, such as registration for custom types (`pyrallis`), -different approaches for serialization and config files (`tap`, `pyrallis`), -simultaneous parsing of multiple dataclasses (`simple-parsing`), etc. Pull -requests are welcome if you're missing any of these. :slightly_smiling_face: +For a full list of features and usage examples, see +[**our documentation**](https://brentyi.github.io/dcargs). diff --git a/_update_readme.py b/_update_readme.py deleted file mode 100644 index 4b0dcbac3..000000000 --- a/_update_readme.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Helper script for updating the auto-generated parts of the README. (docstring, -examples list)""" -import concurrent.futures -import dataclasses -import html -import inspect -import os -import pathlib -import shlex -import subprocess - -import dcargs -import dcargs._strings - - -@dataclasses.dataclass(frozen=True) -class Markers: - signature_start: str = "" - signature_end: str = "" - - docstring_start: str = "" - docstring_end: str = "" - - examples_start: str = "" - examples_end: str = "" - - -def replace_between_markers( - content: str, marker_start: str, marker_end: str, inner: str -) -> str: - """Puts `inner` between `marker_start` and `marker_end`.""" - assert marker_start in content - assert marker_end in content - before, _, after = content.partition(marker_start) - _, _, after = content.rpartition(marker_end) - return "".join([before, marker_start, inner, marker_end, after]) - - -def format_script_for_readme(path: pathlib.Path) -> str: - print("Handling:", path) - - # 01_functions -> 01, _, functions. - index, _, title = path.stem.partition("_") - - # 01 -> 1. - index = str(int(index)) - - # functions -> Functions. - title = title.replace("_", " ").title() - - source = path.read_text().strip() - example_output_lines = [] - - docstring = source.split('"""')[1].strip() - assert "Usage:" in docstring - description_text, _, usage_text = docstring.partition("Usage:") - example_usages = map( - lambda x: x[1:-1], - filter( - lambda line: line.startswith("`") and line.endswith("`"), - usage_text.split("\n"), - ), - ) - for usage in example_usages: - # Example usage: `ENV=something python ./path --help` - args = shlex.split(usage) - python_index = args.index("python") - - env_vars = {} - if python_index > 0: - env_vars = { - k: v for (k, v) in map(lambda x: x.split("="), args[:python_index]) - } - process_output = subprocess.run( - args=["python", str(path)] + args[python_index + 2 :], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf8", - env=dict(os.environ, **env_vars), - ) - if process_output.stderr != "": - output = dcargs._strings.strip_ansi_sequences(process_output.stderr).strip() - elif process_output.stdout != "": - output = dcargs._strings.strip_ansi_sequences(process_output.stdout).strip() - else: - assert False - example_output_lines.extend( - [ - "", - "
",
-                f"$ {usage}",
-                f"{html.escape(output)}",
-                "
", - ] - ) - - return "\n".join( - [ - "", - "
", - "", - f"{index}. {title}", - "", - "
", - "", - description_text, - "", - "\n".join(example_usages), - "", - f"**Code ([link]({path})):**", - "", - "```python", - source[3:].partition('"""')[2].strip(), - "```", - "", - "
", - "", - "**Example usage:**", - "", - ] - + example_output_lines - + [ - "", - "
", - "
", - ] - ) - - -def get_examples_str(examples_dir: pathlib.Path, markers: Markers) -> str: - script_paths = filter( - lambda p: not p.name.startswith("_"), sorted(examples_dir.glob("*.py")) - ) - with concurrent.futures.ThreadPoolExecutor() as executor: - return "\n".join( - executor.map( - format_script_for_readme, - script_paths, - ) - ) - - -REPO_ROOT = pathlib.Path(__file__).parent - - -def main( - readme_path: pathlib.Path = REPO_ROOT / "README.md", - examples_dir: pathlib.Path = REPO_ROOT / "examples", - markers: Markers = Markers(), -) -> None: - """Helper script for generating the examples list in the README.""" - - # Read. - content = readme_path.read_text(encoding="utf8") - - # Update signature. - signature_lines, _ = inspect.getsourcelines(dcargs.cli) - for i in range(len(signature_lines)): - if signature_lines[i].endswith(":\n"): - signature_lines = signature_lines[: i + 1] - break - content = replace_between_markers( - content, - markers.signature_start, - markers.signature_end, - "\n```python\n" + "".join(signature_lines).strip()[:-1] + "\n```\n", - ) - - # Update examples. - content = replace_between_markers( - content, - markers.examples_start, - markers.examples_end, - get_examples_str(examples_dir, markers), - ) - - # Update docstring. - content = replace_between_markers( - content, - markers.docstring_start, - markers.docstring_end, - f"\n\n```\n{inspect.getdoc(dcargs.cli)}\n```\n\n", - ) - - # Write. - readme_path.write_text(content) - - # Format. - try: - subprocess.run(args=["prettier", "-w", readme_path.as_posix()]) - print("Successfully formatted with `prettier`!") - except FileNotFoundError: - print("Tried to format README with `prettier`, but not in PATH.") - - -if __name__ == "__main__": - dcargs.cli(main) diff --git a/dcargs/__init__.py b/dcargs/__init__.py index 2e368c857..1910a2f71 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,14 +1,15 @@ -from ._cli import cli, parse +from . import extras +from ._cli import cli from ._fields import MISSING_PUBLIC as MISSING from ._instantiators import UnsupportedTypeAnnotationError -from ._serialization import from_yaml, to_yaml __all__ = [ + "extras", "MISSING", "cli", - # Deprecated. - # "parse", "UnsupportedTypeAnnotationError", - "from_yaml", - "to_yaml", ] + +# Deprecated interface. We use a star import to prevent these from showing up in +# autocomplete engines, etc. +from ._deprecated import * # noqa diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 7ed80cd05..c07f9c895 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -8,25 +8,6 @@ T = TypeVar("T") -def parse( - f: Callable[..., T], - *, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default_instance: Optional[T] = None, -) -> T: # pragma: no cover - """Deprecated alias for `dcargs.cli()`.""" - # import warnings - # - # warnings.warn( - # "`dcargs.parse()` has been renamed `dcargs.cli()`. It will be removed" - # " soon.", - # DeprecationWarning, - # stacklevel=2, - # ) - return cli(f, description=description, args=args, default_instance=default_instance) - - def cli( f: Callable[..., T], *, @@ -44,36 +25,34 @@ def cli( The parser is generated by populating helptext from docstrings and types from annotations; a broad range of core type annotations are supported... - - Types natively accepted by `argparse`: str, int, float, pathlib.Path, etc. - - Default values for optional parameters. - - Booleans, which are automatically converted to flags when provided a default - value. - - Enums (via `enum.Enum`). - - Various annotations from the standard typing library. Some examples: - - `typing.ClassVar[T]`. - - `typing.Optional[T]`. - - `typing.Literal[T]`. - - `typing.Sequence[T]`. - - `typing.List[T]`. - - `typing.Dict[K, V]`. - - `typing.Tuple`, such as `typing.Tuple[T1, T2, T3]` or - `typing.Tuple[T, ...]`. - - `typing.Set[T]`. - - `typing.Final[T]` and `typing.Annotated[T]`. - - `typing.Union[T1, T2]`. - - Various nested combinations of the above: `Optional[Literal[T]]`, - `Final[Optional[Sequence[T]]]`, etc. - - Hierarchical structures via nested dataclasses, TypedDict, NamedTuple, - classes. - - Simple nesting. - - Unions over nested structures (subparsers). - - Optional unions over nested structures (optional subparsers). - - Generics (including nested generics). + - Types natively accepted by `argparse`: str, int, float, pathlib.Path, etc. + - Default values for optional parameters. + - Booleans, which are automatically converted to flags when provided a default + value. + - Enums (via `enum.Enum`). + - Various annotations from the standard typing library. Some examples: + - `typing.ClassVar[T]`. + - `typing.Optional[T]`. + - `typing.Literal[T]`. + - `typing.Sequence[T]`. + - `typing.List[T]`. + - `typing.Dict[K, V]`. + - `typing.Tuple`, such as `typing.Tuple[T1, T2, T3]` or + `typing.Tuple[T, ...]`. + - `typing.Set[T]`. + - `typing.Final[T]` and `typing.Annotated[T]`. + - `typing.Union[T1, T2]`. + - Various nested combinations of the above: `Optional[Literal[T]]`, + `Final[Optional[Sequence[T]]]`, etc. + - Hierarchical structures via nested dataclasses, TypedDict, NamedTuple, + classes. + - Simple nesting. + - Unions over nested structures (subparsers). + - Optional unions over nested structures (optional subparsers). + - Generics (including nested generics). Args: f: Callable. - - Keyword Args: prog: The name of the program printed in helptext. Mirrors argument from `argparse.ArgumentParser()`. description: Description text for the parser, displayed when the --help flag is diff --git a/dcargs/_deprecated.py b/dcargs/_deprecated.py new file mode 100644 index 000000000..c879522a8 --- /dev/null +++ b/dcargs/_deprecated.py @@ -0,0 +1,2 @@ +from ._cli import cli as parse # noqa +from .extras._serialization import from_yaml, to_yaml # noqa diff --git a/dcargs/_fields.py b/dcargs/_fields.py index a6b852864..b7a384eed 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -57,8 +57,8 @@ class ExcludeFromKwargsType(_Singleton): # Note that our "public" missing API will always be the propagating missing sentinel. MISSING_PUBLIC: Any = MISSING_PROP -"""Sentinel value to mark fields as missing. Should generally only be used to mark -fields passed in as a `default_instance` for `dcargs.cli()` as required.""" +"""Sentinel value to mark fields as missing. Can be used to mark fields passed in as a +`default_instance` for `dcargs.cli()` as required.""" MISSING_TYPE = Union[PropagatingMissingType, NonpropagatingMissingType] diff --git a/dcargs/extras/__init__.py b/dcargs/extras/__init__.py new file mode 100644 index 000000000..71ab76515 --- /dev/null +++ b/dcargs/extras/__init__.py @@ -0,0 +1,3 @@ +from ._serialization import from_yaml, to_yaml + +__all__ = ["to_yaml", "from_yaml"] diff --git a/dcargs/_serialization.py b/dcargs/extras/_serialization.py similarity index 96% rename from dcargs/_serialization.py rename to dcargs/extras/_serialization.py index 41c4a05e8..77fb647a5 100644 --- a/dcargs/_serialization.py +++ b/dcargs/extras/_serialization.py @@ -7,7 +7,7 @@ import yaml from typing_extensions import get_origin -from . import _fields, _resolver +from .. import _fields, _resolver ENUM_YAML_TAG_PREFIX = "!enum:" DATACLASS_YAML_TAG_PREFIX = "!dataclass:" @@ -176,7 +176,12 @@ def from_yaml( stream: Union[str, IO[str], bytes, IO[bytes]], ) -> DataclassType: """Re-construct a dataclass instance from a yaml-compatible string, which should be - generated from `dcargs.to_yaml()`.""" + generated from `dcargs.to_yaml()`. + + Args: + cls: Type to reconstruct. + stream: YAML to read from. + """ out = yaml.load(stream, Loader=_make_loader(cls)) origin_cls = get_origin(cls) assert isinstance(out, origin_cls if origin_cls is not None else cls) @@ -185,5 +190,9 @@ def from_yaml( def to_yaml(instance: Any) -> str: """Serialize a dataclass; returns a yaml-compatible string that can be deserialized - via `dcargs.from_yaml()`.""" + via `dcargs.from_yaml()`. + + Args: + instance: Dataclass instance to serialize. + """ return "# dcargs YAML.\n" + yaml.dump(instance, Dumper=_make_dumper(instance)) diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 000000000..567609b12 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..201f60130 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = fannypack +SOURCEDIR = source +BUILDDIR = ./build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..bc6a3b4a7 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,9 @@ +sphinx +# sphinx_rtd_theme +furo +sphinx_math_dollar +sphinx-argparse +sphinx-autoapi +m2r2 +git+https://github.com/brentyi/sphinxcontrib-programoutput.git +git+https://github.com/brentyi/ansi.git diff --git a/docs/source/alternatives.md b/docs/source/alternatives.md new file mode 100644 index 000000000..600aa62e6 --- /dev/null +++ b/docs/source/alternatives.md @@ -0,0 +1,24 @@ +# Alternative tools + +The core functionality of `dcargs` — generating argument parsers from type +annotations — can be found as a subset of the features offered by many other +libraries. A summary of some distinguishing features: + +| | Literals | Generics | Docstrings as helptext | Nesting | Subparsers | Containers | +| -------------------------------------------------------------------------------------------------------- | -------------------------------------------------------- | -------- | ---------------------- | ------- | ---------- | ---------- | +| **dcargs** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| [datargs](https://github.com/roee30/datargs) | ✓ | | | | ✓ | ✓ | +| [tap](https://github.com/swansonk14/typed-argument-parser) | ✓ | | ✓ | | ✓ | ✓ | +| [simple-parsing](https://github.com/lebrice/SimpleParsing) | [soon](https://github.com/lebrice/SimpleParsing/pull/86) | | ✓ | ✓ | ✓ | ✓ | +| [argparse-dataclass](https://pypi.org/project/argparse-dataclass/) | | | | | | | +| [argparse-dataclasses](https://pypi.org/project/argparse-dataclasses/) | | | | | | | +| [dataclass-cli](https://github.com/malte-soe/dataclass-cli) | | | | | | | +| [clout](https://pypi.org/project/clout/) | | | | ✓ | | | +| [hf_argparser](https://github.com/huggingface/transformers/blob/master/src/transformers/hf_argparser.py) | | | | | | ✓ | +| [pyrallis](https://github.com/eladrich/pyrallis/) | | | ✓ | ✓ | | ✓ | + +Note that most of these other libraries are generally aimed specifically at +_dataclasses_ rather than general typed callables, but offer other features that +you might find useful, such as registration for custom types (`pyrallis`), +different approaches for serialization and config files (`tap`, `pyrallis`), +simultaneous parsing of multiple dataclasses (`simple-parsing`), etc. diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 000000000..f0ca66668 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,382 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/stable/config + +from typing import Dict, List, Optional + +import m2r2 + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# + +# -- Project information ----------------------------------------------------- + +project = "dcargs" +copyright = "2022" +author = "brentyi" + +# The short X.Y version +version = "" +# The full version, including alpha/beta/rc tags +release = "" + + +# -- General configuration --------------------------------------------------- + +napoleon_numpy_docstring = False # Force consistency, leave only Google +napoleon_use_rtype = False # More legible + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.todo", + "sphinx.ext.coverage", + # "sphinx.ext.mathjax", + "sphinx.ext.githubpages", + "sphinx.ext.napoleon", + # "sphinx.ext.inheritance_diagram", + "autoapi.extension", + # "sphinx_math_dollar", + "sphinx.ext.viewcode", + "sphinxarg.ext", + "m2r2", + "sphinxcontrib.programoutput", + "sphinxcontrib.ansi", +] +programoutput_use_ansi = True +html_ansi_stylesheet = "black-on-white.css" +html_theme_options = { + "light_css_variables": { + "color-code-background": "#f4f4f4", + "color-code-foreground": "#000", + }, +} + +# Pull documentation types from hints +autodoc_typehints = "both" + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = [".rst", ".md"] +# source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language: Optional[str] = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns: List[str] = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "monokai" + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "furo" +html_title = "dcargs" + + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = "dcargs_doc" + + +# -- Options for Github output ------------------------------------------------ + +sphinx_to_github = True +sphinx_to_github_verbose = True +sphinx_to_github_encoding = "utf-8" + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements: Dict[str, str] = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ( + master_doc, + "dcargs.tex", + "dcargs", + "brentyi", + "manual", + ), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "dcargs", "dcargs documentation", [author], 1)] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "dcargs", + "dcargs", + author, + "dcargs", + "dcargs", + "Miscellaneous", + ), +] + + +# -- Extension configuration -------------------------------------------------- + +# -- Options for autoapi extension -------------------------------------------- +autoapi_dirs = ["../../dcargs"] +autoapi_root = "api" +autoapi_options = [ + "members", + "undoc-members", + "imported-members", + "show-inheritance", + # "show-inheritance-diagram", + "special-members", + # "show-module-summary", +] +autoapi_add_toctree_entry = False + +# # Generate name aliases +# def _gen_name_aliases(): +# """Generate a name alias dictionary, which maps private names to ones in the public +# API. A little bit hardcoded/hacky.""" +# +# name_alias = {} +# +# def recurse(module, prefixes): +# if hasattr(module, "__name__") and module.__name__.startswith("dcargs"): +# MAX_DEPTH = 5 +# if len(prefixes) > MAX_DEPTH: +# # Prevent infinite loops from cyclic imports +# return +# else: +# return +# +# for member_name in dir(module): +# if member_name == "dcargs": +# continue +# +# member = getattr(module, member_name) +# if callable(member): +# full_name = ".".join(["dcargs"] + prefixes + [member_name]) +# +# shortened_name = "dcargs" +# current = dcargs +# success = True +# for p in prefixes + [member_name]: +# if p.startswith("_"): +# continue +# if not hasattr(current, p): +# success = False +# break +# current = getattr(current, p) +# shortened_name += "." + p +# +# if success and shortened_name != full_name: +# if full_name in name_alias: +# assert full_name == name_alias[shortened_name], full_name +# else: +# name_alias[full_name] = shortened_name +# elif not member_name.startswith("__"): +# recurse(member, prefixes + [member_name]) +# +# import dcargs +# +# recurse(dcargs, prefixes=[]) +# return name_alias +# +# +# _name_aliases = _gen_name_aliases() +# +# # Set inheritance_alias setting for inheritance diagrams +# inheritance_alias = _name_aliases +# +# +# def _apply_name_aliases(name: Optional[str]) -> Optional[str]: +# if name is None: +# return None +# +# name = name.strip() +# +# if "[" in name: +# # Generics. +# name, _, suffix = name.partition("[") +# assert suffix[-1] == "]" +# suffix = suffix[:-1] +# if "," in suffix: +# suffix = ", ".join(_apply_name_aliases(x) for x in suffix.split(",")) # type: ignore +# else: +# suffix = _apply_name_aliases(suffix) # type: ignore +# suffix = "[" + suffix + "]" +# else: +# suffix = "" +# +# if name in _name_aliases: +# name = _name_aliases[name] +# +# return name + suffix # type: ignore +# +# +# # Apply our inheritance alias to autoapi base classes +# def _override_class_documenter(): +# import autoapi +# +# orig_init = autoapi.mappers.python.PythonClass.__init__ +# +# def __init__(self, obj, **kwargs): +# bases = obj["bases"] +# for i in range(len(bases)): +# bases[i] = _apply_name_aliases(bases[i]) +# +# args = obj["args"] +# if args is not None: +# for i in range(len(args)): +# assert isinstance(args[i], tuple) and len(args[i]) == 4 +# args[i] = ( +# args[i][0], +# args[i][1], +# _apply_name_aliases(args[i][2]), +# args[i][3], +# ) +# orig_init(self, obj, **kwargs) +# +# autoapi.mappers.python.PythonClass.__init__ = __init__ +# +# +# _override_class_documenter() +# +# +# # Apply our inheritance alias to autoapi type annotations +# def _override_function_documenter(): +# import autoapi +# +# orig_init = autoapi.mappers.python.PythonFunction.__init__ +# +# def __init__(self, obj, **kwargs): +# args = obj["args"] +# if args is not None: +# for i in range(len(args)): +# assert isinstance(args[i], tuple) and len(args[i]) == 4 +# args[i] = ( +# args[i][0], +# args[i][1], +# _apply_name_aliases(args[i][2]), +# args[i][3], +# ) +# +# obj["return_annotation"] = _apply_name_aliases(obj["return_annotation"]) +# orig_init(self, obj, **kwargs) +# +# autoapi.mappers.python.PythonFunction.__init__ = __init__ +# +# +# _override_function_documenter() +# +# # Apply our inheritance alias to autoapi attribute annotations +# def _override_attribute_documenter(): +# import autoapi +# +# orig_init = autoapi.mappers.python.PythonAttribute.__init__ +# +# def __init__(self, obj, **kwargs): +# obj["annotation"] = _apply_name_aliases(obj["annotation"]) +# orig_init(self, obj, **kwargs) +# +# autoapi.mappers.python.PythonAttribute.__init__ = __init__ +# +# +# _override_attribute_documenter() + + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + +# -- Enable Markdown -> RST conversion ---------------------------------------- +def docstring(app, what, name, obj, options, lines): + md = "\n".join(lines) + rst = m2r2.convert(md) + lines.clear() + lines += rst.splitlines() + + +def setup(app): + app.connect("autodoc-process-docstring", docstring) diff --git a/docs/source/example_01.rst b/docs/source/example_01.rst new file mode 100644 index 000000000..868830584 --- /dev/null +++ b/docs/source/example_01.rst @@ -0,0 +1,46 @@ +1. Functions +========================================== + +In the simplest case, `dcargs.cli()` can be used to run a function with arguments +populated from the CLI. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + import dcargs + + + def main( + field1: str, + field2: int = 3, + ) -> None: + """Function, whose arguments will be populated from a CLI interface. + + Args: + field1: A string field. + field2: A numeric field, with a default value. + """ + print(field1, field2) + + + if __name__ == "__main__": + dcargs.cli(main) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/01_functions.py --help + +.. command-output:: python ../../examples/01_functions.py --field1 hello + +.. command-output:: python ../../examples/01_functions.py --field1 hello --field2 10 diff --git a/docs/source/example_02.rst b/docs/source/example_02.rst new file mode 100644 index 000000000..77c464812 --- /dev/null +++ b/docs/source/example_02.rst @@ -0,0 +1,45 @@ +2. Dataclasses +========================================== + +Common pattern: use `dcargs.cli()` to instantiate a dataclass. The outputted instance +can be used as a typed alternative for an argparse namespace. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + import dataclasses + + import dcargs + + + @dataclasses.dataclass + class Args: + """Description. + This should show up in the helptext!""" + + field1: str # A string field. + field2: int = 3 # A numeric field, with a default value. + + + if __name__ == "__main__": + args = dcargs.cli(Args) + print(args) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/02_dataclasses.py --help + +.. command-output:: python ../../examples/02_dataclasses.py --field1 hello + +.. command-output:: python ../../examples/02_dataclasses.py --field1 hello --field2 5 diff --git a/docs/source/example_03.rst b/docs/source/example_03.rst new file mode 100644 index 000000000..4410e12db --- /dev/null +++ b/docs/source/example_03.rst @@ -0,0 +1,64 @@ +3. Enums And Containers +========================================== + +We can generate argument parsers from more advanced type annotations, like enums and +tuple types. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + import dataclasses + import enum + import pathlib + from typing import Optional, Tuple + + import dcargs + + + class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + + @dataclasses.dataclass(frozen=True) + class TrainConfig: + # Example of a variable-length tuple. `typing.List`, `typing.Sequence`, + # `typing.Set`, `typing.Dict`, etc are all supported as well. + dataset_sources: Tuple[pathlib.Path, ...] + """Paths to load training data from. This can be multiple!""" + + # Fixed-length tuples are also okay. + image_dimensions: Tuple[int, int] = (32, 32) + """Height and width of some image data.""" + + # Enums are handled seamlessly. + optimizer_type: OptimizerType = OptimizerType.ADAM + """Gradient-based optimizer to use.""" + + # We can also explicitly mark arguments as optional. + checkpoint_interval: Optional[int] = None + """Interval to save checkpoints at.""" + + + if __name__ == "__main__": + config = dcargs.cli(TrainConfig) + print(config) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/03_enums_and_containers.py --help + +.. command-output:: python ../../examples/03_enums_and_containers.py --dataset-sources ./data --image-dimensions 16 16 + +.. command-output:: python ../../examples/03_enums_and_containers.py --dataset-sources ./data --optimizer-type SGD diff --git a/docs/source/example_04.rst b/docs/source/example_04.rst new file mode 100644 index 000000000..c04e1a87c --- /dev/null +++ b/docs/source/example_04.rst @@ -0,0 +1,54 @@ +4. Flags +========================================== + +Booleans can either be expected to be explicitly passed in, or, if given a default +value, automatically converted to flags. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + import dataclasses + from typing import Optional + + import dcargs + + + @dataclasses.dataclass + class Args: + # Boolean. This expects an explicit "True" or "False". + boolean: bool + + # Optional boolean. Same as above, but can be omitted. + optional_boolean: Optional[bool] = None + + # Pass --flag-a in to set this value to True. + flag_a: bool = False + + # Pass --no-flag-b in to set this value to False. + flag_b: bool = True + + + if __name__ == "__main__": + args = dcargs.cli(Args) + print(args) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/04_flags.py --help + +.. command-output:: python ../../examples/04_flags.py --boolean True + +.. command-output:: python ../../examples/04_flags.py --boolean False --flag-a + +.. command-output:: python ../../examples/04_flags.py --boolean False --no-flag-b diff --git a/docs/source/example_05.rst b/docs/source/example_05.rst new file mode 100644 index 000000000..dde67bdc2 --- /dev/null +++ b/docs/source/example_05.rst @@ -0,0 +1,89 @@ +5. Hierarchical Configs +========================================== + +Parsing of nested types (in this case nested dataclasses) enables hierarchical +configuration objects that are both modular and highly expressive. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + import dataclasses + import enum + import pathlib + + import dcargs + + + class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + + @dataclasses.dataclass(frozen=True) + class OptimizerConfig: + # Gradient-based optimizer to use. + algorithm: OptimizerType = OptimizerType.ADAM + + # Learning rate to use. + learning_rate: float = 3e-4 + + # Coefficient for L2 regularization. + weight_decay: float = 1e-2 + + + @dataclasses.dataclass(frozen=True) + class ExperimentConfig: + # Various configurable options for our optimizer. + optimizer: OptimizerConfig + + # Batch size. + batch_size: int = 32 + + # Total number of training steps. + train_steps: int = 100_000 + + # Random seed. This is helpful for making sure that our experiments are all + # reproducible! + seed: int = 0 + + + def train( + out_dir: pathlib.Path, + config: ExperimentConfig, + restore_checkpoint: bool = False, + checkpoint_interval: int = 1000, + ) -> None: + """Train a model. + + Args: + out_dir: Where to save logs and checkpoints. + config: Experiment configuration. + restore_checkpoint: Set to restore an existing checkpoint. + checkpoint_interval: Training steps between each checkpoint save. + """ + print(f"{out_dir=}, {restore_checkpoint=}, {checkpoint_interval=}") + print(f"{config=}") + print(dcargs.to_yaml(config)) + + + if __name__ == "__main__": + dcargs.cli(train) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/05_hierarchical_configs.py --help + +.. command-output:: python ../../examples/05_hierarchical_configs.py --out-dir . --config.optimizer.algorithm SGD + +.. command-output:: python ../../examples/05_hierarchical_configs.py --out-dir . --restore-checkpoint diff --git a/docs/source/example_06.rst b/docs/source/example_06.rst new file mode 100644 index 000000000..a3f95eca5 --- /dev/null +++ b/docs/source/example_06.rst @@ -0,0 +1,136 @@ +6. Base Configs +========================================== + +We can integrate `dcargs.cli()` into common configuration patterns: here, we select +one of multiple possible base configurations, and then use the CLI to either override +(existing) or fill in (missing) values. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + import sys + from dataclasses import dataclass + from typing import Callable, Dict, Literal, Tuple, TypeVar, Union + + from torch import nn + + import dcargs + + + @dataclass(frozen=True) + class AdamOptimizer: + learning_rate: float = 1e-3 + betas: Tuple[float, float] = (0.9, 0.999) + + + @dataclass(frozen=True) + class SgdOptimizer: + learning_rate: float = 3e-4 + + + @dataclass(frozen=True) + class ExperimentConfig: + # Dataset to run experiment on. + dataset: Literal["mnist", "imagenet-50"] + + # Optimizer parameters. + optimizer: Union[AdamOptimizer, SgdOptimizer] + + # Model size. + num_layers: int + units: int + + # Batch size. + batch_size: int + + # Total number of training steps. + train_steps: int + + # Random seed. This is helpful for making sure that our experiments are all + # reproducible! + seed: int + + # Activation to use. Not specifiable via the commandline. + activation: Callable[[], nn.Module] + + + # Note that we could also define this library using separate YAML files (similar to + # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type + # checking + IDE support. + base_configs = { + "small": ExperimentConfig( + dataset="mnist", + optimizer=SgdOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + # The dcargs.MISSING sentinel allows us to specify that the seed should have no + # default, and needs to be populated from the CLI. + seed=dcargs.MISSING, + activation=nn.ReLU, + ), + "big": ExperimentConfig( + dataset="imagenet-50", + optimizer=AdamOptimizer(), + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=dcargs.MISSING, + activation=nn.GELU, + ), + } + + + T = TypeVar("T") + + + def cli_from_base_configs(base_library: Dict[str, T]) -> T: + """Populate an instance of `cls`, where the first positional argument is used to + select from a library of named base configs.""" + # Get base configuration name from the first positional argument. + if len(sys.argv) < 2 or sys.argv[1] not in base_library: + valid_usages = map(lambda k: f"{sys.argv[0]} {k} --help", base_library.keys()) + raise SystemExit("usage:\n " + "\n ".join(valid_usages)) + + # Get base configuration from our library, and use it for default CLI parameters. + default_instance = base_library[sys.argv[1]] + return dcargs.cli( + type(default_instance), + prog=" ".join(sys.argv[:2]), + args=sys.argv[2:], + default_instance=default_instance, + # `avoid_subparsers` will avoid making a subparser for unions when a default is + # provided; in this case, it simplifies our CLI but makes it less expressive + # (cannot switch away from the base optimizer types). + avoid_subparsers=True, + ) + + + if __name__ == "__main__": + config = cli_from_base_configs(base_configs) + print(config) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/06_base_configs.py + +.. command-output:: python ../../examples/06_base_configs.py small --help + +.. command-output:: python ../../examples/06_base_configs.py small --seed 94720 + +.. command-output:: python ../../examples/06_base_configs.py big --help + +.. command-output:: python ../../examples/06_base_configs.py big --seed 94720 diff --git a/docs/source/example_07.rst b/docs/source/example_07.rst new file mode 100644 index 000000000..1470950ed --- /dev/null +++ b/docs/source/example_07.rst @@ -0,0 +1,63 @@ +7. Literals And Unions +========================================== + +`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices; +`typing.Union[]` can be used to restrict inputs to a fixed set of types. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + import dataclasses + import enum + from typing import Literal, Optional, Tuple, Union + + import dcargs + + + class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + + @dataclasses.dataclass(frozen=True) + class Args: + # We can use Literal[] to restrict the set of allowable inputs, for example, over + # enums. + restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED + + # Literals can also be marked Optional. + integer: Optional[Literal[0, 1, 2, 3]] = None + + # Unions can be used to specify multiple allowable types. + union_over_types: Union[int, str] = 0 + string_or_enum: Union[Literal["red", "green"], Color] = "red" + + # Unions also work over more complex nested types. + union_over_tuples: Union[Tuple[int, int], Tuple[str]] = ("1",) + + # And can be nested in other types. + tuple_of_string_or_enum: Tuple[Union[Literal["red", "green"], Color], ...] = ( + "red", + Color.RED, + ) + + + if __name__ == "__main__": + args = dcargs.cli(Args) + print(args) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/07_literals_and_unions.py --help diff --git a/docs/source/example_08.rst b/docs/source/example_08.rst new file mode 100644 index 000000000..eda69595d --- /dev/null +++ b/docs/source/example_08.rst @@ -0,0 +1,77 @@ +8. Positional Args +========================================== + +Positional-only arguments in functions are converted to positional CLI arguments. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + from __future__ import annotations + + import dataclasses + import enum + import pathlib + from typing import Tuple + + import dcargs + + + def main( + source: pathlib.Path, + dest: pathlib.Path, + /, # Mark the end of positional arguments. + optimizer: OptimizerConfig, + force: bool = False, + verbose: bool = False, + background_rgb: Tuple[float, float, float] = (1.0, 0.0, 0.0), + ) -> None: + """Command-line interface defined using a function signature. Note that this + docstring is parsed to generate helptext. + + Args: + source: Source path. + dest: Destination path. + optimizer: Configuration for our optimizer object. + force: Do not prompt before overwriting. + verbose: Explain what is being done. + background_rgb: Background color. Red by default. + """ + print(f"{source=}\n{dest=}\n{optimizer=}\n{force=}\n{verbose=}\n{background_rgb=}") + + + class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + + @dataclasses.dataclass(frozen=True) + class OptimizerConfig: + algorithm: OptimizerType = OptimizerType.ADAM + """Gradient-based optimizer to use.""" + + learning_rate: float = 3e-4 + """Learning rate to use.""" + + weight_decay: float = 1e-2 + """Coefficient for L2 regularization.""" + + + if __name__ == "__main__": + dcargs.cli(main) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/08_positional_args.py --help + +.. command-output:: python ../../examples/08_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 diff --git a/docs/source/example_09.rst b/docs/source/example_09.rst new file mode 100644 index 000000000..7a7e8a997 --- /dev/null +++ b/docs/source/example_09.rst @@ -0,0 +1,60 @@ +9. Subparsers +========================================== + +Unions over nested types (classes or dataclasses) are populated using subparsers. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + from __future__ import annotations + + import dataclasses + from typing import Union + + import dcargs + + + @dataclasses.dataclass(frozen=True) + class Checkout: + """Checkout a branch.""" + + branch: str + + + @dataclasses.dataclass(frozen=True) + class Commit: + """Commit changes.""" + + message: str + all: bool = False + + + def main(cmd: Union[Checkout, Commit]) -> None: + print(cmd) + + + if __name__ == "__main__": + dcargs.cli(main) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/09_subparsers.py --help + +.. command-output:: python ../../examples/09_subparsers.py commit --help + +.. command-output:: python ../../examples/09_subparsers.py commit --cmd.message hello --cmd.all + +.. command-output:: python ../../examples/09_subparsers.py checkout --help + +.. command-output:: python ../../examples/09_subparsers.py checkout --cmd.branch main diff --git a/docs/source/example_10.rst b/docs/source/example_10.rst new file mode 100644 index 000000000..86817b884 --- /dev/null +++ b/docs/source/example_10.rst @@ -0,0 +1,87 @@ +10. Multiple Subparsers +========================================== + +Multiple unions over nested types are populated using a series of subparsers. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + from __future__ import annotations + + import dataclasses + from typing import Literal, Tuple, Union + + import dcargs + + # Possible dataset configurations. + + + @dataclasses.dataclass + class MnistDataset: + binary: bool = False + """Set to load binary version of MNIST dataset.""" + + + @dataclasses.dataclass + class ImageNetDataset: + subset: Literal[50, 100, 1000] + """Choose between ImageNet-50, ImageNet-100, ImageNet-1000, etc.""" + + + # Possible optimizer configurations. + + + @dataclasses.dataclass + class AdamOptimizer: + learning_rate: float = 1e-3 + betas: Tuple[float, float] = (0.9, 0.999) + + + @dataclasses.dataclass + class SgdOptimizer: + learning_rate: float = 3e-4 + + + # Train script. + + + def train( + dataset: Union[MnistDataset, ImageNetDataset] = MnistDataset(), + optimizer: Union[AdamOptimizer, SgdOptimizer] = AdamOptimizer(), + ) -> None: + """Example training script. + + Args: + dataset: Dataset to train on. + optimizer: Optimizer to train with. + + Returns: + None: + """ + print(dataset) + print(optimizer) + + + if __name__ == "__main__": + dcargs.cli(train) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/10_multiple_subparsers.py + +.. command-output:: python ../../examples/10_multiple_subparsers.py --help + +.. command-output:: python ../../examples/10_multiple_subparsers.py mnist-dataset --help + +.. command-output:: python ../../examples/10_multiple_subparsers.py mnist-dataset adam-optimizer --optimizer.learning-rate 3e-4 diff --git a/docs/source/example_11.rst b/docs/source/example_11.rst new file mode 100644 index 000000000..e1da89f7f --- /dev/null +++ b/docs/source/example_11.rst @@ -0,0 +1,58 @@ +11. Dictionaries +========================================== + +Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation, +or a `TypedDict` type. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + from typing import Dict, Tuple, TypedDict + + import dcargs + + + class DictionarySchema( + TypedDict, + # Setting `total=False` specifies that not all keys need to exist. + total=False, + ): + learning_rate: float + betas: Tuple[float, float] + + + def main( + typed_dict: DictionarySchema, + standard_dict: Dict[str, float] = { + "learning_rate": 3e-4, + "beta1": 0.9, + "beta2": 0.999, + }, + ) -> None: + assert isinstance(standard_dict, dict) + assert isinstance(typed_dict, dict) + print("Standard dict:", standard_dict) + print("Typed dict:", typed_dict) + + + if __name__ == "__main__": + dcargs.cli(main) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/11_dictionaries.py --help + +.. command-output:: python ../../examples/11_dictionaries.py --typed-dict.learning-rate 3e-4 + +.. command-output:: python ../../examples/11_dictionaries.py --typed-dict.betas 0.9 0.999 diff --git a/docs/source/example_12.rst b/docs/source/example_12.rst new file mode 100644 index 000000000..a4e41ebf1 --- /dev/null +++ b/docs/source/example_12.rst @@ -0,0 +1,43 @@ +12. Named Tuples +========================================== + +Example using `dcargs.cli()` to instantiate a named tuple. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + from typing import NamedTuple + + import dcargs + + + class TupleType(NamedTuple): + """Description. + This should show up in the helptext!""" + + field1: str # A string field. + field2: int = 3 # A numeric field, with a default value. + flag: bool = False # A boolean flag. + + + if __name__ == "__main__": + x = dcargs.cli(TupleType) + assert isinstance(x, tuple) + print(x) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/12_named_tuples.py --help + +.. command-output:: python ../../examples/12_named_tuples.py --field1 hello diff --git a/docs/source/example_13.rst b/docs/source/example_13.rst new file mode 100644 index 000000000..f3615d7a7 --- /dev/null +++ b/docs/source/example_13.rst @@ -0,0 +1,49 @@ +13. Standard Classes +========================================== + +In addition to functions and dataclasses, we can also generate CLIs from (the +constructors of) standard Python classes. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + import dcargs + + + class Args: + def __init__( + self, + field1: str, + field2: int, + flag: bool = False, + ): + """Arguments. + + Args: + field1: A string field. + field2: A numeric field. + flag: A boolean flag. + """ + self.data = [field1, field2, flag] + + + if __name__ == "__main__": + args = dcargs.cli(Args) + print(args.data) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/13_standard_classes.py --help + +.. command-output:: python ../../examples/13_standard_classes.py --field1 hello --field2 7 diff --git a/docs/source/example_14.rst b/docs/source/example_14.rst new file mode 100644 index 000000000..a0f6935a7 --- /dev/null +++ b/docs/source/example_14.rst @@ -0,0 +1,57 @@ +14. Generics +========================================== + +Example of parsing for generic dataclasses. + + + + +Example +------------------------------------------ + + + +.. code-block:: python + :linenos: + + import dataclasses + from typing import Generic, TypeVar + + import dcargs + + ScalarType = TypeVar("ScalarType") + ShapeType = TypeVar("ShapeType") + + + @dataclasses.dataclass(frozen=True) + class Point3(Generic[ScalarType]): + x: ScalarType + y: ScalarType + z: ScalarType + frame_id: str + + + @dataclasses.dataclass(frozen=True) + class Triangle: + a: Point3[float] + b: Point3[float] + c: Point3[float] + + + @dataclasses.dataclass(frozen=True) + class Args(Generic[ShapeType]): + point_continuous: Point3[float] + point_discrete: Point3[int] + shape: ShapeType + + + if __name__ == "__main__": + args = dcargs.cli(Args[Triangle]) + print(args) + + + +Usage +------------------------------------------ + +.. command-output:: python ../../examples/14_generics.py --help diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 000000000..04f58f62a --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,115 @@ +dcargs +========================================== + +|build| |nbsp| |mypy| |nbsp| |lint| |nbsp| |coverage| |nbsp| |versions| + +:code:`dcargs` is a library for typed CLI interfaces and configuration objects. + +Our core interface, :func:`dcargs.cli`, generates argument parsers from +type-annotated callables. In the simplest case, this can be used as a drop-in +replacement for :code:`argparse`: + + +.. code:: python + + # With argparse. + + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--a", type=int, required=True) + parser.add_argument("--b", type=int, default=3) + args = parser.parse_args() + + print(args.a + args.b) + + +.. code:: python + + # With dcargs. + + import dcargs + + def main(a: int, b: int = 3) -> None: + print(a + b) + + dcargs.cli(main) + + + +The broader goal is also a replacement for tools like :code:`hydra`, +:code:`gin-config`, and :code:`ml_collections` that's: + +- **Low effort.** Standard Python type annotations, docstrings, and default + values are parsed to automatically generate command-line interfaces with + informative helptext. + +- **Expressive.** :func:`dcargs.cli` understands functions, classes, + dataclasses, and *nested* classes and dataclasses, as well as frequently used + annotations like unions, literals, and collections, which can be composed into + hierarchical configuration objects built on standard Python features. + +- **Typed.** Unlike dynamic configuration namespaces produced by libraries like + :code:`argparse`, :code:`YACS`, :code:`abseil`, :code:`hydra`, or + :code:`ml_collections`, typed outputs mean that IDE-assisted autocomplete, + rename, refactor, and go-to-definition operations work out-of-the-box, as well + as static checking tools like :code:`mypy` and :code:`pyright`. + +- **Modular.** Most approaches to configuration objects require a centralized + definition of all configurable fields. Hierarchically nesting configuration + structures, however, makes it easy to distribute definitions, defaults, and + documentation of configurable fields across modules or source files. A model + configuration dataclass, for example, can be co-located in its entirety with + the model implementation and dropped into any experiment configuration with an + import — this eliminates redundancy and makes entire modules easy to port + across codebases. + + +.. toctree:: + :caption: Examples + :maxdepth: 5 + :titlesonly: + :glob: + + example_* + + +.. toctree:: + :caption: Notes + :maxdepth: 5 + :titlesonly: + :glob: + + serialization + alternatives + + +.. toctree:: + :caption: API Reference + :maxdepth: 5 + :titlesonly: + + api/dcargs/index + + + +.. |build| image:: https://github.com/brentyi/dcargs/workflows/build/badge.svg + :alt: Build status icon + :target: https://github.com/brentyi/dcargs +.. |mypy| image:: https://github.com/brentyi/dcargs/workflows/mypy/badge.svg?branch=master + :alt: Mypy status icon + :target: https://github.com/brentyi/dcargs +.. |lint| image:: https://github.com/brentyi/dcargs/workflows/lint/badge.svg + :alt: Lint status icon + :target: https://github.com/brentyi/dcargs +.. |coverage| image:: https://codecov.io/gh/brentyi/dcargs/branch/master/graph/badge.svg + :alt: Test coverage status icon + :target: https://codecov.io/gh/brentyi/dcargs +.. |downloads| image:: https://pepy.tech/badge/dcargs + :alt: Download count icon + :target: https://pypi.org/project/dcargs/ +.. |versions| image:: https://img.shields.io/pypi/pyversions/dcargs + :alt: Version icon + :target: https://pypi.org/project/dcargs/ +.. |nbsp| unicode:: 0xA0 + :trim: diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst new file mode 100644 index 000000000..2ace9e32b --- /dev/null +++ b/docs/source/serialization.rst @@ -0,0 +1,22 @@ +Serialization +==================================== + +As a secondary feature aimed at enabling the use of :func:`dcargs.cli` for general +configuration use cases, we also introduce functions for human-readable +dataclass serialization: + +- :func:`dcargs.to_yaml` and :func:`dcargs.from_yaml` convert between YAML-style + strings and dataclass instances. + +The functions attempt to strike a balance between flexibility and robustness — +in contrast to naively dumping or loading dataclass instances (via pickle, +PyYAML, etc), explicit type references enable custom tags that are robust +against code reorganization and refactor, while a PyYAML backend enables +serialization of arbitrary Python objects. + +Note that we generally prefer to use YAML purely for serialization, as opposed +to a configuration interface that humans are expected to manually write or +modify. Specifying things like loadable base configurations can be done directly +in Python, which enables all of the usual autocompletion and type checking +features. + diff --git a/docs/update_example_docs.py b/docs/update_example_docs.py new file mode 100644 index 000000000..d671179f7 --- /dev/null +++ b/docs/update_example_docs.py @@ -0,0 +1,119 @@ +"""Helper script for updating the auto-generated examples pages in the documentation.""" + +from __future__ import annotations + +import dataclasses +import pathlib +import shlex +from typing import Iterable + +import dcargs + + +@dataclasses.dataclass +class ExampleMetadata: + index: str + index_with_zero: str + source: str + title: str + usages: Iterable[str] + description: str + + @staticmethod + def from_path(path: pathlib.Path) -> ExampleMetadata: + # 01_functions -> 01, _, functions. + index, _, title = path.stem.partition("_") + + # 01 -> 1. + index_with_zero = index + index = str(int(index)) + + # functions -> Functions. + title = title.replace("_", " ").title() + + source = path.read_text().strip() + + docstring = source.split('"""')[1].strip() + assert "Usage:" in docstring + description, _, usage_text = docstring.partition("Usage:") + example_usages = map( + lambda x: x[1:-1], + filter( + lambda line: line.startswith("`") and line.endswith("`"), + usage_text.split("\n"), + ), + ) + return ExampleMetadata( + index=index, + index_with_zero=index_with_zero, + source=source[3:].partition('"""')[2].strip(), + title=title, + usages=example_usages, + description=description, + ) + + +def get_example_paths(examples_dir: pathlib.Path) -> Iterable[pathlib.Path]: + return filter( + lambda p: not p.name.startswith("_"), sorted(examples_dir.glob("*.py")) + ) + + +REPO_ROOT = pathlib.Path(__file__).absolute().parent.parent + + +def main( + examples_dir: pathlib.Path = REPO_ROOT / "examples", + sphinx_source_dir: pathlib.Path = REPO_ROOT / "docs" / "source", +) -> None: + for path in get_example_paths(examples_dir): + ex = ExampleMetadata.from_path(path) + path_for_sphinx = pathlib.Path("..") / ".." / path.relative_to(REPO_ROOT) + + usage_lines = [] + for usage in ex.usages: + args = shlex.split(usage) + python_index = args.index("python") + sphinx_usage = shlex.join( + args[:python_index] + + ["python", path_for_sphinx.as_posix()] + + args[python_index + 2 :] + ) + + usage_lines += [ + f".. command-output:: {sphinx_usage}", + "", + ] + + (sphinx_source_dir / f"example_{ex.index_with_zero}.rst").write_text( + "\n".join( + [ + f"{ex.index}. {ex.title}", + "==========================================", + "", + ex.description, + "", + "", + "Example", + "------------------------------------------", + "", + "", + "", + ".. code-block:: python", + " :linenos:", + "", + " " + "\n ".join(ex.source.split("\n")), + "", + "", + "", + "Usage", + "------------------------------------------", + "", + ] + + usage_lines + ) + ) + + +if __name__ == "__main__": + dcargs.cli(main, description=__doc__) diff --git a/examples/04_flags.py b/examples/04_flags.py index 8d2c9f749..233e5fb48 100644 --- a/examples/04_flags.py +++ b/examples/04_flags.py @@ -20,7 +20,7 @@ class Args: boolean: bool # Optional boolean. Same as above, but can be omitted. - optional_boolean: Optional[bool] + optional_boolean: Optional[bool] = None # Pass --flag-a in to set this value to True. flag_a: bool = False diff --git a/examples/05_hierarchical_configs.py b/examples/05_hierarchical_configs.py index cc70f856b..0850f787f 100644 --- a/examples/05_hierarchical_configs.py +++ b/examples/05_hierarchical_configs.py @@ -3,8 +3,8 @@ Usage: `python ./05_hierarchical_configs.py --help` -`python ./05_hierarchical_configs.py . --config.optimizer.algorithm SGD` -`python ./05_hierarchical_configs.py . --restore-checkpoint` +`python ./05_hierarchical_configs.py --out-dir . --config.optimizer.algorithm SGD` +`python ./05_hierarchical_configs.py --out-dir . --restore-checkpoint` """ import dataclasses diff --git a/setup.py b/setup.py index 07998abcb..14edcd354 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.1.8", + version="0.1.9", description="Strongly typed, zero-effort CLIs", long_description=long_description, long_description_content_type="text/markdown", diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index e5119804d..547e2b634 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -12,7 +12,7 @@ def _check_serialization_identity(cls: Type[T], instance: T) -> None: - assert dcargs.from_yaml(cls, dcargs.to_yaml(instance)) == instance + assert dcargs.extras.from_yaml(cls, dcargs.extras.to_yaml(instance)) == instance ScalarType = TypeVar("ScalarType") @@ -339,9 +339,11 @@ class TupleGenericVariableMissing(Generic[ScalarType]): xyz: Tuple[ScalarType, ...] x = TupleGenericVariableMissing[int](xyz=(dcargs.MISSING, dcargs.MISSING)) - assert dcargs.to_yaml(x).count("!missing") == 2 + assert dcargs.extras.to_yaml(x).count("!missing") == 2 _check_serialization_identity(TupleGenericVariableMissing[int], x) assert ( - dcargs.from_yaml(TupleGenericVariableMissing[int], dcargs.to_yaml(x)).xyz[0] + dcargs.extras.from_yaml( + TupleGenericVariableMissing[int], dcargs.extras.to_yaml(x) + ).xyz[0] is dcargs.MISSING ) From f522d320a3c146a5b7b4c0a6eab9824b45b6fb33 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 21 Jul 2022 01:33:59 -0700 Subject: [PATCH 066/491] Clean up docs --- README.md | 74 +++++++++++-------- dcargs/_calling.py | 1 + dcargs/_cli.py | 2 +- dcargs/_fields.py | 2 +- docs/source/example_01.rst | 46 ------------ docs/source/example_02.rst | 45 ----------- docs/source/examples/01_functions.rst | 57 ++++++++++++++ docs/source/examples/02_dataclasses.rst | 56 ++++++++++++++ .../03_enums and containers.rst} | 35 ++++++--- .../{example_04.rst => examples/04_flags.rst} | 41 +++++++--- .../05_hierarchical configs.rst} | 35 ++++++--- .../06_base configs.rst} | 55 ++++++++++---- .../07_literals and unions.rst} | 19 +++-- .../08_positional args.rst} | 25 ++++--- .../09_subparsers.rst} | 49 ++++++++---- .../10_multiple subparsers.rst} | 41 +++++++--- .../11_dictionaries.rst} | 35 ++++++--- .../12_named tuples.rst} | 25 ++++--- .../13_standard classes.rst} | 25 ++++--- .../14_generics.rst} | 17 ++--- docs/source/index.rst | 29 +++++--- docs/source/installation.rst | 37 ++++++++++ docs/source/serialization.rst | 4 +- docs/update_example_docs.py | 38 ++++++---- examples/03_enums_and_containers.py | 2 +- examples/05_hierarchical_configs.py | 4 +- examples/06_base_configs.py | 6 ++ examples/11_dictionaries.py | 2 +- 28 files changed, 521 insertions(+), 286 deletions(-) delete mode 100644 docs/source/example_01.rst delete mode 100644 docs/source/example_02.rst create mode 100644 docs/source/examples/01_functions.rst create mode 100644 docs/source/examples/02_dataclasses.rst rename docs/source/{example_03.rst => examples/03_enums and containers.rst} (69%) rename docs/source/{example_04.rst => examples/04_flags.rst} (57%) rename docs/source/{example_05.rst => examples/05_hierarchical configs.rst} (72%) rename docs/source/{example_06.rst => examples/06_base configs.rst} (74%) rename docs/source/{example_07.rst => examples/07_literals and unions.rst} (76%) rename docs/source/{example_08.rst => examples/08_positional args.rst} (81%) rename docs/source/{example_09.rst => examples/09_subparsers.rst} (50%) rename docs/source/{example_10.rst => examples/10_multiple subparsers.rst} (69%) rename docs/source/{example_11.rst => examples/11_dictionaries.rst} (58%) rename docs/source/{example_12.rst => examples/12_named tuples.rst} (57%) rename docs/source/{example_13.rst => examples/13_standard classes.rst} (66%) rename docs/source/{example_14.rst => examples/14_generics.rst} (80%) create mode 100644 docs/source/installation.rst diff --git a/README.md b/README.md index e80533399..df24b109c 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,45 @@ -# dcargs - -![build](https://github.com/brentyi/dcargs/workflows/build/badge.svg) -![mypy](https://github.com/brentyi/dcargs/workflows/mypy/badge.svg?branch=master) -![lint](https://github.com/brentyi/dcargs/workflows/lint/badge.svg) -[![codecov](https://codecov.io/gh/brentyi/dcargs/branch/master/graph/badge.svg)](https://codecov.io/gh/brentyi/dcargs) -[![PyPI Python Version][pypi-versions-badge]][pypi] - -[pypi-versions-badge]: https://img.shields.io/pypi/pyversions/dcargs -[pypi-badge]: https://badge.fury.io/py/dcargs.svg -[pypi]: https://pypi.org/project/dcargs/ - -**`dcargs`** is a library for typed CLI interfaces and configuration objects. - -``` -pip install dcargs -``` - -Our core interface generates argument parsers from type-annotated callables. In -the simplest case, this can be used as a drop-in replacement for `argparse`: - - +

dcargs

+ +

+ build + mypy + lint + + codecov + + + codecov + +

+ +

+ pip install dcargs +   •   + Documentation +

+ +
+ +

+ dcargs is a library for typed CLI interfaces + and configuration objects. +

+ +

+ Our core interface, dcargs.cli(), generates argument parsers from type-annotated +
callables: functions, classes, dataclasses, and nested dataclasses and classes. +

+ +

+ This can be used as a drop-in replacement for argparse: +

+ +
- - + + +
with argparsewith dcargswith argparsewith dcargs
@@ -62,9 +78,7 @@ dcargs.cli(main)
-The broader goal is to enable replacing configuration frameworks like `hydra`, -`gin-config`, and `ml_collections` with hierarchical structures built using -standard Python dataclasses and type annotations. - -For a full list of features and usage examples, see -[**our documentation**](https://brentyi.github.io/dcargs). +

+ For more sophisticated examples, see + our documentation. +

diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 6c57551cb..14cdc9396 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -109,6 +109,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: subparser_dest = _strings.SUBPARSER_DEST_FMT.format( name=prefixed_field_name ) + consumed_keywords.add(subparser_dest) if subparser_dest in value_from_prefixed_field_name: subparser_name = get_value_from_arg(subparser_dest) else: diff --git a/dcargs/_cli.py b/dcargs/_cli.py index c07f9c895..8c5ab8688 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -114,5 +114,5 @@ def cli( print(e.args[0]) raise SystemExit() - # assert consumed_keywords == value_from_prefixed_field_name.keys() + assert len(value_from_prefixed_field_name.keys() - consumed_keywords) == 0 return out diff --git a/dcargs/_fields.py b/dcargs/_fields.py index b7a384eed..ce56971b8 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -36,7 +36,7 @@ def init(self, *args, **kwds): pass -class PropagatingMissingType(_Singleton): # pragma: no cover +class PropagatingMissingType(_Singleton): pass diff --git a/docs/source/example_01.rst b/docs/source/example_01.rst deleted file mode 100644 index 868830584..000000000 --- a/docs/source/example_01.rst +++ /dev/null @@ -1,46 +0,0 @@ -1. Functions -========================================== - -In the simplest case, `dcargs.cli()` can be used to run a function with arguments -populated from the CLI. - - - - -Example ------------------------------------------- - - - -.. code-block:: python - :linenos: - - import dcargs - - - def main( - field1: str, - field2: int = 3, - ) -> None: - """Function, whose arguments will be populated from a CLI interface. - - Args: - field1: A string field. - field2: A numeric field, with a default value. - """ - print(field1, field2) - - - if __name__ == "__main__": - dcargs.cli(main) - - - -Usage ------------------------------------------- - -.. command-output:: python ../../examples/01_functions.py --help - -.. command-output:: python ../../examples/01_functions.py --field1 hello - -.. command-output:: python ../../examples/01_functions.py --field1 hello --field2 10 diff --git a/docs/source/example_02.rst b/docs/source/example_02.rst deleted file mode 100644 index 77c464812..000000000 --- a/docs/source/example_02.rst +++ /dev/null @@ -1,45 +0,0 @@ -2. Dataclasses -========================================== - -Common pattern: use `dcargs.cli()` to instantiate a dataclass. The outputted instance -can be used as a typed alternative for an argparse namespace. - - - - -Example ------------------------------------------- - - - -.. code-block:: python - :linenos: - - import dataclasses - - import dcargs - - - @dataclasses.dataclass - class Args: - """Description. - This should show up in the helptext!""" - - field1: str # A string field. - field2: int = 3 # A numeric field, with a default value. - - - if __name__ == "__main__": - args = dcargs.cli(Args) - print(args) - - - -Usage ------------------------------------------- - -.. command-output:: python ../../examples/02_dataclasses.py --help - -.. command-output:: python ../../examples/02_dataclasses.py --field1 hello - -.. command-output:: python ../../examples/02_dataclasses.py --field1 hello --field2 5 diff --git a/docs/source/examples/01_functions.rst b/docs/source/examples/01_functions.rst new file mode 100644 index 000000000..def4a7c8d --- /dev/null +++ b/docs/source/examples/01_functions.rst @@ -0,0 +1,57 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +1. Functions +========================================== + + +In the simplest case, ``dcargs.cli()`` can be used to run a function with arguments +populated from the CLI. + + + +.. code-block:: python + :linenos: + + import dcargs + + + def main( + field1: str, + field2: int = 3, + ) -> None: + """Function, whose arguments will be populated from a CLI interface. + + Args: + field1: A string field. + field2: A numeric field, with a default value. + """ + print(field1, field2) + + + if __name__ == "__main__": + dcargs.cli(main) + +------------ + +.. raw:: html + + python 01_functions.py --help + +.. program-output:: python ../../examples/01_functions.py --help + +------------ + +.. raw:: html + + python 01_functions.py --field1 hello + +.. program-output:: python ../../examples/01_functions.py --field1 hello + +------------ + +.. raw:: html + + python 01_functions.py --field1 hello --field2 10 + +.. program-output:: python ../../examples/01_functions.py --field1 hello --field2 10 diff --git a/docs/source/examples/02_dataclasses.rst b/docs/source/examples/02_dataclasses.rst new file mode 100644 index 000000000..42400276c --- /dev/null +++ b/docs/source/examples/02_dataclasses.rst @@ -0,0 +1,56 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +2. Dataclasses +========================================== + + +Common pattern: use ``dcargs.cli()`` to instantiate a dataclass. The outputted instance +can be used as a typed alternative for an argparse namespace. + + + +.. code-block:: python + :linenos: + + import dataclasses + + import dcargs + + + @dataclasses.dataclass + class Args: + """Description. + This should show up in the helptext!""" + + field1: str # A string field. + field2: int = 3 # A numeric field, with a default value. + + + if __name__ == "__main__": + args = dcargs.cli(Args) + print(args) + +------------ + +.. raw:: html + + python 02_dataclasses.py --help + +.. program-output:: python ../../examples/02_dataclasses.py --help + +------------ + +.. raw:: html + + python 02_dataclasses.py --field1 hello + +.. program-output:: python ../../examples/02_dataclasses.py --field1 hello + +------------ + +.. raw:: html + + python 02_dataclasses.py --field1 hello --field2 5 + +.. program-output:: python ../../examples/02_dataclasses.py --field1 hello --field2 5 diff --git a/docs/source/example_03.rst b/docs/source/examples/03_enums and containers.rst similarity index 69% rename from docs/source/example_03.rst rename to docs/source/examples/03_enums and containers.rst index 4410e12db..9d98a95d9 100644 --- a/docs/source/example_03.rst +++ b/docs/source/examples/03_enums and containers.rst @@ -1,14 +1,12 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + 3. Enums And Containers ========================================== -We can generate argument parsers from more advanced type annotations, like enums and -tuple types. - - - -Example ------------------------------------------- +We can generate argument parsers from more advanced type annotations, like enums and +container types. @@ -52,13 +50,26 @@ Example config = dcargs.cli(TrainConfig) print(config) +------------ + +.. raw:: html + + python 03_enums_and_containers.py --help + +.. program-output:: python ../../examples/03_enums_and_containers.py --help + +------------ + +.. raw:: html + + python 03_enums_and_containers.py --dataset-sources ./data --image-dimensions 16 16 +.. program-output:: python ../../examples/03_enums_and_containers.py --dataset-sources ./data --image-dimensions 16 16 -Usage ------------------------------------------- +------------ -.. command-output:: python ../../examples/03_enums_and_containers.py --help +.. raw:: html -.. command-output:: python ../../examples/03_enums_and_containers.py --dataset-sources ./data --image-dimensions 16 16 + python 03_enums_and_containers.py --dataset-sources ./data --optimizer-type SGD -.. command-output:: python ../../examples/03_enums_and_containers.py --dataset-sources ./data --optimizer-type SGD +.. program-output:: python ../../examples/03_enums_and_containers.py --dataset-sources ./data --optimizer-type SGD diff --git a/docs/source/example_04.rst b/docs/source/examples/04_flags.rst similarity index 57% rename from docs/source/example_04.rst rename to docs/source/examples/04_flags.rst index c04e1a87c..ceece83d1 100644 --- a/docs/source/example_04.rst +++ b/docs/source/examples/04_flags.rst @@ -1,17 +1,15 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + 4. Flags ========================================== + Booleans can either be expected to be explicitly passed in, or, if given a default value, automatically converted to flags. - -Example ------------------------------------------- - - - .. code-block:: python :linenos: @@ -40,15 +38,34 @@ Example args = dcargs.cli(Args) print(args) +------------ + +.. raw:: html + + python 04_flags.py --help + +.. program-output:: python ../../examples/04_flags.py --help + +------------ + +.. raw:: html + + python 04_flags.py --boolean True + +.. program-output:: python ../../examples/04_flags.py --boolean True + +------------ + +.. raw:: html + python 04_flags.py --boolean False --flag-a -Usage ------------------------------------------- +.. program-output:: python ../../examples/04_flags.py --boolean False --flag-a -.. command-output:: python ../../examples/04_flags.py --help +------------ -.. command-output:: python ../../examples/04_flags.py --boolean True +.. raw:: html -.. command-output:: python ../../examples/04_flags.py --boolean False --flag-a + python 04_flags.py --boolean False --no-flag-b -.. command-output:: python ../../examples/04_flags.py --boolean False --no-flag-b +.. program-output:: python ../../examples/04_flags.py --boolean False --no-flag-b diff --git a/docs/source/example_05.rst b/docs/source/examples/05_hierarchical configs.rst similarity index 72% rename from docs/source/example_05.rst rename to docs/source/examples/05_hierarchical configs.rst index dde67bdc2..fb87546f9 100644 --- a/docs/source/example_05.rst +++ b/docs/source/examples/05_hierarchical configs.rst @@ -1,14 +1,12 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + 5. Hierarchical Configs ========================================== -Parsing of nested types (in this case nested dataclasses) enables hierarchical -configuration objects that are both modular and highly expressive. - - - -Example ------------------------------------------- +Structures (typically dataclasses) can be nested to build hierarchical configuration +objects. This helps with modularity and grouping in larger projects. @@ -77,13 +75,26 @@ Example if __name__ == "__main__": dcargs.cli(train) +------------ + +.. raw:: html + + python 05_hierarchical_configs.py --help + +.. program-output:: python ../../examples/05_hierarchical_configs.py --help + +------------ + +.. raw:: html + + python 05_hierarchical_configs.py --out-dir . --config.optimizer.algorithm SGD +.. program-output:: python ../../examples/05_hierarchical_configs.py --out-dir . --config.optimizer.algorithm SGD -Usage ------------------------------------------- +------------ -.. command-output:: python ../../examples/05_hierarchical_configs.py --help +.. raw:: html -.. command-output:: python ../../examples/05_hierarchical_configs.py --out-dir . --config.optimizer.algorithm SGD + python 05_hierarchical_configs.py --out-dir . --restore-checkpoint -.. command-output:: python ../../examples/05_hierarchical_configs.py --out-dir . --restore-checkpoint +.. program-output:: python ../../examples/05_hierarchical_configs.py --out-dir . --restore-checkpoint diff --git a/docs/source/example_06.rst b/docs/source/examples/06_base configs.rst similarity index 74% rename from docs/source/example_06.rst rename to docs/source/examples/06_base configs.rst index a3f95eca5..f28c00416 100644 --- a/docs/source/example_06.rst +++ b/docs/source/examples/06_base configs.rst @@ -1,15 +1,19 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + 6. Base Configs ========================================== -We can integrate `dcargs.cli()` into common configuration patterns: here, we select + +We can integrate ``dcargs.cli()`` into common configuration patterns: here, we select one of multiple possible base configurations, and then use the CLI to either override (existing) or fill in (missing) values. - - - -Example ------------------------------------------- +Note that our interfaces don't prescribe any of the mechanics used for storing or +choosing between base configurations. A Hydra-style YAML approach could just as easily +be used for the config libary (although we generally prefer to avoid YAMLs; staying in +Python is convenient for autocompletion and type checking). For selection, we could also +avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. @@ -120,17 +124,42 @@ Example config = cli_from_base_configs(base_configs) print(config) +------------ + +.. raw:: html + + python 06_base_configs.py + +.. program-output:: python ../../examples/06_base_configs.py + +------------ + +.. raw:: html + + python 06_base_configs.py small --help + +.. program-output:: python ../../examples/06_base_configs.py small --help + +------------ + +.. raw:: html + + python 06_base_configs.py small --seed 94720 + +.. program-output:: python ../../examples/06_base_configs.py small --seed 94720 + +------------ +.. raw:: html -Usage ------------------------------------------- + python 06_base_configs.py big --help -.. command-output:: python ../../examples/06_base_configs.py +.. program-output:: python ../../examples/06_base_configs.py big --help -.. command-output:: python ../../examples/06_base_configs.py small --help +------------ -.. command-output:: python ../../examples/06_base_configs.py small --seed 94720 +.. raw:: html -.. command-output:: python ../../examples/06_base_configs.py big --help + python 06_base_configs.py big --seed 94720 -.. command-output:: python ../../examples/06_base_configs.py big --seed 94720 +.. program-output:: python ../../examples/06_base_configs.py big --seed 94720 diff --git a/docs/source/example_07.rst b/docs/source/examples/07_literals and unions.rst similarity index 76% rename from docs/source/example_07.rst rename to docs/source/examples/07_literals and unions.rst index 1470950ed..5d8e3b0a2 100644 --- a/docs/source/example_07.rst +++ b/docs/source/examples/07_literals and unions.rst @@ -1,14 +1,12 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + 7. Literals And Unions ========================================== -`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices; -`typing.Union[]` can be used to restrict inputs to a fixed set of types. - - - -Example ------------------------------------------- +``typing.Literal[]`` can be used to restrict inputs to a fixed set of literal choices; +``typing.Union[]`` can be used to restrict inputs to a fixed set of types. @@ -55,9 +53,10 @@ Example args = dcargs.cli(Args) print(args) +------------ +.. raw:: html -Usage ------------------------------------------- + python 07_literals_and_unions.py --help -.. command-output:: python ../../examples/07_literals_and_unions.py --help +.. program-output:: python ../../examples/07_literals_and_unions.py --help diff --git a/docs/source/example_08.rst b/docs/source/examples/08_positional args.rst similarity index 81% rename from docs/source/example_08.rst rename to docs/source/examples/08_positional args.rst index eda69595d..6dd19374a 100644 --- a/docs/source/example_08.rst +++ b/docs/source/examples/08_positional args.rst @@ -1,13 +1,11 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + 8. Positional Args ========================================== -Positional-only arguments in functions are converted to positional CLI arguments. - - - -Example ------------------------------------------- +Positional-only arguments in functions are converted to positional CLI arguments. @@ -67,11 +65,18 @@ Example if __name__ == "__main__": dcargs.cli(main) +------------ + +.. raw:: html + + python 08_positional_args.py --help + +.. program-output:: python ../../examples/08_positional_args.py --help +------------ -Usage ------------------------------------------- +.. raw:: html -.. command-output:: python ../../examples/08_positional_args.py --help + python 08_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 -.. command-output:: python ../../examples/08_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 +.. program-output:: python ../../examples/08_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 diff --git a/docs/source/example_09.rst b/docs/source/examples/09_subparsers.rst similarity index 50% rename from docs/source/example_09.rst rename to docs/source/examples/09_subparsers.rst index 7a7e8a997..c6fb8c5da 100644 --- a/docs/source/example_09.rst +++ b/docs/source/examples/09_subparsers.rst @@ -1,13 +1,11 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + 9. Subparsers ========================================== -Unions over nested types (classes or dataclasses) are populated using subparsers. - - - -Example ------------------------------------------- +Unions over nested types (classes or dataclasses) are populated using subparsers. @@ -44,17 +42,42 @@ Example if __name__ == "__main__": dcargs.cli(main) +------------ + +.. raw:: html + + python 09_subparsers.py --help + +.. program-output:: python ../../examples/09_subparsers.py --help + +------------ + +.. raw:: html + + python 09_subparsers.py commit --help + +.. program-output:: python ../../examples/09_subparsers.py commit --help + +------------ + +.. raw:: html + + python 09_subparsers.py commit --cmd.message hello --cmd.all + +.. program-output:: python ../../examples/09_subparsers.py commit --cmd.message hello --cmd.all + +------------ +.. raw:: html -Usage ------------------------------------------- + python 09_subparsers.py checkout --help -.. command-output:: python ../../examples/09_subparsers.py --help +.. program-output:: python ../../examples/09_subparsers.py checkout --help -.. command-output:: python ../../examples/09_subparsers.py commit --help +------------ -.. command-output:: python ../../examples/09_subparsers.py commit --cmd.message hello --cmd.all +.. raw:: html -.. command-output:: python ../../examples/09_subparsers.py checkout --help + python 09_subparsers.py checkout --cmd.branch main -.. command-output:: python ../../examples/09_subparsers.py checkout --cmd.branch main +.. program-output:: python ../../examples/09_subparsers.py checkout --cmd.branch main diff --git a/docs/source/example_10.rst b/docs/source/examples/10_multiple subparsers.rst similarity index 69% rename from docs/source/example_10.rst rename to docs/source/examples/10_multiple subparsers.rst index 86817b884..88ef9e2fd 100644 --- a/docs/source/example_10.rst +++ b/docs/source/examples/10_multiple subparsers.rst @@ -1,13 +1,11 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + 10. Multiple Subparsers ========================================== -Multiple unions over nested types are populated using a series of subparsers. - - - -Example ------------------------------------------- +Multiple unions over nested types are populated using a series of subparsers. @@ -73,15 +71,34 @@ Example if __name__ == "__main__": dcargs.cli(train) +------------ + +.. raw:: html + + python 10_multiple_subparsers.py + +.. program-output:: python ../../examples/10_multiple_subparsers.py + +------------ + +.. raw:: html + + python 10_multiple_subparsers.py --help + +.. program-output:: python ../../examples/10_multiple_subparsers.py --help + +------------ + +.. raw:: html + python 10_multiple_subparsers.py mnist-dataset --help -Usage ------------------------------------------- +.. program-output:: python ../../examples/10_multiple_subparsers.py mnist-dataset --help -.. command-output:: python ../../examples/10_multiple_subparsers.py +------------ -.. command-output:: python ../../examples/10_multiple_subparsers.py --help +.. raw:: html -.. command-output:: python ../../examples/10_multiple_subparsers.py mnist-dataset --help + python 10_multiple_subparsers.py mnist-dataset adam-optimizer --optimizer.learning-rate 3e-4 -.. command-output:: python ../../examples/10_multiple_subparsers.py mnist-dataset adam-optimizer --optimizer.learning-rate 3e-4 +.. program-output:: python ../../examples/10_multiple_subparsers.py mnist-dataset adam-optimizer --optimizer.learning-rate 3e-4 diff --git a/docs/source/example_11.rst b/docs/source/examples/11_dictionaries.rst similarity index 58% rename from docs/source/example_11.rst rename to docs/source/examples/11_dictionaries.rst index e1da89f7f..a2b681f8f 100644 --- a/docs/source/example_11.rst +++ b/docs/source/examples/11_dictionaries.rst @@ -1,14 +1,12 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + 11. Dictionaries ========================================== -Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation, -or a `TypedDict` type. - - - -Example ------------------------------------------- +Dictionary inputs can be specified using either a standard ``Dict[K, V]`` annotation, +or a ``TypedDict`` subclass. @@ -46,13 +44,26 @@ Example if __name__ == "__main__": dcargs.cli(main) +------------ + +.. raw:: html + + python 11_dictionaries.py --help + +.. program-output:: python ../../examples/11_dictionaries.py --help + +------------ + +.. raw:: html + + python 11_dictionaries.py --typed-dict.learning-rate 3e-4 +.. program-output:: python ../../examples/11_dictionaries.py --typed-dict.learning-rate 3e-4 -Usage ------------------------------------------- +------------ -.. command-output:: python ../../examples/11_dictionaries.py --help +.. raw:: html -.. command-output:: python ../../examples/11_dictionaries.py --typed-dict.learning-rate 3e-4 + python 11_dictionaries.py --typed-dict.betas 0.9 0.999 -.. command-output:: python ../../examples/11_dictionaries.py --typed-dict.betas 0.9 0.999 +.. program-output:: python ../../examples/11_dictionaries.py --typed-dict.betas 0.9 0.999 diff --git a/docs/source/example_12.rst b/docs/source/examples/12_named tuples.rst similarity index 57% rename from docs/source/example_12.rst rename to docs/source/examples/12_named tuples.rst index a4e41ebf1..9e65e21d9 100644 --- a/docs/source/example_12.rst +++ b/docs/source/examples/12_named tuples.rst @@ -1,13 +1,11 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + 12. Named Tuples ========================================== -Example using `dcargs.cli()` to instantiate a named tuple. - - - -Example ------------------------------------------- +Example using ``dcargs.cli()`` to instantiate a named tuple. @@ -33,11 +31,18 @@ Example assert isinstance(x, tuple) print(x) +------------ + +.. raw:: html + + python 12_named_tuples.py --help + +.. program-output:: python ../../examples/12_named_tuples.py --help +------------ -Usage ------------------------------------------- +.. raw:: html -.. command-output:: python ../../examples/12_named_tuples.py --help + python 12_named_tuples.py --field1 hello -.. command-output:: python ../../examples/12_named_tuples.py --field1 hello +.. program-output:: python ../../examples/12_named_tuples.py --field1 hello diff --git a/docs/source/example_13.rst b/docs/source/examples/13_standard classes.rst similarity index 66% rename from docs/source/example_13.rst rename to docs/source/examples/13_standard classes.rst index f3615d7a7..784c45aaa 100644 --- a/docs/source/example_13.rst +++ b/docs/source/examples/13_standard classes.rst @@ -1,17 +1,15 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + 13. Standard Classes ========================================== + In addition to functions and dataclasses, we can also generate CLIs from (the constructors of) standard Python classes. - -Example ------------------------------------------- - - - .. code-block:: python :linenos: @@ -39,11 +37,18 @@ Example args = dcargs.cli(Args) print(args.data) +------------ + +.. raw:: html + + python 13_standard_classes.py --help + +.. program-output:: python ../../examples/13_standard_classes.py --help +------------ -Usage ------------------------------------------- +.. raw:: html -.. command-output:: python ../../examples/13_standard_classes.py --help + python 13_standard_classes.py --field1 hello --field2 7 -.. command-output:: python ../../examples/13_standard_classes.py --field1 hello --field2 7 +.. program-output:: python ../../examples/13_standard_classes.py --field1 hello --field2 7 diff --git a/docs/source/example_14.rst b/docs/source/examples/14_generics.rst similarity index 80% rename from docs/source/example_14.rst rename to docs/source/examples/14_generics.rst index a0f6935a7..a65f6232b 100644 --- a/docs/source/example_14.rst +++ b/docs/source/examples/14_generics.rst @@ -1,13 +1,11 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + 14. Generics ========================================== -Example of parsing for generic dataclasses. - - - -Example ------------------------------------------- +Example of parsing for generic dataclasses. @@ -49,9 +47,10 @@ Example args = dcargs.cli(Args[Triangle]) print(args) +------------ +.. raw:: html -Usage ------------------------------------------- + python 14_generics.py --help -.. command-output:: python ../../examples/14_generics.py --help +.. program-output:: python ../../examples/14_generics.py --help diff --git a/docs/source/index.rst b/docs/source/index.rst index 04f58f62a..b8fbb573a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -5,12 +5,15 @@ dcargs :code:`dcargs` is a library for typed CLI interfaces and configuration objects. -Our core interface, :func:`dcargs.cli`, generates argument parsers from -type-annotated callables. In the simplest case, this can be used as a drop-in -replacement for :code:`argparse`: +Our core interface, :func:`dcargs.cli()`, generates argument parsers from +type-annotated callables: functions, classes, dataclasses, and *nested* +dataclasses and classes. +This can be used as a drop-in replacement for :code:`argparse`: -.. code:: python + + +.. code-block:: # With argparse. @@ -24,7 +27,7 @@ replacement for :code:`argparse`: print(args.a + args.b) -.. code:: python +.. code-block:: # With dcargs. @@ -64,20 +67,27 @@ The broader goal is also a replacement for tools like :code:`hydra`, import — this eliminates redundancy and makes entire modules easy to port across codebases. +.. toctree:: + :caption: Getting started + :maxdepth: 1 + :hidden: + :titlesonly: + + installation .. toctree:: :caption: Examples - :maxdepth: 5 + :maxdepth: 1 + :hidden: :titlesonly: :glob: - example_* - + examples/* .. toctree:: :caption: Notes :maxdepth: 5 - :titlesonly: + :hidden: :glob: serialization @@ -87,6 +97,7 @@ The broader goal is also a replacement for tools like :code:`hydra`, .. toctree:: :caption: API Reference :maxdepth: 5 + :hidden: :titlesonly: api/dcargs/index diff --git a/docs/source/installation.rst b/docs/source/installation.rst new file mode 100644 index 000000000..3980e2722 --- /dev/null +++ b/docs/source/installation.rst @@ -0,0 +1,37 @@ +Installation +========================================== + +Standard +------------------------------------------ + +Installation is supported on Python >=3.7 via pip. This is typically all that's +required. + + +.. code-block:: + + pip install dcargs + + +Development +------------------------------------------ + +If you're interested in development, :code:`dcargs` can also be installed from +source. + +.. code-block:: + + # Clone repository. + git clone git@github.com:brentyi/dcargs.git + cd dcargs + + # Install development dependencies. + pip install -e ".[testing,type-checking]" + + # Run tests. + pytest + + # Check types. + mypy --install-types . + + diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst index 2ace9e32b..6c47af1ef 100644 --- a/docs/source/serialization.rst +++ b/docs/source/serialization.rst @@ -5,8 +5,8 @@ As a secondary feature aimed at enabling the use of :func:`dcargs.cli` for gener configuration use cases, we also introduce functions for human-readable dataclass serialization: -- :func:`dcargs.to_yaml` and :func:`dcargs.from_yaml` convert between YAML-style - strings and dataclass instances. +- :func:`dcargs.extras.to_yaml` and :func:`dcargs.extras.from_yaml` convert + between YAML-style strings and dataclass instances. The functions attempt to strike a balance between flexibility and robustness — in contrast to naively dumping or loading dataclass instances (via pickle, diff --git a/docs/update_example_docs.py b/docs/update_example_docs.py index d671179f7..c0267b674 100644 --- a/docs/update_example_docs.py +++ b/docs/update_example_docs.py @@ -7,6 +7,8 @@ import shlex from typing import Iterable +import m2r2 + import dcargs @@ -80,23 +82,38 @@ def main( + args[python_index + 2 :] ) + # Note that :kbd: in Sphinx does unnecessary stuff we want to avoid, see: + # https://github.com/sphinx-doc/sphinx/issues/7530 + # + # Instead, we just use raw HTML. + assert "../../examples/" in sphinx_usage + command = sphinx_usage.replace("../../examples/", "") usage_lines += [ - f".. command-output:: {sphinx_usage}", + "------------", + "", + ".. raw:: html", + "", + f" {command}", + "", + f".. program-output:: {sphinx_usage}", "", ] - (sphinx_source_dir / f"example_{ex.index_with_zero}.rst").write_text( + ( + sphinx_source_dir + / "examples" + / f"{ex.index_with_zero}_{ex.title.lower()}.rst" + ).write_text( "\n".join( [ + ".. Comment: this file is automatically generated by" + " `update_example_docs.py`.", + " It should not be modified manually.", + "", f"{ex.index}. {ex.title}", "==========================================", "", - ex.description, - "", - "", - "Example", - "------------------------------------------", - "", + m2r2.convert(ex.description), "", "", ".. code-block:: python", @@ -104,11 +121,6 @@ def main( "", " " + "\n ".join(ex.source.split("\n")), "", - "", - "", - "Usage", - "------------------------------------------", - "", ] + usage_lines ) diff --git a/examples/03_enums_and_containers.py b/examples/03_enums_and_containers.py index 946ad4664..fe0facba4 100644 --- a/examples/03_enums_and_containers.py +++ b/examples/03_enums_and_containers.py @@ -1,5 +1,5 @@ """We can generate argument parsers from more advanced type annotations, like enums and -tuple types. +container types. Usage: `python ./03_enums_and_containers.py --help` diff --git a/examples/05_hierarchical_configs.py b/examples/05_hierarchical_configs.py index 0850f787f..d6a8f6520 100644 --- a/examples/05_hierarchical_configs.py +++ b/examples/05_hierarchical_configs.py @@ -1,5 +1,5 @@ -"""Parsing of nested types (in this case nested dataclasses) enables hierarchical -configuration objects that are both modular and highly expressive. +"""Structures (typically dataclasses) can be nested to build hierarchical configuration +objects. This helps with modularity and grouping in larger projects. Usage: `python ./05_hierarchical_configs.py --help` diff --git a/examples/06_base_configs.py b/examples/06_base_configs.py index 702c88798..0e721b5f0 100644 --- a/examples/06_base_configs.py +++ b/examples/06_base_configs.py @@ -2,6 +2,12 @@ one of multiple possible base configurations, and then use the CLI to either override (existing) or fill in (missing) values. +Note that our interfaces don't prescribe any of the mechanics used for storing or +choosing between base configurations. A Hydra-style YAML approach could just as easily +be used for the config libary (although we generally prefer to avoid YAMLs; staying in +Python is convenient for autocompletion and type checking). For selection, we could also +avoid fussing with `sys.argv` by using a `BASE_CONFIG` environment variable. + Usage: `python ./06_base_configs.py` `python ./06_base_configs.py small --help` diff --git a/examples/11_dictionaries.py b/examples/11_dictionaries.py index ef788b71f..0f91a0f31 100644 --- a/examples/11_dictionaries.py +++ b/examples/11_dictionaries.py @@ -1,5 +1,5 @@ """Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation, -or a `TypedDict` type. +or a `TypedDict` subclass. Usage: `python ./11_dictionaries.py --help` From 52d841c67a10bf34a5deb5aa69fec07cea60d901 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 21 Jul 2022 12:23:55 -0700 Subject: [PATCH 067/491] Update README.md --- README.md | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index df24b109c..1aa9a0a68 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ -

dcargs

+

dcargs

-

+

+ Documentation +   •   + pip install dcargs +

+

build mypy lint @@ -12,29 +17,21 @@

-

- pip install dcargs -   •   - Documentation -

- -
- -

+

dcargs is a library for typed CLI interfaces and configuration objects.

-

+

Our core interface, dcargs.cli(), generates argument parsers from type-annotated
callables: functions, classes, dataclasses, and nested dataclasses and classes.

-

+

This can be used as a drop-in replacement for argparse:

- +
@@ -78,7 +75,7 @@ dcargs.cli(main)
with argparse with dcargs
-

+

For more sophisticated examples, see our documentation.

From 0ff29605f11cd7d6176747e13956b7cb19b80fae Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 21 Jul 2022 15:11:34 -0700 Subject: [PATCH 068/491] Add py.typed --- dcargs/py.typed | 0 setup.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 dcargs/py.typed diff --git a/dcargs/py.typed b/dcargs/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/setup.py b/setup.py index 14edcd354..340d9de2a 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="dcargs", - version="0.1.9", + version="0.1.10", description="Strongly typed, zero-effort CLIs", long_description=long_description, long_description_content_type="text/markdown", From 191555fbf31a868c95e5c85f503cb22393d06de8 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 23 Jul 2022 00:49:54 -0700 Subject: [PATCH 069/491] Support mixed-type literals --- dcargs/_instantiators.py | 44 ++++++++++--------- docs/source/conf.py | 12 +++++ .../examples/07_literals and unions.rst | 3 ++ examples/07_literals_and_unions.py | 3 ++ tests/test_dcargs.py | 10 +++++ tests/test_errors.py | 9 ---- 6 files changed, 52 insertions(+), 29 deletions(-) diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index 37627703f..e6a09eee8 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -609,25 +609,29 @@ def _instantiator_from_literal( typ: Type, type_from_typevar: Dict[TypeVar, Type] ) -> Tuple[Instantiator, InstantiatorMetadata]: choices = get_args(typ) - if not len(set(map(type, choices))) == 1: - raise UnsupportedTypeAnnotationError( - "All choices in literal must have the same type!" + str_choices = tuple(x.name if isinstance(x, enum.Enum) else str(x) for x in choices) + + def instantiator(string: str) -> Any: + # Any situation where string is not in str_choices should be caught earlier. + assert string in str_choices + inner_type = type(choices[str_choices.index(string)]) + inner_instantiator, metadata = _instantiator_from_type_inner( + inner_type, + type_from_typevar, + allow_sequences=False, ) - contained_type = type(next(iter(choices))) - if issubclass(contained_type, enum.Enum): - choices = tuple(map(lambda x: x.name, choices)) - else: - choices = tuple(map(str, choices)) - instantiator, metadata = _instantiator_from_type_inner( - contained_type, - type_from_typevar, - allow_sequences=False, - ) - if metadata.choices is not None: - assert all(map(lambda t: isinstance(t, str), metadata.choices)) - assert len(set(choices) - set(metadata.choices)) == 0 - return instantiator, dataclasses.replace( - metadata, - choices=choices, - metavar="{" + ",".join(map(_format_metavar, map(str, choices))) + "}", + + # These should pass for all valid Literal types. + if inner_type is bool or issubclass(inner_type, enum.Enum): + assert metadata.choices is not None + else: + assert metadata.choices is None + assert metadata.nargs is None + + return inner_instantiator(string) + + return instantiator, InstantiatorMetadata( + nargs=None, + metavar="{" + ",".join(map(_format_metavar, str_choices)) + "}", + choices=str_choices, ) diff --git a/docs/source/conf.py b/docs/source/conf.py index f0ca66668..6173128eb 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -64,6 +64,18 @@ "color-code-background": "#f4f4f4", "color-code-foreground": "#000", }, + "footer_icons": [ + { + "name": "GitHub", + "url": "https://github.com/brentyi/dcargs", + "html": """ + + + + """, + "class": "", + }, + ], } # Pull documentation types from hints diff --git a/docs/source/examples/07_literals and unions.rst b/docs/source/examples/07_literals and unions.rst index 5d8e3b0a2..4a616d7d4 100644 --- a/docs/source/examples/07_literals and unions.rst +++ b/docs/source/examples/07_literals and unions.rst @@ -32,6 +32,9 @@ # enums. restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED + # Mixed types are okay. + mixed: Literal[Color.RED, Color.GREEN, "blue"] = "blue" + # Literals can also be marked Optional. integer: Optional[Literal[0, 1, 2, 3]] = None diff --git a/examples/07_literals_and_unions.py b/examples/07_literals_and_unions.py index ea3300837..b8231ae11 100644 --- a/examples/07_literals_and_unions.py +++ b/examples/07_literals_and_unions.py @@ -24,6 +24,9 @@ class Args: # enums. restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED + # Or mix them with other types! + mixed: Literal[Color.RED, Color.GREEN, "blue"] = "blue" + # Literals can also be marked Optional. integer: Optional[Literal[0, 1, 2, 3]] = None diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index adb2aecc0..a5312d023 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -328,6 +328,16 @@ class A: assert dcargs.cli(A, args=[]) == A(x=None) +def test_multitype_literal(): + def main(x: Literal[0, "5"]) -> Any: + return x + + assert dcargs.cli(main, args=["--x", "0"]) == 0 + assert dcargs.cli(main, args=["--x", "5"]) == "5" + with pytest.raises(SystemExit): + dcargs.cli(main, args=["--x", "6"]) + + def test_annotated(): """Annotated[] is a no-op.""" diff --git a/tests/test_errors.py b/tests/test_errors.py index 240370292..aa6b88277 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -2,7 +2,6 @@ from typing import Generic, List, Tuple, TypeVar, Union import pytest -from typing_extensions import Literal import dcargs @@ -32,14 +31,6 @@ def main(x: List[Union[Tuple[int, int], Tuple[int, int, int]]]) -> None: dcargs.cli(main, args=["--help"]) -def test_unsupported_literal(): - def main(x: Literal[0, "5"]) -> None: - return - - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main, args=[]) - - # Must be global. @dataclasses.dataclass class _CycleDataclass: From e8f09df6af8445bde95a84229666bc3c35b859b3 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 23 Jul 2022 21:30:12 -0700 Subject: [PATCH 070/491] Proximal step : ' ) --- dcargs/_arguments.py | 7 +- dcargs/_calling.py | 12 ++- dcargs/_instantiators.py | 208 +++++++++++++++------------------------ 3 files changed, 90 insertions(+), 137 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 571c87d95..e5eb1bdf0 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -221,9 +221,6 @@ def as_str(x: Any) -> Tuple[str, ...]: or lowered.action is not None ): return lowered - elif lowered.nargs is None: - (str_default,) = as_str(lowered.default) - return dataclasses.replace(lowered, default=str_default) else: return dataclasses.replace(lowered, default=as_str(lowered.default)) @@ -316,7 +313,9 @@ def _rule_positional_special_handling( nargs = lowered.nargs else: metavar = "[" + metavar + "]" - if lowered.nargs is None: + if lowered.nargs == 1: + # Optional positional arguments. Note that this needs to be special-cased in + # _calling.py. nargs = "?" else: # If lowered.nargs is either + or an int. diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 14cdc9396..f6c2ad8df 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -65,7 +65,15 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: consumed_keywords.add(prefixed_field_name) if not arg.lowered.is_fixed(): value = get_value_from_arg(prefixed_field_name) - if value not in _fields.MISSING_SINGLETONS: + + if value in _fields.MISSING_SINGLETONS: + value = arg.field.default + else: + if arg.lowered.nargs == "?": + # Special case for optional positional arguments: this is the + # only time that arguments don't come back as a list. + value = [value] + try: assert arg.lowered.instantiator is not None value = arg.lowered.instantiator(value) @@ -73,8 +81,6 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: raise InstantiationError( f"Parsing error for {arg.lowered.name_or_flag}: {e.args[0]}" ) - else: - value = arg.field.default else: assert arg.field.default not in _fields.MISSING_SINGLETONS value = arg.field.default diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index e6a09eee8..b2482cb4c 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -1,11 +1,11 @@ """Helper for using type annotations to recursively generate instantiator functions, -which map strings (or, in some cases, sequences of strings) to the annotated type. +which map sequences of strings to the annotated type. Some examples of type annotations and the desired instantiators: ``` int - lambda string: int(str) + lambda strings: int(str[0]) List[int] @@ -32,7 +32,6 @@ ) ``` """ - import collections.abc import dataclasses import enum @@ -57,28 +56,31 @@ import termcolor from typing_extensions import Annotated, Final, Literal, get_args, get_origin -# Most standard fields: these are converted from strings from the CLI. -_StandardInstantiator = Callable[[str], Any] -# Sequence fields! This should be used whenever argparse's `nargs` field is set. -_SequenceInstantiator = Callable[[List[str]], Any] +_StandardInstantiator = Callable[[List[str]], Any] # Special case: the only time that argparse doesn't give us a string is when the # argument action is set to `store_true` or `store_false`. In this case, we get # a bool directly, and the field action can be a no-op. _FlagInstantiator = Callable[[bool], bool] -Instantiator = Union[_StandardInstantiator, _SequenceInstantiator, _FlagInstantiator] +Instantiator = Union[_StandardInstantiator, _FlagInstantiator] NoneType = type(None) @dataclasses.dataclass class InstantiatorMetadata: - nargs: Union[None, int, Literal["+"]] - # Note: unlike in vanilla argparse, our metavar is always a string. We handle + # Unlike in vanilla argparse, we never set nargs to None. To make things simpler, we + # instead use nargs=1. + nargs: Union[int, Literal["+"]] + # Unlike in vanilla argparse, our metavar is always a string. We handle # sequences, multiple arguments, etc, manually. metavar: str choices: Optional[Tuple[str, ...]] + def check_choices(self, strings: List[str]) -> None: + if self.choices is not None and any(s not in self.choices for s in strings): + raise ValueError(f"invalid choice: {strings} (choose from {self.choices}))") + class UnsupportedTypeAnnotationError(Exception): """Exception raised when an unsupported type annotation is detected.""" @@ -121,14 +123,14 @@ def instantiator_from_type( # Handle NoneType. if typ is NoneType: - def instantiator(string: str) -> None: + def instantiator(strings: List[str]) -> None: # Note that other inputs should be caught by `choices` before the # instantiator runs. - assert string == "None" + assert strings == ["None"] return None return instantiator, InstantiatorMetadata( - nargs=None, + nargs=1, metavar="{" + _format_metavar("None") + "}", choices=("None",), ) @@ -187,7 +189,7 @@ def instantiator(string: str) -> None: elif isinstance(typ, type) and issubclass(typ, enum.Enum): auto_choices = tuple(x.name for x in typ) - def instantiator_base_case(string: str) -> Any: + def instantiator_base_case(strings: List[str]) -> Any: """Given a type and and a string from the command-line, reconstruct an object. Not intended to deal with containers. @@ -204,6 +206,7 @@ def instantiator_base_case(string: str) -> Any: ``` """ assert len(get_args(typ)) == 0, f"Type {typ} cannot be instantiated." + (string,) = strings if typ is bool: return {"True": True, "False": False}[string] # type: ignore elif isinstance(typ, type) and issubclass(typ, enum.Enum): @@ -214,7 +217,7 @@ def instantiator_base_case(string: str) -> Any: return typ(string) # type: ignore return instantiator_base_case, InstantiatorMetadata( - nargs=None, + nargs=1, metavar=_format_metavar(typ.__name__.upper()) if auto_choices is None else "{" + ",".join(map(_format_metavar, map(str, auto_choices))) + "}", @@ -258,11 +261,10 @@ def _instantiator_from_type_inner( errors.""" out = instantiator_from_type(typ, type_from_typevar) if out[1].nargs is not None: - if allow_sequences is False: - # We currently only use allow_sequences=False for options in Literal types, - # which are evaluated using `type()`. It should not be possible to hit this - # condition from polling a runtime type. - assert False + # We currently only use allow_sequences=False for options in Literal types, + # which are evaluated using `type()`. It should not be possible to hit this + # condition from polling a runtime type. + assert allow_sequences if allow_sequences == "fixed_length" and not isinstance(out[1].nargs, int): raise UnsupportedTypeAnnotationError( f"Found an unsupported (variable-length) nested sequence of type {typ}." @@ -320,8 +322,8 @@ def _instantiator_from_tuple( return _instantiator_from_sequence(typ, type_from_typevar) else: - instantiators = [] - metas = [] + instantiators: List[_StandardInstantiator] = [] + metas: List[InstantiatorMetadata] = [] nargs = 0 for t in types: a, b = _instantiator_from_type_inner( @@ -329,34 +331,22 @@ def _instantiator_from_tuple( type_from_typevar, allow_sequences="fixed_length", ) - instantiators.append(a) + instantiators.append(a) # type: ignore metas.append(b) - assert type(b.nargs) in (int, NoneType) - nargs += 1 if b.nargs is None else b.nargs # type: ignore + assert isinstance(b.nargs, int) + nargs += b.nargs def fixed_length_tuple_instantiator(strings: List[str]) -> Any: - # Validate nargs. - if len(strings) != nargs: - raise ValueError( - f"input {strings} is length {len(strings)}, but expected {nargs}." - ) + assert len(strings) == nargs # Make tuple. out = [] i = 0 for make, meta in zip(instantiators, metas): - inner_nargs = cast(Optional[int], meta.nargs) - if inner_nargs is None: - if meta.choices is not None and strings[i] not in meta.choices: - raise ValueError( - f" {strings[i]} does not match choices {meta.choices}" - ) - out.append(make(strings[i])) # type: ignore - i += 1 - else: - assert meta.choices is None - out.append(make(strings[i : i + inner_nargs])) # type: ignore - i += inner_nargs + assert isinstance(meta.nargs, int) + meta.check_choices(strings[i : i + meta.nargs]) + out.append(make(strings[i : i + meta.nargs])) + i += meta.nargs return tuple(out) return fixed_length_tuple_instantiator, InstantiatorMetadata( @@ -417,7 +407,7 @@ def _instantiator_from_union( # right. instantiators = [] metas = [] - nargs = None + nargs: Union[int, Literal["+"]] = 1 first = True for t in options: a, b = _instantiator_from_type_inner( @@ -441,56 +431,34 @@ def _instantiator_from_union( metavar: str metavar = _join_union_metavars(map(lambda x: cast(str, x.metavar), metas)) - def union_instantiator(string_or_strings: Union[str, List[str]]) -> Any: + def union_instantiator(strings: List[str]) -> Any: metadata: InstantiatorMetadata errors = [] for i, (instantiator, metadata) in enumerate(zip(instantiators, metas)): # Check choices. - if metadata.choices is not None and ( - ( - isinstance(string_or_strings, str) - and string_or_strings not in metadata.choices - ) - or ( - isinstance(string_or_strings, list) - and any(x not in metadata.choices for x in string_or_strings) - ) + if metadata.choices is not None and any( + x not in metadata.choices for x in strings ): errors.append( - f"{options[i]}: {string_or_strings} does not match choices" - f" {metadata.choices}" + f"{options[i]}: {strings} does not match choices {metadata.choices}" ) continue - # Try passing input directly into instantiator. - if metadata.nargs == nargs or ( - isinstance(metadata.nargs, int) and nargs == "+" - ): - try: - return instantiator(string_or_strings) # type: ignore - except ValueError as e: - # Failed, try next instantiator. - errors.append(f"{options[i]}: {e.args[0]}") - - # Try passing unwrapped length-1 input into instantiator. - elif ( - metadata.nargs is None - and nargs is not None - and len(string_or_strings) == 1 - ): + # Try passing input into instantiator. + if len(strings) == metadata.nargs or (metadata.nargs == "+"): try: - return instantiator(string_or_strings[0]) # type: ignore + return instantiator(strings) # type: ignore except ValueError as e: # Failed, try next instantiator. errors.append(f"{options[i]}: {e.args[0]}") else: errors.append( - f"{options[i]}: did not attempt," - f" {metadata} {nargs} {len(string_or_strings)}" + f"{options[i]}: input length {len(strings)} did not match expected" + f" argument count {metadata.nargs}" ) raise ValueError( f"no type in {options} could be instantiated from" - f" {string_or_strings}.\n\nGot errors: \n- " + "\n- ".join(errors) + f" {strings}.\n\nGot errors: \n- " + "\n- ".join(errors) ) return union_instantiator, InstantiatorMetadata( @@ -504,24 +472,22 @@ def _instantiator_from_dict( typ: Type, type_from_typevar: Dict[TypeVar, Type] ) -> Tuple[Instantiator, InstantiatorMetadata]: key_type, val_type = get_args(typ) - key_instantiator, key_metadata = _instantiator_from_type_inner( + key_instantiator, key_meta = _instantiator_from_type_inner( key_type, type_from_typevar, allow_sequences="fixed_length", ) - val_instantiator, val_metadata = _instantiator_from_type_inner( + val_instantiator, val_meta = _instantiator_from_type_inner( val_type, type_from_typevar, allow_sequences="fixed_length", ) - key_nargs = cast(Optional[int], key_metadata.nargs) - key_nargs_int = 1 if key_nargs is None else key_nargs - val_nargs = cast(Optional[int], val_metadata.nargs) - val_nargs_int = 1 if key_nargs is None else key_nargs - assert type(key_nargs) in (int, NoneType) - assert type(val_nargs) in (int, NoneType) - pair_nargs = key_nargs_int + val_nargs_int + key_nargs = cast(int, key_meta.nargs) # Casts needed for mypy but not pyright! + val_nargs = cast(int, val_meta.nargs) + assert isinstance(key_nargs, int) + assert isinstance(val_nargs, int) + pair_nargs = key_nargs + val_nargs def dict_instantiator(strings: List[str]) -> Any: out = {} @@ -530,27 +496,27 @@ def dict_instantiator(strings: List[str]) -> Any: index = 0 for _ in range(len(strings) // pair_nargs): - k: Union[str, List[str]] = strings[index : index + key_nargs_int] - index += key_nargs_int - v: Union[str, List[str]] = strings[index : index + val_nargs_int] - index += val_nargs_int - - if key_nargs is None: - (k,) = cast(List[str], k) - if key_metadata.choices is not None and k not in key_metadata.choices: - raise ValueError( - f"invalid choice: {k} (choose from {key_metadata.choices}))" - ) - if val_nargs is None: - (v,) = cast(List[str], v) - if val_metadata.choices is not None and v not in val_metadata.choices: - raise ValueError( - f"invalid choice: {v} (choose from {val_metadata.choices}))" - ) + k = strings[index : index + key_nargs] + index += key_nargs + v = strings[index : index + val_nargs] + index += val_nargs + + if key_meta.choices is not None and any( + kj not in key_meta.choices for kj in k + ): + raise ValueError( + f"invalid choice: {k} (choose from {key_meta.choices}))" + ) + if val_meta.choices is not None and any( + vj not in val_meta.choices for vj in v + ): + raise ValueError( + f"invalid choice: {v} (choose from {val_meta.choices}))" + ) out[key_instantiator(k)] = val_instantiator(v) # type: ignore return out - pair_metavar = f"{key_metadata.metavar} {val_metadata.metavar}" + pair_metavar = f"{key_meta.metavar} {val_meta.metavar}" return dict_instantiator, InstantiatorMetadata( nargs="+", metavar=f"{pair_metavar} [{pair_metavar} ...]", @@ -591,10 +557,7 @@ def sequence_instantiator(strings: List[str]) -> Any: out = [] step = inner_meta.nargs if isinstance(inner_meta.nargs, int) else 1 for i in range(0, len(strings), step): - if inner_meta.nargs is None: - out.append(make(strings[i])) # type: ignore - else: - out.append(make(strings[i : i + inner_meta.nargs])) # type: ignore + out.append(make(strings[i : i + inner_meta.nargs])) # type: ignore assert container_type is not None return container_type(out) @@ -610,28 +573,13 @@ def _instantiator_from_literal( ) -> Tuple[Instantiator, InstantiatorMetadata]: choices = get_args(typ) str_choices = tuple(x.name if isinstance(x, enum.Enum) else str(x) for x in choices) - - def instantiator(string: str) -> Any: - # Any situation where string is not in str_choices should be caught earlier. - assert string in str_choices - inner_type = type(choices[str_choices.index(string)]) - inner_instantiator, metadata = _instantiator_from_type_inner( - inner_type, - type_from_typevar, - allow_sequences=False, - ) - - # These should pass for all valid Literal types. - if inner_type is bool or issubclass(inner_type, enum.Enum): - assert metadata.choices is not None - else: - assert metadata.choices is None - assert metadata.nargs is None - - return inner_instantiator(string) - - return instantiator, InstantiatorMetadata( - nargs=None, - metavar="{" + ",".join(map(_format_metavar, str_choices)) + "}", - choices=str_choices, + return ( + # Note that if string is not in str_choices, it will be caught from setting + # `choices` below. + lambda strings: choices[str_choices.index(strings[0])], + InstantiatorMetadata( + nargs=1, + metavar="{" + ",".join(map(_format_metavar, str_choices)) + "}", + choices=str_choices, + ), ) From 6fd187923ef86ea278e669cfb0c44198c87b44e0 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 24 Jul 2022 14:37:23 -0700 Subject: [PATCH 071/491] Serialization docstring fix --- dcargs/extras/_serialization.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dcargs/extras/_serialization.py b/dcargs/extras/_serialization.py index 77fb647a5..c4ba73a46 100644 --- a/dcargs/extras/_serialization.py +++ b/dcargs/extras/_serialization.py @@ -176,11 +176,14 @@ def from_yaml( stream: Union[str, IO[str], bytes, IO[bytes]], ) -> DataclassType: """Re-construct a dataclass instance from a yaml-compatible string, which should be - generated from `dcargs.to_yaml()`. + generated from `dcargs.extra.to_yaml()`. Args: cls: Type to reconstruct. stream: YAML to read from. + + Returns: + Instantiated dataclass. """ out = yaml.load(stream, Loader=_make_loader(cls)) origin_cls = get_origin(cls) @@ -190,9 +193,12 @@ def from_yaml( def to_yaml(instance: Any) -> str: """Serialize a dataclass; returns a yaml-compatible string that can be deserialized - via `dcargs.from_yaml()`. + via `dcargs.extras.from_yaml()`. Args: instance: Dataclass instance to serialize. + + Returns: + YAML string. """ return "# dcargs YAML.\n" + yaml.dump(instance, Dumper=_make_dumper(instance)) From 2edeee9631d26a14bda2181b1141023400a60fa8 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 24 Jul 2022 18:12:02 -0700 Subject: [PATCH 072/491] Fix docs build --- docs/requirements.txt | 12 +++++------- docs/source/conf.py | 6 ++---- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index bc6a3b4a7..b39d0c586 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,9 +1,7 @@ -sphinx -# sphinx_rtd_theme -furo -sphinx_math_dollar -sphinx-argparse -sphinx-autoapi -m2r2 +sphinx==5.0.2 +furo==2022.6.21 +docutils==0.17.1 +sphinx-autoapi==1.8.4 +m2r2==0.3.2 git+https://github.com/brentyi/sphinxcontrib-programoutput.git git+https://github.com/brentyi/ansi.git diff --git a/docs/source/conf.py b/docs/source/conf.py index 6173128eb..3aa5e3222 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -6,7 +6,7 @@ # full list see the documentation: # http://www.sphinx-doc.org/en/stable/config -from typing import Dict, List, Optional +from typing import Dict, List import m2r2 @@ -50,9 +50,7 @@ "sphinx.ext.napoleon", # "sphinx.ext.inheritance_diagram", "autoapi.extension", - # "sphinx_math_dollar", "sphinx.ext.viewcode", - "sphinxarg.ext", "m2r2", "sphinxcontrib.programoutput", "sphinxcontrib.ansi", @@ -98,7 +96,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language: Optional[str] = None +language: str = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From 0a460d895fea9e54bcd3414f013d3ca15dded36f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 27 Jul 2022 02:02:50 -0700 Subject: [PATCH 073/491] Try poetry --- .coveragerc | 27 -------------- .github/workflows/build.yml | 6 ++-- .github/workflows/coverage.yml | 6 ++-- .github/workflows/docs.yml | 11 +++--- .github/workflows/mypy.yml | 15 +++----- .github/workflows/publish.yml | 11 +++--- .gitignore | 1 + docs/source/installation.rst | 10 ++---- examples/01_functions.py | 2 ++ mypy.ini | 2 -- pyproject.toml | 65 ++++++++++++++++++++++++++++++++++ setup.py | 47 ------------------------ 12 files changed, 91 insertions(+), 112 deletions(-) delete mode 100644 .coveragerc delete mode 100644 mypy.ini create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 74c0558ed..000000000 --- a/.coveragerc +++ /dev/null @@ -1,27 +0,0 @@ -[report] -exclude_lines = - # Have to re-enable the standard pragma - pragma: no cover - - # Don't compute coverage for abstract methods, properties - @abstract - @abc\.abstract - - # or warnings - warnings - - # or empty function bodies - pass - \.\.\. - - # or typing imports - TYPE_CHECKING - - # or assert statements & errors - assert - - # or anything that's not implemented - NotImplementedError() - - # or fallback imports - except ImportError: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index be5c679c0..b5b79ba08 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,8 +21,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -e ".[testing]" + curl -sSL https://install.python-poetry.org | python3 - + poetry install - name: Test with pytest run: | - pytest + poetry run pytest diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 679a55132..c89e3dc5f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,11 +17,11 @@ jobs: python-version: 3.8 - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -e ".[testing]" + curl -sSL https://install.python-poetry.org | python3 - + poetry install - name: Generate coverage report run: | - pytest --cov=dcargs --cov-report=xml + poetry run pytest --cov=dcargs --cov-report=xml - name: Upload to Codecov uses: codecov/codecov-action@v1 with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 63108f0ab..43211e3e1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -7,8 +7,6 @@ on: jobs: docs: runs-on: ubuntu-latest - container: - image: python:3.8 steps: # Check out source @@ -17,11 +15,10 @@ jobs: # Build documentation - name: Building documentation run: | - apt-get update - apt-get install -y graphviz - pip install -e ".[testing]" - pip install -r docs/requirements.txt - sphinx-build docs/source docs/build -b dirhtml + curl -sSL https://install.python-poetry.org | python3 - + poetry install + poetry run pip install -r docs/requirements.txt + poetry run sphinx-build docs/source docs/build -b dirhtml # Deploy - name: Deploy to GitHub Pages diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 7e98a8a7b..b315c9c88 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -9,21 +9,16 @@ on: jobs: mypy: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8"] - steps: - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} + - name: "Set up Python 3.8" uses: actions/setup-python@v1 with: - python-version: ${{ matrix.python-version }} + python-version: "3.8" - name: Install dependencies run: | - sudo apt update - python -m pip install --upgrade pip - pip install -e .[type-checking] + curl -sSL https://install.python-poetry.org | python3 - + poetry install - name: Test with mypy run: | - mypy --install-types --non-interactive . + poetry run mypy --install-types --non-interactive . diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d5f3859d1..62aa21212 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,12 +20,11 @@ jobs: python-version: '3.x' - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine + curl -sSL https://install.python-poetry.org | python3 - + poetry install - name: Build and publish env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - python setup.py sdist bdist_wheel - twine upload dist/* + poetry publish --username $PYPI_USERNAME --password $PYPI_PASSWORD diff --git a/.gitignore b/.gitignore index 3abe36ed2..eb82eb73b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ __pycache__ .coverage build/ dist/ +poetry.lock diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 3980e2722..697fe4931 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -16,17 +16,14 @@ required. Development ------------------------------------------ -If you're interested in development, :code:`dcargs` can also be installed from -source. +If you're interested in development, the recommended way to install :code:`dcargs` is via `poetry`: .. code-block:: - # Clone repository. + # Clone repository and install. git clone git@github.com:brentyi/dcargs.git cd dcargs - - # Install development dependencies. - pip install -e ".[testing,type-checking]" + poetry install # Run tests. pytest @@ -34,4 +31,3 @@ source. # Check types. mypy --install-types . - diff --git a/examples/01_functions.py b/examples/01_functions.py index c51af61f8..ab31be917 100644 --- a/examples/01_functions.py +++ b/examples/01_functions.py @@ -1,3 +1,5 @@ +# PYTHON_ARGCOMPLETE_OK + """In the simplest case, `dcargs.cli()` can be used to run a function with arguments populated from the CLI. diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 976ba0294..000000000 --- a/mypy.ini +++ /dev/null @@ -1,2 +0,0 @@ -[mypy] -ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..46b75caf4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,65 @@ +[tool.poetry] +name = "dcargs" +version = "0.1.11" +description = "Strongly typed, zero-effort CLI interfaces" +authors = ["brentyi "] +include = ["./dcargs/**/*"] + +[tool.poetry.dependencies] +python = "^3.7" +docstring-parser = "^0.14.1" +typing-extensions = "^4.3.0" +PyYAML = "^6.0" +termcolor = "^1.1.0" +"backports.cached-property" = { version = "^1.0.2", python = "~3.7" } + +[tool.poetry.dev-dependencies] +pytest = "^7.1.2" +pytest-cov = "^3.0.0" +omegaconf = "^2.2.2" +attrs = "^21.4.0" +torch = "^1.10.0" +mypy = "^0.971" +pyright = "^1.1.264" +coverage = {extras = ["toml"], version = "^6.4.2"} + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.isort] +profile = "black" + +[tool.mypy] +python_version = "3.8" +ignore_missing_imports = true +warn_unused_configs = true + +[tool.coverage.report] +exclude_lines = [ + # Have to re-enable the standard pragma + "pragma: no cover", + + # Don't compute coverage for abstract methods, properties + "@abstract", + "@abc.abstract", + + # or warnings + "warnings", + + # or empty function bodies + "pass", + "...", + + # or typing imports + "TYPE_CHECKING", + + # or assert statements & errors + "assert", + + # or anything that's not implemented + "NotImplementedError()", + + # or fallback imports + "except ImportError:", +] diff --git a/setup.py b/setup.py deleted file mode 100644 index 340d9de2a..000000000 --- a/setup.py +++ /dev/null @@ -1,47 +0,0 @@ -from setuptools import find_packages, setup - -with open("README.md", "r") as fh: - long_description = fh.read() - -setup( - name="dcargs", - version="0.1.10", - description="Strongly typed, zero-effort CLIs", - long_description=long_description, - long_description_content_type="text/markdown", - url="http://github.com/brentyi/dcargs", - author="brentyi", - author_email="brentyi@berkeley.edu", - license="MIT", - packages=find_packages(), - package_data={"dcargs": ["py.typed"]}, - python_requires=">=3.7", - install_requires=[ - "docstring_parser", - "typing_extensions>=4.3.0", - "pyyaml", - "termcolor", - "backports.cached-property; python_version < '3.8.0'", - ], - extras_require={ - "testing": [ - "pytest", - "pytest-cov", - "omegaconf", - "attrs", - "torch", - ], - "type-checking": [ - "mypy", - "pyright", - ], - }, - classifiers=[ - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - ], -) From df1cf543a92b44050e7c1970c9563f3651ae6b06 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 27 Jul 2022 03:33:23 -0700 Subject: [PATCH 074/491] Add `generate_parser()` for autocompletion tools --- dcargs/__init__.py | 5 ++-- dcargs/_cli.py | 71 +++++++++++++++++++++++++++++++++++++++----- tests/test_dcargs.py | 3 ++ 3 files changed, 69 insertions(+), 10 deletions(-) diff --git a/dcargs/__init__.py b/dcargs/__init__.py index 1910a2f71..4022ff6a4 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,12 +1,13 @@ from . import extras -from ._cli import cli +from ._cli import cli, generate_parser from ._fields import MISSING_PUBLIC as MISSING from ._instantiators import UnsupportedTypeAnnotationError __all__ = [ "extras", - "MISSING", "cli", + "generate_parser", + "MISSING", "UnsupportedTypeAnnotationError", ] diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 8c5ab8688..f7aeb5caf 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -3,11 +3,42 @@ import argparse from typing import Callable, Optional, Sequence, TypeVar, Union +from typing_extensions import Literal, assert_never + from . import _argparse_formatter, _calling, _fields, _parsers T = TypeVar("T") +def generate_parser( + f: Callable[..., T], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + default_instance: Optional[T] = None, + avoid_subparsers: bool = False, +) -> argparse.ArgumentParser: + """Returns the argparse parser that would be used under-the-hood if `dcargs.cli()` + was called with the same arguments. + + This can be useful for libraries like argcomplete, pyzshcomplete, or shtab, which + enable autocompletion for argparse parsers.""" + # Potentially we could do some caching in the future, to reduce the redundant work + # done when both `generate_parser()` and `cli()` are called. + out = _cli_impl( + "parser", + f, + prog=prog, + description=description, + args=args, + default_instance=default_instance, + avoid_subparsers=avoid_subparsers, + ) + assert isinstance(out, argparse.ArgumentParser) + return out + + def cli( f: Callable[..., T], *, @@ -70,13 +101,32 @@ def cli( Returns: The output of `f(...)`. """ + out = _cli_impl( + "f_out", + f, + prog=prog, + description=description, + args=args, + default_instance=default_instance, + avoid_subparsers=avoid_subparsers, + ) + assert not isinstance(out, argparse.ArgumentParser) + return out + - default_instance_internal: Union[_fields.NonpropagatingMissingType, T] - if default_instance is None: - default_instance_internal = _fields.MISSING_NONPROP - else: - default_instance_internal = default_instance - del default_instance +def _cli_impl( + _return_stage: Literal["parser", "f_out"], + f: Callable[..., T], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + default_instance: Optional[T] = None, + avoid_subparsers: bool = False, +) -> Union[T, argparse.ArgumentParser]: + default_instance_internal: Union[_fields.NonpropagatingMissingType, T] = ( + _fields.MISSING_NONPROP if default_instance is None else default_instance + ) # Map a callable to the relevant CLI arguments + subparsers. parser_definition = _parsers.ParserSpecification.from_callable( @@ -89,11 +139,13 @@ def cli( avoid_subparsers=avoid_subparsers, ) - # Parse using argparse! + # Generate parser! with _argparse_formatter.argparse_ansi_monkey_patch(): parser = argparse.ArgumentParser( prog=prog, formatter_class=_argparse_formatter.ArgparseHelpFormatter ) + if _return_stage == "parser": + return parser parser_definition.apply(parser) value_from_prefixed_field_name = vars(parser.parse_args(args=args)) @@ -115,4 +167,7 @@ def cli( raise SystemExit() assert len(value_from_prefixed_field_name.keys() - consumed_keywords) == 0 - return out + if _return_stage == "f_out": + return out + + assert_never(_return_stage) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index a5312d023..2fffeec94 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -1,3 +1,4 @@ +import argparse import copy import dataclasses import enum @@ -28,6 +29,8 @@ class ManyTypes: f: float p: pathlib.Path + assert isinstance(dcargs.generate_parser(ManyTypes), argparse.ArgumentParser) + # We can directly pass a dataclass to `dcargs.cli()`: assert dcargs.cli( ManyTypes, From f0d6a046f2d1a03be0efff54eda12c50a2f225ef Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 7 Aug 2022 04:18:33 -0700 Subject: [PATCH 075/491] Add KW_ONLY test for Python 3.10 --- dcargs/_arguments.py | 1 - dcargs/_parsers.py | 1 - tests/conftest.py | 6 +++++- tests/test_kw_only_only_py310.py | 23 +++++++++++++++++++++++ tests/test_strings.py | 2 +- 5 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 tests/test_kw_only_only_py310.py diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index e5eb1bdf0..e7bb12b5a 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -1,6 +1,5 @@ """Rules for taking high-level field definitions and lowering them into inputs for argparse's `add_argument()`.""" - from __future__ import annotations import argparse diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index aff0fce1f..0eed2cb9c 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -1,5 +1,4 @@ """Interface for generating `argparse.ArgumentParser()` definitions from callables.""" - from __future__ import annotations import argparse diff --git a/tests/conftest.py b/tests/conftest.py index 7b7e4729b..0e7694219 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,8 @@ import sys +collect_ignore_glob = [] if sys.version_info.major == 3 and sys.version_info.minor == 7: - collect_ignore_glob = ["*_ignore_py37.py"] + collect_ignore_glob.append("*_ignore_py37.py") + +if not (sys.version_info.major == 3 and sys.version_info.minor == 10): + collect_ignore_glob.append("*_only_py310.py") diff --git a/tests/test_kw_only_only_py310.py b/tests/test_kw_only_only_py310.py new file mode 100644 index 000000000..f7cef225a --- /dev/null +++ b/tests/test_kw_only_only_py310.py @@ -0,0 +1,23 @@ +import dataclasses + +import pytest + +import dcargs + + +@dataclasses.dataclass +class Args: + a: int + b: int + _: dataclasses.KW_ONLY # type: ignore + c: int = 7 + d: int # type: ignore + + +def test_kw_only(): + assert dcargs.cli(Args, args="--a 5 --b 3 --c 2 --d 1".split(" ")) == Args( + 5, 3, c=2, d=1 + ) + assert dcargs.cli(Args, args="--a 5 --b 3 --d 1".split(" ")) == Args(5, 3, c=7, d=1) + with pytest.raises(SystemExit): + dcargs.cli(Args, args="--a 5 --b 3".split(" ")) diff --git a/tests/test_strings.py b/tests/test_strings.py index 1f3635c69..a654eee0d 100644 --- a/tests/test_strings.py +++ b/tests/test_strings.py @@ -1,4 +1,4 @@ -import dcargs._strings as _strings +from dcargs import _strings as _strings def test_words_from_name(): From 55c635ea8ff38e74adcfb699b1aee3c85404a7a0 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 7 Aug 2022 04:23:22 -0700 Subject: [PATCH 076/491] More consistent helptext formatting for subparsers --- dcargs/_parsers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 0eed2cb9c..87141d018 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -301,7 +301,7 @@ def format_group_name(nested_field_name: str) -> str: + _strings.SUBPARSER_DEST_FMT.format(name=subparsers.name), description=subparsers.description, required=subparsers.required, - title=title, + title=termcolor.colored(title, attrs=["bold"]), metavar=metavar, ) for name, subparser_def in subparsers.parser_from_name.items(): @@ -405,19 +405,27 @@ def from_field( ): required = True + # Make description. description_parts = [] if field.helptext is not None: description_parts.append(field.helptext) if not required and field.default not in _fields.MISSING_SINGLETONS: default = _strings.subparser_name_from_type(type(field.default)) description_parts.append(f" (default: {default})") + description = ( + # We use `None` instead of an empty string to prevent a line break from + # being created where the description would be. + " ".join(description_parts) + if len(description_parts) > 0 + else None + ) return SubparsersSpecification( name=field.name, # If we wanted, we could add information about the default instance # automatically, as is done for normal fields. But for now we just rely on # the user to include it in the docstring. - description=" ".join(description_parts), + description=description, parser_from_name=parser_from_name, required=required, default_instance=field.default From bd4e0e68411e80446784a96cc3b3433acc3428ec Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 7 Aug 2022 21:45:27 -0700 Subject: [PATCH 077/491] Fix subclass comment edge case for helptext --- dcargs/_docstrings.py | 15 ++++++++++----- tests/test_helptext.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 7030436b5..da38b6dc0 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -194,17 +194,20 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: current_actual_line = field_data.actual_line - 1 while current_actual_line in tokenization.tokens_from_actual_line: actual_line_tokens = tokenization.tokens_from_actual_line[current_actual_line] - current_actual_line -= 1 # We stop looking if we find an empty line. if len(actual_line_tokens) == 0: break + # We don't look in the first logical line. This includes all comments that come + # before the end parentheses in the class definition (eg comments in the + # subclass list). + logical_line = actual_line_tokens[0].logical_line + if logical_line == tokenization.tokens[0].logical_line: + break + # Record single comments! - if ( - len(actual_line_tokens) == 1 - and actual_line_tokens[0].token_type == tokenize.COMMENT - ): + if len(actual_line_tokens) == 1: (comment_token,) = actual_line_tokens assert comment_token.content.startswith("#") comments.append(comment_token.content[1:].strip()) @@ -212,6 +215,8 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: # Comments should be contiguous. break + current_actual_line -= 1 + if len(comments) > 0: return "\n".join(reversed(comments)) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 59287de62..53092099f 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -454,3 +454,19 @@ def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> d assert ( "--x {INT INT}|{STR STR} INT INT [{INT INT}|{STR STR} INT INT ...]" in helptext ) + + +def test_comment_in_subclass_list(): + @dataclasses.dataclass + class Something( + # This text should not show up in the helptext anywhere. + object, + ): + a: int + + # But this text should! + b: int + + helptext = _get_helptext(Something) + assert "This text should not" not in helptext + assert "But this text should!" in helptext From 6700f12086d74c5b3932a829ed9f8d79a476efb7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 7 Aug 2022 22:30:32 -0700 Subject: [PATCH 078/491] Fix for Python 3.9 tokenization behavior changes --- dcargs/_docstrings.py | 14 ++++++++++++-- examples/01_functions.py | 1 + pyproject.toml | 1 + tests/test_helptext.py | 1 - 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index da38b6dc0..77b6c94d2 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -190,6 +190,17 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: # # In this case, 'Optimizer hyperparameters' will be treated as the docstring for all # 3 fields. + + # Get first line of the class definition, excluding decorators. This logic is only + # needed for Python >= 3.9; in 3.8, we can simply use + # `tokenization.tokens[0].logical_line`. + classdef_logical_line = -1 + for token in tokenization.tokens: + if token.content == "class": + classdef_logical_line = token.logical_line + break + assert classdef_logical_line != -1 + comments: List[str] = [] current_actual_line = field_data.actual_line - 1 while current_actual_line in tokenization.tokens_from_actual_line: @@ -202,8 +213,7 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: # We don't look in the first logical line. This includes all comments that come # before the end parentheses in the class definition (eg comments in the # subclass list). - logical_line = actual_line_tokens[0].logical_line - if logical_line == tokenization.tokens[0].logical_line: + if actual_line_tokens[0].logical_line <= classdef_logical_line: break # Record single comments! diff --git a/examples/01_functions.py b/examples/01_functions.py index ab31be917..77a3a82e2 100644 --- a/examples/01_functions.py +++ b/examples/01_functions.py @@ -24,6 +24,7 @@ def main( """ print(field1, field2) +parser = lambda: dcargs.generate_parser(main) if __name__ == "__main__": dcargs.cli(main) diff --git a/pyproject.toml b/pyproject.toml index 46b75caf4..19d8bf512 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ torch = "^1.10.0" mypy = "^0.971" pyright = "^1.1.264" coverage = {extras = ["toml"], version = "^6.4.2"} +numpy = ">=1.20.0" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 53092099f..a0dc0ca84 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -103,7 +103,6 @@ def main_no_docstring(a: Inner) -> None: assert "Hello world!" in helptext helptext = _get_helptext(main_no_docstring) - print(helptext) assert "Something" in helptext assert "Args:" not in helptext assert "Hello world!" in helptext From a1a819c66accfc9667c4e6b3cf1e30bb62b773da Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 7 Aug 2022 23:53:53 -0700 Subject: [PATCH 079/491] Fix `dcargs.generate_parser()` --- dcargs/_argparse_formatter.py | 9 ++++++--- dcargs/_cli.py | 2 +- examples/01_functions.py | 1 - tests/test_helptext.py | 15 ++++++++++++--- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index fe6ffcba5..e493c497e 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -16,9 +16,12 @@ def monkeypatched_len(obj: Any) -> int: else: return len(obj) - argparse.len = monkeypatched_len # type: ignore - yield - del argparse.len # type: ignore + if not hasattr(argparse, "len"): + argparse.len = monkeypatched_len # type: ignore + yield + del argparse.len # type: ignore + else: + yield class ArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): diff --git a/dcargs/_cli.py b/dcargs/_cli.py index f7aeb5caf..b3d17338e 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -144,9 +144,9 @@ def _cli_impl( parser = argparse.ArgumentParser( prog=prog, formatter_class=_argparse_formatter.ArgparseHelpFormatter ) + parser_definition.apply(parser) if _return_stage == "parser": return parser - parser_definition.apply(parser) value_from_prefixed_field_name = vars(parser.parse_args(args=args)) try: diff --git a/examples/01_functions.py b/examples/01_functions.py index 77a3a82e2..ab31be917 100644 --- a/examples/01_functions.py +++ b/examples/01_functions.py @@ -24,7 +24,6 @@ def main( """ print(field1, field2) -parser = lambda: dcargs.generate_parser(main) if __name__ == "__main__": dcargs.cli(main) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index a0dc0ca84..bcef2f2a5 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -10,14 +10,23 @@ from typing_extensions import Literal import dcargs +import dcargs._argparse_formatter import dcargs._strings def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: target = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(target): - dcargs.cli(f, args=args) + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + dcargs.cli(f, args=args) + + # Check against dcargs.generate_parser(); this should return the same underlying + # parser. + target2 = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): + with dcargs._argparse_formatter.argparse_ansi_monkey_patch(): + dcargs.generate_parser(f).parse_args(args) + assert target.getvalue() == target2.getvalue() + return dcargs._strings.strip_ansi_sequences(target.getvalue()) From 6e81a10bbab5ed41fa83be526b795ed447f4782a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 8 Aug 2022 01:22:57 -0700 Subject: [PATCH 080/491] Improve Windows support for helptext formatting --- dcargs/_argparse_formatter.py | 61 ++++++++++++++++++++++++++--------- dcargs/_cli.py | 2 +- pyproject.toml | 1 + tests/test_helptext.py | 2 +- 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index e493c497e..b88251f72 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -1,27 +1,56 @@ import argparse import contextlib -from typing import Any +from typing import Any, ContextManager, Generator from . import _strings -@contextlib.contextmanager -def argparse_ansi_monkey_patch(): - """Temporary monkey patch for making argparse ignore ANSI codes when wrapping usage - text.""" +def ansi_context() -> ContextManager[None]: + """Context for working with ANSI codes + argparse: + - Applies a temporary monkey patch for making argparse ignore ANSI codes when + wrapping usage text. + - Enables support for Windows via colorama. + """ - def monkeypatched_len(obj: Any) -> int: - if isinstance(obj, str): - return len(_strings.strip_ansi_sequences(obj)) + @contextlib.contextmanager + def inner() -> Generator[None, None, None]: + def monkeypatched_len(obj: Any) -> int: + if isinstance(obj, str): + return len(_strings.strip_ansi_sequences(obj)) + else: + return len(obj) + + if not hasattr(argparse, "len"): + argparse.len = monkeypatched_len # type: ignore + try: + # Use Colorama to support coloring in Windows shells. + import colorama # type: ignore + + # Notes: + # + # (1) This context manager looks very nice and local, but under-the-hood + # does some global operations which look likely to cause unexpected + # behavior if another library relies on `colorama.init()` and + # `colorama.deinit()`. + # + # (2) SSHed into a non-Windows machine from a WinAPI terminal => this + # won't work. + # + # Fixing these issues doesn't seem worth it: it doesn't seem like there + # are low-effort solutions for either problem, and more modern terminals + # in Windows (PowerShell, MSYS2, ...) do support ANSI codes anyways. + with colorama.colorama_text(): + yield + + except ImportError: + yield + + del argparse.len # type: ignore else: - return len(obj) - - if not hasattr(argparse, "len"): - argparse.len = monkeypatched_len # type: ignore - yield - del argparse.len # type: ignore - else: - yield + # No-op when the context manager is nested. + yield + + return inner() class ArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): diff --git a/dcargs/_cli.py b/dcargs/_cli.py index b3d17338e..ab338dfd1 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -140,7 +140,7 @@ def _cli_impl( ) # Generate parser! - with _argparse_formatter.argparse_ansi_monkey_patch(): + with _argparse_formatter.ansi_context(): parser = argparse.ArgumentParser( prog=prog, formatter_class=_argparse_formatter.ArgparseHelpFormatter ) diff --git a/pyproject.toml b/pyproject.toml index 19d8bf512..792fc0d04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ typing-extensions = "^4.3.0" PyYAML = "^6.0" termcolor = "^1.1.0" "backports.cached-property" = { version = "^1.0.2", python = "~3.7" } +colorama = {version = "^0.4.0", platform = "win32"} [tool.poetry.dev-dependencies] pytest = "^7.1.2" diff --git a/tests/test_helptext.py b/tests/test_helptext.py index bcef2f2a5..77007bb00 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -23,7 +23,7 @@ def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: # parser. target2 = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): - with dcargs._argparse_formatter.argparse_ansi_monkey_patch(): + with dcargs._argparse_formatter.ansi_context(): dcargs.generate_parser(f).parse_args(args) assert target.getvalue() == target2.getvalue() From 372843f35da18340b42a63c925bb1812e4d6ef86 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 8 Aug 2022 01:40:07 -0700 Subject: [PATCH 081/491] Poetry cleanup --- .gitignore | 1 - docs/source/installation.md | 28 ++ docs/source/installation.rst | 33 -- poetry.lock | 581 +++++++++++++++++++++++++++++++++++ 4 files changed, 609 insertions(+), 34 deletions(-) create mode 100644 docs/source/installation.md delete mode 100644 docs/source/installation.rst create mode 100644 poetry.lock diff --git a/.gitignore b/.gitignore index eb82eb73b..3abe36ed2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,3 @@ __pycache__ .coverage build/ dist/ -poetry.lock diff --git a/docs/source/installation.md b/docs/source/installation.md new file mode 100644 index 000000000..efe0d88fc --- /dev/null +++ b/docs/source/installation.md @@ -0,0 +1,28 @@ +# Installation + +## Standard + +Installation is supported on Python >=3.7 via pip. This is typically all that's +required. + +```bash +pip install dcargs +``` + +## Development + +If you're interested in development, the recommended way to install `dcargs` is +via [poetry](https://github.com/python-poetry/poetry). + +```bash +# Clone repository and install. +git clone git@github.com:brentyi/dcargs.git +cd dcargs +poetry install + +# Run tests. +poetry run pytest + +# Check types. +poetry run mypy --install-types . +``` diff --git a/docs/source/installation.rst b/docs/source/installation.rst deleted file mode 100644 index 697fe4931..000000000 --- a/docs/source/installation.rst +++ /dev/null @@ -1,33 +0,0 @@ -Installation -========================================== - -Standard ------------------------------------------- - -Installation is supported on Python >=3.7 via pip. This is typically all that's -required. - - -.. code-block:: - - pip install dcargs - - -Development ------------------------------------------- - -If you're interested in development, the recommended way to install :code:`dcargs` is via `poetry`: - -.. code-block:: - - # Clone repository and install. - git clone git@github.com:brentyi/dcargs.git - cd dcargs - poetry install - - # Run tests. - pytest - - # Check types. - mypy --install-types . - diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 000000000..f55188f57 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,581 @@ +[[package]] +name = "antlr4-python3-runtime" +version = "4.9.3" +description = "ANTLR 4.9.3 runtime for Python 3.7" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "attrs" +version = "21.4.0" +description = "Classes Without Boilerplate" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] + +[[package]] +name = "backports.cached-property" +version = "1.0.2" +description = "cached_property() - computed once per instance, cached as attribute" +category = "main" +optional = false +python-versions = ">=3.6.0" + +[[package]] +name = "colorama" +version = "0.4.5" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "coverage" +version = "6.4.3" +description = "Code coverage measurement for Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "docstring-parser" +version = "0.14.1" +description = "Parse Python docstrings in reST, Google and Numpydoc format" +category = "main" +optional = false +python-versions = ">=3.6,<4.0" + +[[package]] +name = "importlib-metadata" +version = "4.12.0" +description = "Read metadata from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +perf = ["ipython"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "mypy" +version = "0.971" +description = "Optional static typing for Python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +mypy-extensions = ">=0.4.3" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} +typing-extensions = ">=3.10" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "0.4.3" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "nodeenv" +version = "1.7.0" +description = "Node.js virtual environment builder" +category = "dev" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" + +[[package]] +name = "numpy" +version = "1.21.1" +description = "NumPy is the fundamental package for array computing with Python." +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "omegaconf" +version = "2.2.2" +description = "A flexible configuration library" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +antlr4-python3-runtime = ">=4.9.0,<4.10.0" +PyYAML = ">=5.1.0" + +[[package]] +name = "packaging" +version = "21.3" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} + +[package.extras] +testing = ["pytest-benchmark", "pytest"] +dev = ["tox", "pre-commit"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "dev" +optional = false +python-versions = ">=3.6.8" + +[package.extras] +diagrams = ["railroad-diagrams", "jinja2"] + +[[package]] +name = "pyright" +version = "1.1.266" +description = "Command line wrapper for pyright" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +nodeenv = ">=1.6.0" +typing-extensions = {version = ">=3.7", markers = "python_version < \"3.8\""} + +[package.extras] +all = ["twine (>=3.4.1)"] +dev = ["twine (>=3.4.1)"] + +[[package]] +name = "pytest" +version = "7.1.2" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +tomli = ">=1.0.0" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "pytest-cov" +version = "3.0.0" +description = "Pytest plugin for measuring coverage." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["virtualenv", "pytest-xdist", "six", "process-tests", "hunter", "fields"] + +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "termcolor" +version = "1.1.0" +description = "ANSII Color formatting for output in terminal." +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "torch" +version = "1.12.1" +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +category = "dev" +optional = false +python-versions = ">=3.7.0" + +[package.dependencies] +typing-extensions = "*" + +[[package]] +name = "typed-ast" +version = "1.5.4" +description = "a fork of Python 2 and 3 ast modules with type comment support" +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "typing-extensions" +version = "4.3.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "zipp" +version = "3.8.1" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.7" +content-hash = "277f1e54b73327250e08fd0cf5b127efab41d324617bf7368ea8249d1c2ffa02" + +[metadata.files] +antlr4-python3-runtime = [ + {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, +] +atomicwrites = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] +attrs = [ + {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, + {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, +] +"backports.cached-property" = [ + {file = "backports.cached-property-1.0.2.tar.gz", hash = "sha256:9306f9eed6ec55fd156ace6bc1094e2c86fae5fb2bf07b6a9c00745c656e75dd"}, + {file = "backports.cached_property-1.0.2-py3-none-any.whl", hash = "sha256:baeb28e1cd619a3c9ab8941431fe34e8490861fb998c6c4590693d50171db0cc"}, +] +colorama = [ + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, +] +coverage = [ + {file = "coverage-6.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f50d3a822947572496ea922ee7825becd8e3ae6fbd2400cd8236b7d64b17f285"}, + {file = "coverage-6.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d5191d53afbe5b6059895fa7f58223d3751c42b8101fb3ce767e1a0b1a1d8f87"}, + {file = "coverage-6.4.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04010af3c06ce2bfeb3b1e4e05d136f88d88c25f76cd4faff5d1fd84d11581ea"}, + {file = "coverage-6.4.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6630d8d943644ea62132789940ca97d05fac83f73186eaf0930ffa715fbdab6b"}, + {file = "coverage-6.4.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05de0762c1caed4a162b3e305f36cf20a548ff4da0be6766ad5c870704be3660"}, + {file = "coverage-6.4.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0e3a41aad5919613483aad9ebd53336905cab1bd6788afd3995c2a972d89d795"}, + {file = "coverage-6.4.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a2738ba1ee544d6f294278cfb6de2dc1f9a737a780469b5366e662a218f806c3"}, + {file = "coverage-6.4.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a0d2df4227f645a879010461df2cea6b7e3fb5a97d7eafa210f7fb60345af9e8"}, + {file = "coverage-6.4.3-cp310-cp310-win32.whl", hash = "sha256:73a10939dc345460ca0655356a470dd3de9759919186a82383c87b6eb315faf2"}, + {file = "coverage-6.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:53c8edd3b83a4ddba3d8c506f1359401e7770b30f2188f15c17a338adf5a14db"}, + {file = "coverage-6.4.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f1eda5cae434282712e40b42aaf590b773382afc3642786ac3ed39053973f61f"}, + {file = "coverage-6.4.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59fc88bc13e30f25167e807b8cad3c41b7218ef4473a20c86fd98a7968733083"}, + {file = "coverage-6.4.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75314b00825d70e1e34b07396e23f47ed1d4feedc0122748f9f6bd31a544840"}, + {file = "coverage-6.4.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52f8b9fcf3c5e427d51bbab1fb92b575a9a9235d516f175b24712bcd4b5be917"}, + {file = "coverage-6.4.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5a559aab40c716de80c7212295d0dc96bc1b6c719371c20dd18c5187c3155518"}, + {file = "coverage-6.4.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:306788fd019bb90e9cbb83d3f3c6becad1c048dd432af24f8320cf38ac085684"}, + {file = "coverage-6.4.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:920a734fe3d311ca01883b4a19aa386c97b82b69fbc023458899cff0a0d621b9"}, + {file = "coverage-6.4.3-cp37-cp37m-win32.whl", hash = "sha256:ab9ef0187d6c62b09dec83a84a3b94f71f9690784c84fd762fb3cf2d2b44c914"}, + {file = "coverage-6.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:39ebd8e120cb77a06ee3d5fc26f9732670d1c397d7cd3acf02f6f62693b89b80"}, + {file = "coverage-6.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc698580216050b5f4a34d2cdd2838b429c53314f1c4835fab7338200a8396f2"}, + {file = "coverage-6.4.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:877ee5478fd78e100362aed56db47ccc5f23f6e7bb035a8896855f4c3e49bc9b"}, + {file = "coverage-6.4.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:555a498999c44f5287cc95500486cd0d4f021af9162982cbe504d4cb388f73b5"}, + {file = "coverage-6.4.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eff095a5aac7011fdb51a2c82a8fae9ec5211577f4b764e1e59cfa27ceeb1b59"}, + {file = "coverage-6.4.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5de1e9335e2569974e20df0ce31493d315a830d7987e71a24a2a335a8d8459d3"}, + {file = "coverage-6.4.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7856ea39059d75f822ff0df3a51ea6d76307c897048bdec3aad1377e4e9dca20"}, + {file = "coverage-6.4.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:411fdd9f4203afd93b056c0868c8f9e5e16813e765de962f27e4e5798356a052"}, + {file = "coverage-6.4.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cdf7b83f04a313a21afb1f8730fe4dd09577fefc53bbdfececf78b2006f4268e"}, + {file = "coverage-6.4.3-cp38-cp38-win32.whl", hash = "sha256:ab2b1a89d2bc7647622e9eaf06128a5b5451dccf7c242deaa31420b055716481"}, + {file = "coverage-6.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:0e34247274bde982bbc613894d33f9e36358179db2ed231dd101c48dd298e7b0"}, + {file = "coverage-6.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b104b6b1827d6a22483c469e3983a204bcf9c6bf7544bf90362c4654ebc2edf3"}, + {file = "coverage-6.4.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adf1a0d272633b21d645dd6e02e3293429c1141c7d65a58e4cbcd592d53b8e01"}, + {file = "coverage-6.4.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff9832434a9193fbd716fbe05f9276484e18d26cc4cf850853594bb322807ac3"}, + {file = "coverage-6.4.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:923f9084d7e1d31b5f74c92396b05b18921ed01ee5350402b561a79dce3ea48d"}, + {file = "coverage-6.4.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4d64304acf79766e650f7acb81d263a3ea6e2d0d04c5172b7189180ff2c023c"}, + {file = "coverage-6.4.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:fc294de50941d3da66a09dca06e206297709332050973eca17040278cb0918ff"}, + {file = "coverage-6.4.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a42eaaae772f14a5194f181740a67bfd48e8806394b8c67aa4399e09d0d6b5db"}, + {file = "coverage-6.4.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4822327b35cb032ff16af3bec27f73985448f08e874146b5b101e0e558b613dd"}, + {file = "coverage-6.4.3-cp39-cp39-win32.whl", hash = "sha256:f217850ac0e046ede611312703423767ca032a7b952b5257efac963942c055de"}, + {file = "coverage-6.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:0a84376e4fd13cebce2c0ef8c2f037929c8307fb94af1e5dbe50272a1c651b5d"}, + {file = "coverage-6.4.3-pp36.pp37.pp38-none-any.whl", hash = "sha256:068d6f2a893af838291b8809c876973d885543411ea460f3e6886ac0ee941732"}, + {file = "coverage-6.4.3.tar.gz", hash = "sha256:ec2ae1f398e5aca655b7084392d23e80efb31f7a660d2eecf569fb9f79b3fb94"}, +] +docstring-parser = [ + {file = "docstring_parser-0.14.1-py3-none-any.whl", hash = "sha256:14ac6ec1f1ba6905c4d8cb90fd0bc55394f5678183752c90e44812bf28d7a515"}, + {file = "docstring_parser-0.14.1.tar.gz", hash = "sha256:2c77522e31b7c88b1ab457a1f3c9ae38947ad719732260ba77ee8a3deb58622a"}, +] +importlib-metadata = [ + {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, + {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, +] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] +mypy = [ + {file = "mypy-0.971-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f2899a3cbd394da157194f913a931edfd4be5f274a88041c9dc2d9cdcb1c315c"}, + {file = "mypy-0.971-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:98e02d56ebe93981c41211c05adb630d1d26c14195d04d95e49cd97dbc046dc5"}, + {file = "mypy-0.971-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:19830b7dba7d5356d3e26e2427a2ec91c994cd92d983142cbd025ebe81d69cf3"}, + {file = "mypy-0.971-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:02ef476f6dcb86e6f502ae39a16b93285fef97e7f1ff22932b657d1ef1f28655"}, + {file = "mypy-0.971-cp310-cp310-win_amd64.whl", hash = "sha256:25c5750ba5609a0c7550b73a33deb314ecfb559c350bb050b655505e8aed4103"}, + {file = "mypy-0.971-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d3348e7eb2eea2472db611486846742d5d52d1290576de99d59edeb7cd4a42ca"}, + {file = "mypy-0.971-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3fa7a477b9900be9b7dd4bab30a12759e5abe9586574ceb944bc29cddf8f0417"}, + {file = "mypy-0.971-cp36-cp36m-win_amd64.whl", hash = "sha256:2ad53cf9c3adc43cf3bea0a7d01a2f2e86db9fe7596dfecb4496a5dda63cbb09"}, + {file = "mypy-0.971-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:855048b6feb6dfe09d3353466004490b1872887150c5bb5caad7838b57328cc8"}, + {file = "mypy-0.971-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:23488a14a83bca6e54402c2e6435467a4138785df93ec85aeff64c6170077fb0"}, + {file = "mypy-0.971-cp37-cp37m-win_amd64.whl", hash = "sha256:4b21e5b1a70dfb972490035128f305c39bc4bc253f34e96a4adf9127cf943eb2"}, + {file = "mypy-0.971-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:9796a2ba7b4b538649caa5cecd398d873f4022ed2333ffde58eaf604c4d2cb27"}, + {file = "mypy-0.971-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5a361d92635ad4ada1b1b2d3630fc2f53f2127d51cf2def9db83cba32e47c856"}, + {file = "mypy-0.971-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b793b899f7cf563b1e7044a5c97361196b938e92f0a4343a5d27966a53d2ec71"}, + {file = "mypy-0.971-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d1ea5d12c8e2d266b5fb8c7a5d2e9c0219fedfeb493b7ed60cd350322384ac27"}, + {file = "mypy-0.971-cp38-cp38-win_amd64.whl", hash = "sha256:23c7ff43fff4b0df93a186581885c8512bc50fc4d4910e0f838e35d6bb6b5e58"}, + {file = "mypy-0.971-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1f7656b69974a6933e987ee8ffb951d836272d6c0f81d727f1d0e2696074d9e6"}, + {file = "mypy-0.971-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d2022bfadb7a5c2ef410d6a7c9763188afdb7f3533f22a0a32be10d571ee4bbe"}, + {file = "mypy-0.971-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef943c72a786b0f8d90fd76e9b39ce81fb7171172daf84bf43eaf937e9f220a9"}, + {file = "mypy-0.971-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d744f72eb39f69312bc6c2abf8ff6656973120e2eb3f3ec4f758ed47e414a4bf"}, + {file = "mypy-0.971-cp39-cp39-win_amd64.whl", hash = "sha256:77a514ea15d3007d33a9e2157b0ba9c267496acf12a7f2b9b9f8446337aac5b0"}, + {file = "mypy-0.971-py3-none-any.whl", hash = "sha256:0d054ef16b071149917085f51f89555a576e2618d5d9dd70bd6eea6410af3ac9"}, + {file = "mypy-0.971.tar.gz", hash = "sha256:40b0f21484238269ae6a57200c807d80debc6459d444c0489a102d7c6a75fa56"}, +] +mypy-extensions = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] +nodeenv = [ + {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, + {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, +] +numpy = [ + {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e"}, + {file = "numpy-1.21.1-cp37-cp37m-win32.whl", hash = "sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172"}, + {file = "numpy-1.21.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8"}, + {file = "numpy-1.21.1-cp38-cp38-win32.whl", hash = "sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd"}, + {file = "numpy-1.21.1-cp38-cp38-win_amd64.whl", hash = "sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a"}, + {file = "numpy-1.21.1-cp39-cp39-win32.whl", hash = "sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2"}, + {file = "numpy-1.21.1-cp39-cp39-win_amd64.whl", hash = "sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33"}, + {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"}, + {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, +] +omegaconf = [ + {file = "omegaconf-2.2.2-py3-none-any.whl", hash = "sha256:556917181487fb66fe832d3c7b324f51b2f4c8adc373dd5091be921501b7d420"}, + {file = "omegaconf-2.2.2.tar.gz", hash = "sha256:65c85b2a84669a570c70f2df00de3cebcd9b47a8587d3c53b1aa5766bb096f77"}, +] +packaging = [ + {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, + {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, +] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] +py = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] +pyparsing = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] +pyright = [ + {file = "pyright-1.1.266-py3-none-any.whl", hash = "sha256:246b2fb33c9483ac6e684e8e7619e3ecf0fee20180ec22549a4f3cdf47dbd0de"}, + {file = "pyright-1.1.266.tar.gz", hash = "sha256:1154aeaaa0b62f5161af09342bf0a586468d6789377dfd8710ba8d85c22eee94"}, +] +pytest = [ + {file = "pytest-7.1.2-py3-none-any.whl", hash = "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c"}, + {file = "pytest-7.1.2.tar.gz", hash = "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"}, +] +pytest-cov = [ + {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, + {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, +] +pyyaml = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] +termcolor = [ + {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, +] +tomli = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] +torch = [ + {file = "torch-1.12.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:9c038662db894a23e49e385df13d47b2a777ffd56d9bcd5b832593fab0a7e286"}, + {file = "torch-1.12.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:4e1b9c14cf13fd2ab8d769529050629a0e68a6fc5cb8e84b4a3cc1dd8c4fe541"}, + {file = "torch-1.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:e9c8f4a311ac29fc7e8e955cfb7733deb5dbe1bdaabf5d4af2765695824b7e0d"}, + {file = "torch-1.12.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:976c3f997cea38ee91a0dd3c3a42322785414748d1761ef926b789dfa97c6134"}, + {file = "torch-1.12.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:68104e4715a55c4bb29a85c6a8d57d820e0757da363be1ba680fa8cc5be17b52"}, + {file = "torch-1.12.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:743784ccea0dc8f2a3fe6a536bec8c4763bd82c1352f314937cb4008d4805de1"}, + {file = "torch-1.12.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b5dbcca369800ce99ba7ae6dee3466607a66958afca3b740690d88168752abcf"}, + {file = "torch-1.12.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f3b52a634e62821e747e872084ab32fbcb01b7fa7dbb7471b6218279f02a178a"}, + {file = "torch-1.12.1-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:8a34a2fbbaa07c921e1b203f59d3d6e00ed379f2b384445773bd14e328a5b6c8"}, + {file = "torch-1.12.1-cp37-none-macosx_11_0_arm64.whl", hash = "sha256:42f639501928caabb9d1d55ddd17f07cd694de146686c24489ab8c615c2871f2"}, + {file = "torch-1.12.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0b44601ec56f7dd44ad8afc00846051162ef9c26a8579dda0a02194327f2d55e"}, + {file = "torch-1.12.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:cd26d8c5640c3a28c526d41ccdca14cf1cbca0d0f2e14e8263a7ac17194ab1d2"}, + {file = "torch-1.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:42e115dab26f60c29e298559dbec88444175528b729ae994ec4c65d56fe267dd"}, + {file = "torch-1.12.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:a8320ba9ad87e80ca5a6a016e46ada4d1ba0c54626e135d99b2129a4541c509d"}, + {file = "torch-1.12.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:03e31c37711db2cd201e02de5826de875529e45a55631d317aadce2f1ed45aa8"}, + {file = "torch-1.12.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:9b356aea223772cd754edb4d9ecf2a025909b8615a7668ac7d5130f86e7ec421"}, + {file = "torch-1.12.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:6cf6f54b43c0c30335428195589bd00e764a6d27f3b9ba637aaa8c11aaf93073"}, + {file = "torch-1.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:f00c721f489089dc6364a01fd84906348fe02243d0af737f944fddb36003400d"}, + {file = "torch-1.12.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:bfec2843daa654f04fda23ba823af03e7b6f7650a873cdb726752d0e3718dada"}, + {file = "torch-1.12.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:69fe2cae7c39ccadd65a123793d30e0db881f1c1927945519c5c17323131437e"}, +] +typed-ast = [ + {file = "typed_ast-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4"}, + {file = "typed_ast-1.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62"}, + {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac"}, + {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe"}, + {file = "typed_ast-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72"}, + {file = "typed_ast-1.5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec"}, + {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47"}, + {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6"}, + {file = "typed_ast-1.5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1"}, + {file = "typed_ast-1.5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6"}, + {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66"}, + {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c"}, + {file = "typed_ast-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2"}, + {file = "typed_ast-1.5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d"}, + {file = "typed_ast-1.5.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f"}, + {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc"}, + {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6"}, + {file = "typed_ast-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e"}, + {file = "typed_ast-1.5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35"}, + {file = "typed_ast-1.5.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97"}, + {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3"}, + {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72"}, + {file = "typed_ast-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1"}, + {file = "typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"}, +] +typing-extensions = [ + {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, + {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, +] +zipp = [ + {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, + {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, +] From 96a9767acc148fedf707c32591a5e6ad977d1a2e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 9 Aug 2022 21:20:11 +0100 Subject: [PATCH 082/491] Fix isinstance() corner case Previously: an assert would fail if `f` returned an `argparse.ArgumentParser()` object --- dcargs/_cli.py | 31 +++++++++++++++++++++++++++++-- tests/test_dcargs.py | 8 ++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index ab338dfd1..c18ec170c 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -1,7 +1,7 @@ """Core public API.""" import argparse -from typing import Callable, Optional, Sequence, TypeVar, Union +from typing import Callable, Optional, Sequence, TypeVar, Union, overload from typing_extensions import Literal, assert_never @@ -110,10 +110,37 @@ def cli( default_instance=default_instance, avoid_subparsers=avoid_subparsers, ) - assert not isinstance(out, argparse.ArgumentParser) return out +@overload +def _cli_impl( + _return_stage: Literal["parser"], + f: Callable[..., T], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + default_instance: Optional[T] = None, + avoid_subparsers: bool = False, +) -> argparse.ArgumentParser: + ... + + +@overload +def _cli_impl( + _return_stage: Literal["f_out"], + f: Callable[..., T], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + default_instance: Optional[T] = None, + avoid_subparsers: bool = False, +) -> T: + ... + + def _cli_impl( _return_stage: Literal["parser", "f_out"], f: Callable[..., T], diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 2fffeec94..cf44901ea 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -485,3 +485,11 @@ def test_just_dict(): def test_just_list(): assert dcargs.cli(List[int], args="1 2 3 4".split(" ")) == [1, 2, 3, 4] + + +def test_return_parser(): + def main() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + return parser + + assert isinstance(dcargs.cli(main, args=[]), argparse.ArgumentParser) From e6e5aaab37fc1fe7be5a820f26fad28509f87995 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 11 Aug 2022 01:48:09 -0700 Subject: [PATCH 083/491] Support nested types in fixed-length tuples --- dcargs/_arguments.py | 18 +++-- dcargs/_calling.py | 20 ++++-- dcargs/_docstrings.py | 13 +++- dcargs/_fields.py | 31 ++++++++- dcargs/_parsers.py | 40 +++++++---- dcargs/_strings.py | 23 ++++++- docs/source/examples/01_functions.rst | 9 +++ .../examples/05_hierarchical configs.rst | 9 ++- .../examples/07_literals and unions.rst | 2 +- docs/source/examples/12_named tuples.rst | 48 -------------- docs/source/examples/12_tuples.rst | 66 +++++++++++++++++++ examples/05_hierarchical_configs.py | 9 ++- examples/12_named_tuples.py | 25 ------- examples/12_tuples.py | 36 ++++++++++ tests/test_nested.py | 27 +++++++- 15 files changed, 266 insertions(+), 110 deletions(-) delete mode 100644 docs/source/examples/12_named tuples.rst create mode 100644 docs/source/examples/12_tuples.rst delete mode 100644 examples/12_named_tuples.py create mode 100644 examples/12_tuples.py diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index e7bb12b5a..83fb08863 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -23,7 +23,7 @@ import termcolor -from . import _fields, _instantiators +from . import _fields, _instantiators, _strings try: # Python >=3.8. @@ -284,14 +284,20 @@ def _rule_set_name_or_flag( lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: if arg.field.positional: - name_or_flag = arg.prefix + arg.field.name + name_or_flag = _strings.make_field_name([arg.prefix, arg.field.name]) elif lowered.action == "store_false": - name_or_flag = "--" + (arg.prefix + "no-" + arg.field.name).replace("_", "-") + name_or_flag = "--" + _strings.make_field_name( + [arg.prefix, "no-" + arg.field.name] + ).replace("_", "-") else: - name_or_flag = "--" + (arg.prefix + arg.field.name).replace("_", "-") + name_or_flag = "--" + _strings.make_field_name( + [arg.prefix, arg.field.name] + ).replace("_", "-") return dataclasses.replace( - lowered, name_or_flag=name_or_flag, dest=arg.prefix + arg.field.name + lowered, + name_or_flag=name_or_flag, + dest=_strings.make_field_name([arg.prefix, arg.field.name]), ) @@ -302,7 +308,7 @@ def _rule_positional_special_handling( if not arg.field.positional: return lowered - metavar = (arg.prefix + arg.field.name).upper() + metavar = _strings.make_field_name([arg.prefix, arg.field.name]).upper() if lowered.nargs == "+": metavar = f"{metavar} [{metavar} ...]" elif isinstance(lowered.nargs, int): diff --git a/dcargs/_calling.py b/dcargs/_calling.py index f6c2ad8df..7dbcb1509 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -32,7 +32,7 @@ def call_from_args( f, type_from_typevar = _resolver.resolve_generic_types(f) - args: List[str] = [] + args: List[Any] = [] kwargs: Dict[str, Any] = {} consumed_keywords: Set[str] = set() @@ -44,11 +44,13 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: arg_from_prefixed_field_name: Dict[str, _arguments.ArgumentDefinition] = {} for arg in parser_definition.args: - arg_from_prefixed_field_name[arg.prefix + arg.field.name] = arg + arg_from_prefixed_field_name[ + _strings.make_field_name([arg.prefix, arg.field.name]) + ] = arg for field in _fields.field_list_from_callable(f, default_instance=default_instance): # type: ignore value: Any - prefixed_field_name = field_name_prefix + field.name + prefixed_field_name = _strings.make_field_name([field_name_prefix, field.name]) # Resolve field type. field_type = ( @@ -103,7 +105,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: parser_definition, field.default, value_from_prefixed_field_name, - field_name_prefix=prefixed_field_name + _strings.NESTED_FIELD_DELIMETER, + field_name_prefix=prefixed_field_name, avoid_subparsers=avoid_subparsers, ) consumed_keywords |= consumed_keywords_child @@ -156,8 +158,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: ], field.default if type(field.default) is chosen_f else None, value_from_prefixed_field_name, - field_name_prefix=prefixed_field_name - + _strings.NESTED_FIELD_DELIMETER, + field_name_prefix=prefixed_field_name, avoid_subparsers=avoid_subparsers, ) consumed_keywords |= consumed_keywords_child @@ -168,4 +169,9 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: else: kwargs[field.name] = value - return _resolver.unwrap_origin(f)(*args, **kwargs), consumed_keywords # type: ignore + unwrapped_f = _resolver.unwrap_origin(f) + if unwrapped_f is tuple: + assert len(args) == 0 + return tuple(kwargs.values()), consumed_keywords # type: ignore + else: + return unwrapped_f(*args, **kwargs), consumed_keywords # type: ignore diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 77b6c94d2..2b6e5c8e8 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -5,7 +5,7 @@ import inspect import io import tokenize -from typing import Callable, Dict, List, Optional, Type +from typing import Callable, Dict, Hashable, List, Optional, Type import docstring_parser from typing_extensions import get_origin, is_typeddict @@ -233,6 +233,14 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: return None +_builtins = set( + filter( + lambda x: isinstance(x, Hashable), # type: ignore + __builtins__.values(), # type: ignore + ) +) + + def get_callable_description(f: Callable) -> str: """Get description associated with a callable via docstring parsing. @@ -241,6 +249,9 @@ def get_callable_description(f: Callable) -> str: these docstrings.""" f, _unused = _resolver.resolve_generic_types(f) + if _resolver.unwrap_origin(f) in _builtins: + return "" + # Note inspect.getdoc() causes some corner cases with TypedDicts. docstring = f.__doc__ if ( diff --git a/dcargs/_fields.py b/dcargs/_fields.py index ce56971b8..688b727d7 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -4,7 +4,7 @@ import dataclasses import inspect import warnings -from typing import Any, Callable, List, Optional, Type, TypeVar, Union, cast +from typing import Any, Callable, List, Optional, Type, TypeVar, Union, cast, get_args import docstring_parser from typing_extensions import get_type_hints, is_typeddict @@ -174,6 +174,35 @@ def field_list_from_callable( ) ) return field_list + elif _resolver.unwrap_origin(f) is tuple: + # Fixed-length tuples. + field_list = [] + children = get_args(f) + assert Ellipsis not in children + for i, child in enumerate(children): + if default_instance in MISSING_SINGLETONS: + default_i = default_instance + else: + assert isinstance(default_instance, tuple) + assert len(default_instance) == len( + children + ), "Tuple: length of annotation and type don't match." + default_i = default_instance[i] + + field_list.append( + FieldDefinition( + # We'd use an index operator h + name=str(i), + typ=child, + default=default_i, + helptext="", + # This should really be positional=True, but the CLI is more + # intuitive for mixed nested/non-nested types in tuples when we + # stick with kwargs. Tuples are special-cased in _calling.py. + positional=False, + ) + ) + return field_list else: # Handle general callables. assert ( diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 87141d018..78acaf986 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -57,16 +57,13 @@ def is_possibly_nested_type(typ: Any) -> bool: Examples of when we return False: int, str, List[int], List[str], pathlib.Path, etc. """ + typ_orig = typ typ = _resolver.unwrap_origin(typ) # Nested types should be callable. if not callable(typ): return False - # Known parsable types: builtins like int/str/float/bool, collections, annotations. - if typ in _known_parsable_types: - return False - # Simple heuristic: dataclasses should be treated as nested objects and are not # parsable. if dataclasses.is_dataclass(typ): @@ -76,6 +73,20 @@ def is_possibly_nested_type(typ: Any) -> bool: if is_typeddict(typ): return True + # Fixed-length tuple types. + if typ is tuple: + types = get_args(typ_orig) + + # Nested types must be fixed-length. + if Ellipsis in types: + return False + + return any(map(is_possibly_nested_type, types)) + + # Known parsable types: builtins like int/str/float/bool, collections, annotations. + if typ in _known_parsable_types: + return False + # Nested types like nested (data)classes should have fully type-annotated inputs. If # any inputs are unannotated (for example, in the case of pathlib.Path), we can # assume the type is not nested. @@ -172,7 +183,7 @@ def from_callable( field, type_from_typevar=type_from_typevar, parent_classes=parent_classes, - prefix=prefix + field.name + _strings.NESTED_FIELD_DELIMETER, + prefix=_strings.make_field_name([prefix, field.name]), avoid_subparsers=avoid_subparsers, ) if subparsers_attempt is not None: @@ -198,14 +209,14 @@ def from_callable( parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, default_instance=field.default, - prefix=prefix + field.name + _strings.NESTED_FIELD_DELIMETER, + prefix=_strings.make_field_name([prefix, field.name]), avoid_subparsers=avoid_subparsers, ) args.extend(nested_parser.args) for k, v in nested_parser.helptext_from_nested_class_field_name.items(): helptext_from_nested_class_field_name[ - field.name + _strings.NESTED_FIELD_DELIMETER + k + _strings.make_field_name([field.name, k]) ] = v if field.helptext is not None: @@ -274,11 +285,10 @@ def format_group_name(nested_field_name: str) -> str: continue if arg.prefix not in group_from_prefix: - nested_field_name = arg.prefix[:-1] group_from_prefix[arg.prefix] = parser.add_argument_group( - format_group_name(nested_field_name), + format_group_name(arg.prefix), description=self.helptext_from_nested_class_field_name.get( - nested_field_name + arg.prefix ), ) arg.add_argument(group_from_prefix[arg.prefix]) @@ -297,8 +307,14 @@ def format_group_name(nested_field_name: str) -> str: for p in prev_subparser_tree_nodes: # Add subparsers to every node in previous level of the tree. argparse_subparsers = p.add_subparsers( - dest=self.prefix - + _strings.SUBPARSER_DEST_FMT.format(name=subparsers.name), + dest=_strings.make_field_name( + [ + self.prefix, + _strings.SUBPARSER_DEST_FMT.format( + name=subparsers.name + ), + ] + ), description=subparsers.description, required=subparsers.required, title=termcolor.colored(title, attrs=["bold"]), diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 1fba28915..61e8e526f 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -3,11 +3,30 @@ import functools import re import textwrap -from typing import Type +from typing import List, Sequence, Type from . import _resolver -NESTED_FIELD_DELIMETER: str = "." + +def make_field_name(parts: Sequence[str]) -> str: + """Join parts of a field name together. Used for nesting. + + ('parent', 'child') => 'parent.child' + ('parents', '1', 'child') => 'parents:1.child' + """ + out: List[str] = [] + for i, p in enumerate([p for p in parts if len(p) > 0]): + if i > 0: + # Delimeter between parts. We use a colon before integers, which can + # currently only come from indices! (since field names can't start with + # digits) + out.append(":" if p[0].isdigit() else ".") + + out.append(p) + + return "".join(out) + + SUBPARSER_DEST_FMT: str = "{name} (positional)" diff --git a/docs/source/examples/01_functions.rst b/docs/source/examples/01_functions.rst index def4a7c8d..2fe87aa42 100644 --- a/docs/source/examples/01_functions.rst +++ b/docs/source/examples/01_functions.rst @@ -13,6 +13,15 @@ populated from the CLI. .. code-block:: python :linenos: + In the simplest case, `dcargs.cli()` can be used to run a function with arguments + populated from the CLI. + + Usage: + `python ./01_functions.py --help` + `python ./01_functions.py --field1 hello` + `python ./01_functions.py --field1 hello --field2 10` + """ + import dcargs diff --git a/docs/source/examples/05_hierarchical configs.rst b/docs/source/examples/05_hierarchical configs.rst index fb87546f9..031133c71 100644 --- a/docs/source/examples/05_hierarchical configs.rst +++ b/docs/source/examples/05_hierarchical configs.rst @@ -16,6 +16,7 @@ objects. This helps with modularity and grouping in larger projects. import dataclasses import enum import pathlib + from typing import Tuple import dcargs @@ -25,7 +26,7 @@ objects. This helps with modularity and grouping in larger projects. SGD = enum.auto() - @dataclasses.dataclass(frozen=True) + @dataclasses.dataclass class OptimizerConfig: # Gradient-based optimizer to use. algorithm: OptimizerType = OptimizerType.ADAM @@ -37,10 +38,10 @@ objects. This helps with modularity and grouping in larger projects. weight_decay: float = 1e-2 - @dataclasses.dataclass(frozen=True) + @dataclasses.dataclass class ExperimentConfig: # Various configurable options for our optimizer. - optimizer: OptimizerConfig + optimizer: Tuple[OptimizerConfig, OptimizerConfig, OptimizerConfig] # Batch size. batch_size: int = 32 @@ -68,7 +69,9 @@ objects. This helps with modularity and grouping in larger projects. checkpoint_interval: Training steps between each checkpoint save. """ print(f"{out_dir=}, {restore_checkpoint=}, {checkpoint_interval=}") + print() print(f"{config=}") + print() print(dcargs.to_yaml(config)) diff --git a/docs/source/examples/07_literals and unions.rst b/docs/source/examples/07_literals and unions.rst index 4a616d7d4..6f43fa83b 100644 --- a/docs/source/examples/07_literals and unions.rst +++ b/docs/source/examples/07_literals and unions.rst @@ -32,7 +32,7 @@ # enums. restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED - # Mixed types are okay. + # Or mix them with other types! mixed: Literal[Color.RED, Color.GREEN, "blue"] = "blue" # Literals can also be marked Optional. diff --git a/docs/source/examples/12_named tuples.rst b/docs/source/examples/12_named tuples.rst deleted file mode 100644 index 9e65e21d9..000000000 --- a/docs/source/examples/12_named tuples.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -12. Named Tuples -========================================== - - -Example using ``dcargs.cli()`` to instantiate a named tuple. - - - -.. code-block:: python - :linenos: - - from typing import NamedTuple - - import dcargs - - - class TupleType(NamedTuple): - """Description. - This should show up in the helptext!""" - - field1: str # A string field. - field2: int = 3 # A numeric field, with a default value. - flag: bool = False # A boolean flag. - - - if __name__ == "__main__": - x = dcargs.cli(TupleType) - assert isinstance(x, tuple) - print(x) - ------------- - -.. raw:: html - - python 12_named_tuples.py --help - -.. program-output:: python ../../examples/12_named_tuples.py --help - ------------- - -.. raw:: html - - python 12_named_tuples.py --field1 hello - -.. program-output:: python ../../examples/12_named_tuples.py --field1 hello diff --git a/docs/source/examples/12_tuples.rst b/docs/source/examples/12_tuples.rst new file mode 100644 index 000000000..70d8342b7 --- /dev/null +++ b/docs/source/examples/12_tuples.rst @@ -0,0 +1,66 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +12. Tuples +========================================== + + +Example using ``dcargs.cli()`` to instantiate tuple types. + + + +.. code-block:: python + :linenos: + + import dataclasses + from typing import NamedTuple, Tuple + + import dcargs + + + @dataclasses.dataclass + class Color: + r: int + g: int + b: int + + + class TupleType(NamedTuple): + """Description. + This should show up in the helptext!""" + + # Tuple types can contain raw values. + color: Tuple[int, int, int] = (255, 0, 0) + + # Tuple types can contain nested structures. + two_colors: Tuple[Color, Color] = (Color(255, 0, 0), Color(0, 255, 0)) + + + if __name__ == "__main__": + x = dcargs.cli(TupleType) + assert isinstance(x, tuple) + print(x) + +------------ + +.. raw:: html + + python 12_tuples.py --help + +.. program-output:: python ../../examples/12_tuples.py --help + +------------ + +.. raw:: html + + python 12_tuples.py --color 127 127 127 + +.. program-output:: python ../../examples/12_tuples.py --color 127 127 127 + +------------ + +.. raw:: html + + python 12_tuples.py '--two_colors[1].r' 127 '--two_colors[1].g' 0 '--two_colors[1].b' 0 + +.. program-output:: python ../../examples/12_tuples.py '--two_colors[1].r' 127 '--two_colors[1].g' 0 '--two_colors[1].b' 0 diff --git a/examples/05_hierarchical_configs.py b/examples/05_hierarchical_configs.py index d6a8f6520..164282634 100644 --- a/examples/05_hierarchical_configs.py +++ b/examples/05_hierarchical_configs.py @@ -10,6 +10,7 @@ import dataclasses import enum import pathlib +from typing import Tuple import dcargs @@ -19,7 +20,7 @@ class OptimizerType(enum.Enum): SGD = enum.auto() -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass class OptimizerConfig: # Gradient-based optimizer to use. algorithm: OptimizerType = OptimizerType.ADAM @@ -31,10 +32,10 @@ class OptimizerConfig: weight_decay: float = 1e-2 -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass class ExperimentConfig: # Various configurable options for our optimizer. - optimizer: OptimizerConfig + optimizer: Tuple[OptimizerConfig, OptimizerConfig, OptimizerConfig] # Batch size. batch_size: int = 32 @@ -62,7 +63,9 @@ def train( checkpoint_interval: Training steps between each checkpoint save. """ print(f"{out_dir=}, {restore_checkpoint=}, {checkpoint_interval=}") + print() print(f"{config=}") + print() print(dcargs.to_yaml(config)) diff --git a/examples/12_named_tuples.py b/examples/12_named_tuples.py deleted file mode 100644 index 6e2f2c3dc..000000000 --- a/examples/12_named_tuples.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Example using `dcargs.cli()` to instantiate a named tuple. - -Usage: -`python ./12_named_tuples.py --help` -`python ./12_named_tuples.py --field1 hello` -""" - -from typing import NamedTuple - -import dcargs - - -class TupleType(NamedTuple): - """Description. - This should show up in the helptext!""" - - field1: str # A string field. - field2: int = 3 # A numeric field, with a default value. - flag: bool = False # A boolean flag. - - -if __name__ == "__main__": - x = dcargs.cli(TupleType) - assert isinstance(x, tuple) - print(x) diff --git a/examples/12_tuples.py b/examples/12_tuples.py new file mode 100644 index 000000000..f9adb5664 --- /dev/null +++ b/examples/12_tuples.py @@ -0,0 +1,36 @@ +"""Example using `dcargs.cli()` to instantiate tuple types. + +Usage: +`python ./12_tuples.py --help` +`python ./12_tuples.py --color 127 127 127` +`python ./12_tuples.py --two_colors[1].r 127 --two_colors[1].g 0 --two_colors[1].b 0` +""" + +import dataclasses +from typing import NamedTuple, Tuple + +import dcargs + + +@dataclasses.dataclass +class Color: + r: int + g: int + b: int + + +class TupleType(NamedTuple): + """Description. + This should show up in the helptext!""" + + # Tuple types can contain raw values. + color: Tuple[int, int, int] = (255, 0, 0) + + # Tuple types can contain nested structures. + two_colors: Tuple[Color, Color] = (Color(255, 0, 0), Color(0, 255, 0)) + + +if __name__ == "__main__": + x = dcargs.cli(TupleType) + assert isinstance(x, tuple) + print(x) diff --git a/tests/test_nested.py b/tests/test_nested.py index 55ab661e8..d588b8fcc 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Optional, Union +from typing import Optional, Tuple, Union import pytest @@ -600,3 +600,28 @@ class MultipleSubparsers: ) ), ) == MultipleSubparsers(Subcommand2(Subcommand1(3)), Subcommand2(Subcommand1(7))) + + +def test_tuple_nesting(): + @dataclasses.dataclass(frozen=True) + class Color: + r: int + g: int + b: int + + @dataclasses.dataclass(frozen=True) + class Location: + x: float + y: float + z: float + + def main(x: Tuple[Tuple[Color], Location, float]): + return x + + assert dcargs.cli( + main, + args=( + "--x:0:0.r 255 --x:0:0.g 0 --x:0:0.b 0 --x:1.x 5.0 --x:1.y 0.0" + " --x:1.z 2.0 --x:2 4.0".split(" ") + ), + ) == ((Color(255, 0, 0),), Location(5.0, 0.0, 2.0), 4.0) From 3d719d91a93271aa4656d0eb418cac4cd1dc1bc7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 12 Aug 2022 01:54:35 -0700 Subject: [PATCH 084/491] Refactor nested structure detection + handling --- dcargs/_calling.py | 6 +- dcargs/_fields.py | 436 +++++++++++++++++++++++++++---------------- dcargs/_parsers.py | 132 +++---------- dcargs/_resolver.py | 13 ++ tests/test_errors.py | 4 +- 5 files changed, 325 insertions(+), 266 deletions(-) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 7dbcb1509..7aac344d9 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -48,7 +48,11 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: _strings.make_field_name([arg.prefix, arg.field.name]) ] = arg - for field in _fields.field_list_from_callable(f, default_instance=default_instance): # type: ignore + for field in _fields.field_list_from_callable( + f, + default_instance=default_instance, + root_field=True, + ): # type: ignore value: Any prefixed_field_name = _strings.make_field_name([field_name_prefix, field.name]) diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 688b727d7..826e27106 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -1,15 +1,30 @@ """Abstractions for pulling out 'field' definitions, which specify inputs, types, and defaults, from general callables.""" +import collections import dataclasses import inspect +import itertools +import typing import warnings -from typing import Any, Callable, List, Optional, Type, TypeVar, Union, cast, get_args +from typing import ( + Any, + Callable, + Hashable, + List, + Optional, + Type, + TypeVar, + Union, + cast, + get_args, +) import docstring_parser +import typing_extensions from typing_extensions import get_type_hints, is_typeddict -from . import _docstrings, _instantiators, _parsers, _resolver +from . import _docstrings, _instantiators, _resolver @dataclasses.dataclass(frozen=True) @@ -44,7 +59,7 @@ class NonpropagatingMissingType(_Singleton): pass -class ExcludeFromKwargsType(_Singleton): +class ExcludeFromCallType(_Singleton): pass @@ -53,7 +68,7 @@ class ExcludeFromKwargsType(_Singleton): # nonpropagating missing sentinel, which does not override child defaults. MISSING_PROP = PropagatingMissingType() MISSING_NONPROP = NonpropagatingMissingType() -EXCLUDE_FROM_CALL = ExcludeFromKwargsType() +EXCLUDE_FROM_CALL = ExcludeFromCallType() # Note that our "public" missing API will always be the propagating missing sentinel. MISSING_PUBLIC: Any = MISSING_PROP @@ -78,15 +93,47 @@ class ExcludeFromKwargsType(_Singleton): T = TypeVar("T") +DefaultInstanceT = Union[T, PropagatingMissingType, NonpropagatingMissingType] + +_known_parsable_types = set( + filter( + lambda x: isinstance(x, Hashable), # type: ignore + itertools.chain( + __builtins__.values(), # type: ignore + vars(typing).values(), + vars(typing_extensions).values(), + vars(collections.abc).values(), + ), + ) +) + + +def is_possibly_nested_type(typ: Any, default_instance: Any) -> bool: + """Determine whether a type can be treated as a 'nested type', where a single field + has multiple corresponding arguments (eg for nested dataclasses or classes).""" + + try: + # This implementation will result in some computational redundancy, but is nice + # for reusing logic. Should be revisited. + field_list_from_callable(typ, default_instance, root_field=False) + return True + except _instantiators.UnsupportedTypeAnnotationError: + return False + def field_list_from_callable( f: Callable[..., T], - default_instance: Union[T, PropagatingMissingType, NonpropagatingMissingType], + default_instance: DefaultInstanceT, + root_field: bool = False, ) -> List[FieldDefinition]: """Generate a list of generic 'field' objects corresponding to the inputs of some annotated callable. - `f` can be from a dataclass type, regular class type, or function.""" + `f` can be from a dataclass type, regular class type, or function. + + If `root_field` is set to True, we treat `int`, `torch.device`, etc as nested + fields. This is to make sure that these types can be passed directly into + dcargs.cli(); the logic can likely be refactored.""" # Unwrap generics. f, type_from_typevar = _resolver.resolve_generic_types(f) @@ -99,164 +146,230 @@ def field_list_from_callable( cls = f f = cls.__init__ # type: ignore + # Handle each supported nested structure type. if cls is not None and is_typeddict(cls): - # Handle typed dictionaries. - field_list = [] - valid_default_instance = ( - default_instance not in MISSING_SINGLETONS - and default_instance is not EXCLUDE_FROM_CALL + return _field_list_from_typeddict(cls, default_instance) + + elif cls is not None and _resolver.is_namedtuple(cls): + return _field_list_from_namedtuple(cls, default_instance) + + elif cls is not None and _resolver.is_dataclass(cls): + return _field_list_from_dataclass(cls, default_instance) + + elif _resolver.unwrap_origin(f) is tuple: + assert cls is None + return _field_list_from_tuple(f, default_instance) + + elif not root_field and ( + (cls is not None and cls in _known_parsable_types) + or _resolver.unwrap_origin(f) in _known_parsable_types + ): + raise _instantiators.UnsupportedTypeAnnotationError( + f"{f} cannot be interpreted as a nested structure!" ) - assert not valid_default_instance or isinstance(default_instance, dict) - for name, typ in get_type_hints(cls).items(): - if valid_default_instance: - default = default_instance.get(name, MISSING_PROP) # type: ignore - elif getattr(cls, "__total__") is False: - default = EXCLUDE_FROM_CALL - if _parsers.is_possibly_nested_type(typ): - raise _instantiators.UnsupportedTypeAnnotationError( - "`total=False` not supported for nested structures." - ) - else: - default = MISSING_PROP - - field_list.append( - FieldDefinition( - name=name, - typ=typ, - default=default, - helptext=_docstrings.get_field_docstring(cls, name), - positional=False, + + else: + return _field_list_from_general_callable( + f, cls, default_instance, root_field=root_field + ) + + +def _field_list_from_typeddict( + cls: Type[T], default_instance: DefaultInstanceT +) -> List[FieldDefinition]: + field_list = [] + valid_default_instance = ( + default_instance not in MISSING_SINGLETONS + and default_instance is not EXCLUDE_FROM_CALL + ) + assert not valid_default_instance or isinstance(default_instance, dict) + for name, typ in get_type_hints(cls).items(): + if valid_default_instance: + default = default_instance.get(name, MISSING_PROP) # type: ignore + elif getattr(cls, "__total__") is False: + default = EXCLUDE_FROM_CALL + if is_possibly_nested_type(typ, MISSING_NONPROP): + raise _instantiators.UnsupportedTypeAnnotationError( + "`total=False` not supported for nested structures." ) + else: + default = MISSING_PROP + + field_list.append( + FieldDefinition( + name=name, + typ=typ, + default=default, + helptext=_docstrings.get_field_docstring(cls, name), + positional=False, ) - return field_list - elif cls is not None and _resolver.is_namedtuple(cls): - # Handle NamedTuples. - # - # TODO: in terms of helptext, we currently do display the default NamedTuple - # helptext. But we (intentionally) don't for dataclasses; this is somewhat - # inconsistent. - field_list = [] - field_defaults = getattr(cls, "_field_defaults") - - # Note that _field_types is removed in Python 3.9. - for name, typ in _resolver.get_type_hints(cls).items(): - # Get default, with priority for `default_instance`. - default = field_defaults.get(name, MISSING_NONPROP) - if hasattr(default_instance, name): - default = getattr(default_instance, name) - if default_instance is MISSING_PROP: - default = MISSING_PROP - - field_list.append( - FieldDefinition( - name=name, - typ=typ, - default=default, - helptext=_docstrings.get_field_docstring(cls, name), - positional=False, - ) + ) + return field_list + + +def _field_list_from_namedtuple( + cls: Type[T], default_instance: DefaultInstanceT +) -> List[FieldDefinition]: + # Handle NamedTuples. + # + # TODO: in terms of helptext, we currently do display the default NamedTuple + # helptext. But we (intentionally) don't for dataclasses; this is somewhat + # inconsistent. + field_list = [] + field_defaults = getattr(cls, "_field_defaults") + + # Note that _field_types is removed in Python 3.9. + for name, typ in _resolver.get_type_hints(cls).items(): + # Get default, with priority for `default_instance`. + default = field_defaults.get(name, MISSING_NONPROP) + if hasattr(default_instance, name): + default = getattr(default_instance, name) + if default_instance is MISSING_PROP: + default = MISSING_PROP + + field_list.append( + FieldDefinition( + name=name, + typ=typ, + default=default, + helptext=_docstrings.get_field_docstring(cls, name), + positional=False, ) - return field_list - elif cls is not None and _resolver.is_dataclass(cls): - # Handle dataclasses. - field_list = [] - for dc_field in filter( - lambda field: field.init, _resolver.resolved_fields(cls) - ): - default = _get_dataclass_field_default(dc_field, default_instance) - field_list.append( - FieldDefinition( - name=dc_field.name, - typ=dc_field.type, - default=default, - helptext=_docstrings.get_field_docstring(cls, dc_field.name), - positional=False, - ) + ) + return field_list + + +def _field_list_from_dataclass( + cls: Type[T], default_instance: DefaultInstanceT +) -> List[FieldDefinition]: + # Handle dataclasses. + field_list = [] + for dc_field in filter(lambda field: field.init, _resolver.resolved_fields(cls)): + default = _get_dataclass_field_default(dc_field, default_instance) + field_list.append( + FieldDefinition( + name=dc_field.name, + typ=dc_field.type, + default=default, + helptext=_docstrings.get_field_docstring(cls, dc_field.name), + positional=False, ) - return field_list - elif _resolver.unwrap_origin(f) is tuple: - # Fixed-length tuples. - field_list = [] - children = get_args(f) - assert Ellipsis not in children - for i, child in enumerate(children): - if default_instance in MISSING_SINGLETONS: - default_i = default_instance - else: - assert isinstance(default_instance, tuple) - assert len(default_instance) == len( - children - ), "Tuple: length of annotation and type don't match." - default_i = default_instance[i] - - field_list.append( - FieldDefinition( - # We'd use an index operator h - name=str(i), - typ=child, - default=default_i, - helptext="", - # This should really be positional=True, but the CLI is more - # intuitive for mixed nested/non-nested types in tuples when we - # stick with kwargs. Tuples are special-cased in _calling.py. - positional=False, - ) + ) + return field_list + + +def _field_list_from_tuple( + f: Callable, default_instance: DefaultInstanceT +) -> List[FieldDefinition]: + # Fixed-length tuples. + field_list = [] + children = get_args(f) + if Ellipsis in children: + raise _instantiators.UnsupportedTypeAnnotationError( + "For nested structures, only fixed-length tuples are allowed." + ) + + if default_instance in MISSING_SINGLETONS: + default_instance = (default_instance,) * len(children) + assert len(default_instance) == len( # type: ignore + children + ), "Tuple: length of annotation and type don't match." + + for i, child in enumerate(children): + default_i = default_instance[i] # type: ignore + field_list.append( + FieldDefinition( + # We'd use an index operator h + name=str(i), + typ=child, + default=default_i, + helptext="", + # This should really be positional=True, but the CLI is more + # intuitive for mixed nested/non-nested types in tuples when we + # stick with kwargs. Tuples are special-cased in _calling.py. + positional=False, ) - return field_list - else: - # Handle general callables. - assert ( - default_instance in MISSING_SINGLETONS - ), "`default_instance` is only supported for dataclass and TypedDict types." - - # Generate field list from function signature. - field_list = [] - params = list(inspect.signature(f).parameters.values()) - if cls is not None: - # Ignore self parameter. - params = params[1:] - - try: - return _field_list_from_params(f, cls, params) - except TypeError as e: - # Try to support passing things like int, str, Dict[K,V], torch.device - # directly into dcargs.cli(). These aren't "type-annotated callables" but - # this a nice-to-have. - param_count = 0 - has_kw_only = False - has_var_positional = False - for param in params: - if ( - param.kind - in ( - inspect.Parameter.POSITIONAL_ONLY, - inspect.Parameter.POSITIONAL_OR_KEYWORD, - ) - and param.default is inspect.Parameter.empty - ): - param_count += 1 - elif param.kind is inspect.Parameter.KEYWORD_ONLY: - has_kw_only = True - elif param.kind is inspect.Parameter.VAR_POSITIONAL: - has_var_positional = True - - if not has_kw_only and ( - param_count == 1 or (param_count == 0 and has_var_positional) + ) + + contains_nested = False + for field in field_list: + contains_nested |= is_possibly_nested_type(field.typ, field.default) + if not contains_nested: + # We could also check for variable length children, which can be populated when + # the tuple is interpreted as a nested field but not a directly parsed one. + raise _instantiators.UnsupportedTypeAnnotationError( + "Tuple does not contain any nested structures." + ) + + return field_list + + +def _field_list_from_general_callable( + f: Callable, + cls: Optional[Type], + default_instance: DefaultInstanceT, + root_field: bool, +) -> List[FieldDefinition]: + # Handle general callables. + if default_instance not in MISSING_SINGLETONS: + raise _instantiators.UnsupportedTypeAnnotationError( + "`default_instance` is not supported in this context." + ) + + # Generate field list from function signature. + if not callable(f): + raise _instantiators.UnsupportedTypeAnnotationError( + f"Cannot extract annotations from {f}, which is not a callable type." + ) + params = list(inspect.signature(f).parameters.values()) + if cls is not None: + # Ignore self parameter. + params = params[1:] + + try: + return _field_list_from_params(f, cls, params) + except (_instantiators.UnsupportedTypeAnnotationError, TypeError) as e: + if not root_field: + raise e + + # Try to support passing things like int, str, Dict[K,V], torch.device + # directly into dcargs.cli(). These aren't "type-annotated callables" but + # this a nice-to-have. + param_count = 0 + has_kw_only = False + has_var_positional = False + for param in params: + if ( + param.kind + in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + ) + and param.default is inspect.Parameter.empty ): - # Things look ok! - if cls is not None: - f = cls - return [ - FieldDefinition( - name=_resolver.unwrap_origin(f).__name__, - typ=cast(Type, f), - default=MISSING_NONPROP, - helptext=None, - positional=True, - ) - ] - else: - raise e + param_count += 1 + elif param.kind is inspect.Parameter.KEYWORD_ONLY: + has_kw_only = True + elif param.kind is inspect.Parameter.VAR_POSITIONAL: + has_var_positional = True + + if not has_kw_only and ( + param_count == 1 or (param_count == 0 and has_var_positional) + ): + # Things look ok! + if cls is not None: + f = cls + return [ + FieldDefinition( + name=_resolver.unwrap_origin(f).__name__, + typ=cast(Type, f), + default=MISSING_NONPROP, + helptext=None, + positional=True, + ) + ] + else: + raise e def _field_list_from_params( @@ -269,7 +382,14 @@ def _field_list_from_params( for param_doc in docstring_parser.parse(docstring).params: docstring_from_arg_name[param_doc.arg_name] = param_doc.description del docstring - hints = get_type_hints(f) + + # This will throw a type error for torch.device, typing.Dict, etc. + try: + hints = get_type_hints(f) + except TypeError: + raise _instantiators.UnsupportedTypeAnnotationError( + f"Could not get hints for {f}!" + ) field_list = [] for param in params: @@ -282,7 +402,7 @@ def _field_list_from_params( helptext = _docstrings.get_field_docstring(cls, param.name) if param.name not in hints: - raise TypeError( + raise _instantiators.UnsupportedTypeAnnotationError( f"Expected fully type-annotated callable, but {f} with arguments" f" {tuple(map(lambda p: p.name, params))} has no annotation for" f" '{param.name}'." diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 78acaf986..51c4c8c02 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -2,27 +2,12 @@ from __future__ import annotations import argparse -import collections.abc import dataclasses -import inspect import itertools -from typing import ( - Any, - Callable, - Dict, - Hashable, - List, - Optional, - Set, - Type, - TypeVar, - Union, - cast, -) +from typing import Any, Callable, Dict, List, Optional, Set, Type, TypeVar, Union, cast import termcolor -import typing_extensions -from typing_extensions import get_args, get_origin, is_typeddict +from typing_extensions import get_args, get_origin from . import ( _argparse_formatter, @@ -37,70 +22,6 @@ T = TypeVar("T") -_known_parsable_types = set( - filter( - lambda x: isinstance(x, Hashable), # type: ignore - itertools.chain( - __builtins__.values(), # type: ignore - vars(typing_extensions).values(), - vars(collections.abc).values(), - ), - ) -) - - -def is_possibly_nested_type(typ: Any) -> bool: - """Heuristics for determining whether a type can be treated as a 'nested type', - where a single field has multiple corresponding arguments (eg for nested - dataclasses or classes). - - Examples of when we return False: int, str, List[int], List[str], pathlib.Path, etc. - """ - - typ_orig = typ - typ = _resolver.unwrap_origin(typ) - - # Nested types should be callable. - if not callable(typ): - return False - - # Simple heuristic: dataclasses should be treated as nested objects and are not - # parsable. - if dataclasses.is_dataclass(typ): - return True - - # TypedDict types can be unpacked. - if is_typeddict(typ): - return True - - # Fixed-length tuple types. - if typ is tuple: - types = get_args(typ_orig) - - # Nested types must be fixed-length. - if Ellipsis in types: - return False - - return any(map(is_possibly_nested_type, types)) - - # Known parsable types: builtins like int/str/float/bool, collections, annotations. - if typ in _known_parsable_types: - return False - - # Nested types like nested (data)classes should have fully type-annotated inputs. If - # any inputs are unannotated (for example, in the case of pathlib.Path), we can - # assume the type is not nested. - try: - sig = inspect.signature(typ) - except ValueError: - return False - for param in sig.parameters.values(): - if param.annotation is inspect.Parameter.empty: - return False - - return True - - @dataclasses.dataclass(frozen=True) class ParserSpecification: """Each parser contains a list of arguments and optionally some subparsers.""" @@ -151,32 +72,28 @@ def from_callable( subparsers_from_name = {} for field in _fields.field_list_from_callable( - f=f, default_instance=default_instance + f=f, default_instance=default_instance, root_field=True ): field = dataclasses.replace( field, - typ=type_from_typevar.get(field.typ, field.typ), # type: ignore + typ=_resolver.type_from_typevar_constraints( + type_from_typevar.get( # type: ignore + field.typ, + field.typ, + ) + ), ) if isinstance(field.typ, TypeVar): - if field.typ.__bound__ is not None: - # Try to infer type from TypeVar bound. - field = dataclasses.replace(field, typ=field.typ.__bound__) - elif len(field.typ.__constraints__) > 0: - # Try to infer type from TypeVar constraints. - field = dataclasses.replace( - field, typ=Union.__getitem__(field.typ.__constraints__) # type: ignore - ) - else: - # Found an unbound TypeVar. This could be because inheriting from - # generics is currently not implemented. It's unclear whether this is - # feasible, because generics are lost in the mro: - # https://github.com/python/typing/issues/777 - raise _instantiators.UnsupportedTypeAnnotationError( - f"Field {field.name} has an unbound TypeVar: {field.typ}. Note" - " that inheriting from generics is currently not implemented." - " It's unclear whether this is feasible, because generics are" - " lost in the mro: https://github.com/python/typing/issues/777" - ) + # Found an unbound TypeVar. This could be because inheriting from + # generics is currently not implemented. It's unclear whether this is + # feasible, because generics are lost in the mro: + # https://github.com/python/typing/issues/777 + raise _instantiators.UnsupportedTypeAnnotationError( + f"Field {field.name} has an unbound TypeVar: {field.typ}. Note" + " that inheriting from generics is currently not implemented." + " It's unclear whether this is feasible, because generics are" + " lost in the mro: https://github.com/python/typing/issues/777" + ) # (1) Handle Unions over callables; these result in subparsers. subparsers_attempt = SubparsersSpecification.from_field( @@ -199,10 +116,10 @@ def from_callable( continue else: field = dataclasses.replace(field, typ=type(field.default)) - assert is_possibly_nested_type(field.typ) + assert _fields.is_possibly_nested_type(field.typ, field.default) # (2) Handle nested callables. - if is_possibly_nested_type(field.typ): + if _fields.is_possibly_nested_type(field.typ, field.default): nested_parser = ParserSpecification.from_callable( field.typ, description=None, @@ -376,7 +293,12 @@ def from_field( # We don't use sets here to retain order of subcommands. options = [type_from_typevar.get(typ, typ) for typ in get_args(field.typ)] options_no_none = [o for o in options if o != type(None)] # noqa - if not all(map(is_possibly_nested_type, options_no_none)): + if not all( + [ + _fields.is_possibly_nested_type(o, _fields.MISSING_NONPROP) + for o in options_no_none + ] + ): return None parser_from_name: Dict[str, ParserSpecification] = {} diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 2ef8b399c..0e4719e0d 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -64,3 +64,16 @@ def is_namedtuple(cls: Type) -> bool: # and hasattr(cls, "_field_types") and hasattr(cls, "_field_defaults") ) + + +def type_from_typevar_constraints(typ: Union[Type, TypeVar]) -> Union[Type, TypeVar]: + """Try to concretize a type from a TypeVar's bounds or constraints. Identity if + unsuccessful.""" + if isinstance(typ, TypeVar): + if typ.__bound__ is not None: + # Try to infer type from TypeVar bound. + return typ.__bound__ + elif len(typ.__constraints__) > 0: + # Try to infer type from TypeVar constraints. + return Union.__getitem__(typ.__constraints__) # type: ignore + return typ diff --git a/tests/test_errors.py b/tests/test_errors.py index aa6b88277..a3fed8b8e 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -118,7 +118,7 @@ def test_missing_annotation_1(): def main(a, b) -> None: pass - with pytest.raises(TypeError): + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): dcargs.cli(main, args=["--help"]) @@ -126,5 +126,5 @@ def test_missing_annotation_2(): def main(*, a) -> None: pass - with pytest.raises(TypeError): + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): dcargs.cli(main, args=["--help"]) From 789c3edb96a963cc959945240d8863181d9a69f5 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 12 Aug 2022 14:36:48 -0700 Subject: [PATCH 085/491] Draft nested structures in lists, tuples, dicts, etc --- dcargs/_calling.py | 16 +- dcargs/_docstrings.py | 5 +- dcargs/_fields.py | 196 +++++++++++++++--- dcargs/_parsers.py | 6 +- dcargs/_strings.py | 5 +- ...ainers.rst => 03_enums_and_containers.rst} | 0 ...onfigs.rst => 05_hierarchical_configs.rst} | 3 +- ...6_base configs.rst => 06_base_configs.rst} | 0 ... unions.rst => 07_literals_and_unions.rst} | 0 ...tional args.rst => 08_positional_args.rst} | 0 ...parsers.rst => 10_multiple_subparsers.rst} | 0 docs/source/examples/12_tuples.rst | 9 +- ...rd classes.rst => 13_standard_classes.rst} | 0 .../examples/15_nesting_in_containers.rst | 74 +++++++ docs/update_example_docs.py | 10 +- examples/05_hierarchical_configs.py | 3 +- examples/12_tuples.py | 7 +- examples/15_nesting_in_containers.py | 58 ++++++ tests/test_dcargs.py | 22 +- tests/test_errors.py | 7 +- tests/test_nested_in_containers.py | 142 +++++++++++++ 21 files changed, 499 insertions(+), 64 deletions(-) rename docs/source/examples/{03_enums and containers.rst => 03_enums_and_containers.rst} (100%) rename docs/source/examples/{05_hierarchical configs.rst => 05_hierarchical_configs.rst} (96%) rename docs/source/examples/{06_base configs.rst => 06_base_configs.rst} (100%) rename docs/source/examples/{07_literals and unions.rst => 07_literals_and_unions.rst} (100%) rename docs/source/examples/{08_positional args.rst => 08_positional_args.rst} (100%) rename docs/source/examples/{10_multiple subparsers.rst => 10_multiple_subparsers.rst} (100%) rename docs/source/examples/{13_standard classes.rst => 13_standard_classes.rst} (100%) create mode 100644 docs/source/examples/15_nesting_in_containers.rst create mode 100644 examples/15_nesting_in_containers.py create mode 100644 tests/test_nested_in_containers.py diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 7aac344d9..2581fb099 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -3,7 +3,7 @@ from __future__ import annotations -from typing import Any, Callable, Dict, List, Set, Tuple, TypeVar, Union +from typing import Any, Callable, Dict, List, Sequence, Set, Tuple, TypeVar, Union from typing_extensions import get_args, get_origin @@ -174,8 +174,16 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: kwargs[field.name] = value unwrapped_f = _resolver.unwrap_origin(f) - if unwrapped_f is tuple: - assert len(args) == 0 - return tuple(kwargs.values()), consumed_keywords # type: ignore + unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore + if unwrapped_f in (tuple, list, set): + if len(args) == 0: + # When tuples are used as nested structures (eg Tuple[SomeDataclass]), we + # use keyword arguments. + return unwrapped_f(kwargs.values()), consumed_keywords # type: ignore + else: + # When tuples are directly parsed (eg Tuple[int, int]), we end up with a + # single set of positional arguments. + assert len(args) == 1 + return unwrapped_f(args[0]), consumed_keywords # type: ignore else: return unwrapped_f(*args, **kwargs), consumed_keywords # type: ignore diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 2b6e5c8e8..8dda6974b 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -217,7 +217,10 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: break # Record single comments! - if len(actual_line_tokens) == 1: + if ( + len(actual_line_tokens) == 1 + and actual_line_tokens[0].token_type is tokenize.COMMENT + ): (comment_token,) = actual_line_tokens assert comment_token.content.startswith("#") comments.append(comment_token.content[1:].strip()) diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 826e27106..07d87dbf2 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -11,22 +11,31 @@ Any, Callable, Hashable, + Iterable, List, Optional, Type, TypeVar, Union, cast, - get_args, ) import docstring_parser import typing_extensions -from typing_extensions import get_type_hints, is_typeddict +from typing_extensions import get_args, get_type_hints, is_typeddict from . import _docstrings, _instantiators, _resolver +class UnsupportedNestedTypeAnnotationError( + _instantiators.UnsupportedTypeAnnotationError +): + """Narrower version of UnsupportedNestedTypeAnnotationError, which signifies that a type + cannot correspond to a nested field. It may still be parsed directly.""" + + pass + + @dataclasses.dataclass(frozen=True) class FieldDefinition: name: str @@ -108,16 +117,16 @@ class ExcludeFromCallType(_Singleton): ) -def is_possibly_nested_type(typ: Any, default_instance: Any) -> bool: - """Determine whether a type can be treated as a 'nested type', where a single field +def is_nested_type(typ: Any, default_instance: Any) -> bool: + """Determine whether a type should be treated as a 'nested type', where a single field has multiple corresponding arguments (eg for nested dataclasses or classes).""" try: # This implementation will result in some computational redundancy, but is nice - # for reusing logic. Should be revisited. + # for reusing logic. Could be revisited. field_list_from_callable(typ, default_instance, root_field=False) return True - except _instantiators.UnsupportedTypeAnnotationError: + except UnsupportedNestedTypeAnnotationError: return False @@ -138,6 +147,18 @@ def field_list_from_callable( # Unwrap generics. f, type_from_typevar = _resolver.resolve_generic_types(f) + # Type narrowing: if we annotate as Animal but specify a default instance of Cat, we + # should parse as Cat. + # + # TODO: this will not currently handle generics correctly. We should write tests for + # this. + try: + potential_subclass = type(default_instance) + if issubclass(potential_subclass, cast(Type, f)): + f = potential_subclass # type: ignore + except TypeError: + pass + # If `f` is a type: # 1. Set cls to the type. # 2. Consider `f` to be `cls.__init__`. @@ -145,8 +166,10 @@ def field_list_from_callable( if isinstance(f, type): cls = f f = cls.__init__ # type: ignore + f_origin: Callable = cls + f_origin = _resolver.unwrap_origin(f) - # Handle each supported nested structure type. + # Try special cases. if cls is not None and is_typeddict(cls): return _field_list_from_typeddict(cls, default_instance) @@ -156,17 +179,48 @@ def field_list_from_callable( elif cls is not None and _resolver.is_dataclass(cls): return _field_list_from_dataclass(cls, default_instance) - elif _resolver.unwrap_origin(f) is tuple: - assert cls is None - return _field_list_from_tuple(f, default_instance) + # Standard container types. These are special because they can be nested structures + # when + try: + # Note that f_origin will be populated if we annotate as `Tuple[..]`, and cls will + # be populated if we annotated as just `tuple`. + if f_origin is tuple or cls is tuple: + out = _field_list_from_tuple(f, default_instance) + return out + + elif f_origin in (list, set, typing.Sequence) or cls in ( + list, + set, + typing.Sequence, + ): + if len(get_args(f)) == 0: + if default_instance in MISSING_SINGLETONS: + raise _instantiators.UnsupportedTypeAnnotationError( + f"Sequence type {cls} needs either an explicit type or a" + " default to infer from." + ) + assert isinstance(default_instance, Iterable) + contained_type = MISSING_NONPROP + else: + (contained_type,) = get_args(f) + f_origin = list if f_origin is typing.Sequence else f_origin # type: ignore + return _field_list_from_sequence( + contained_type, # type: ignore + default_instance, + ) + elif f_origin is dict: + return _field_list_from_dict(f, default_instance) + except UnsupportedNestedTypeAnnotationError as e: + # For the root field case, we can try again in the general case. + if not root_field: + raise e - elif not root_field and ( + # General cases. + if not root_field and ( (cls is not None and cls in _known_parsable_types) or _resolver.unwrap_origin(f) in _known_parsable_types ): - raise _instantiators.UnsupportedTypeAnnotationError( - f"{f} cannot be interpreted as a nested structure!" - ) + raise UnsupportedNestedTypeAnnotationError(f"{f} should be parsed directly!") else: return _field_list_from_general_callable( @@ -188,8 +242,8 @@ def _field_list_from_typeddict( default = default_instance.get(name, MISSING_PROP) # type: ignore elif getattr(cls, "__total__") is False: default = EXCLUDE_FROM_CALL - if is_possibly_nested_type(typ, MISSING_NONPROP): - raise _instantiators.UnsupportedTypeAnnotationError( + if is_nested_type(typ, MISSING_NONPROP): + raise UnsupportedNestedTypeAnnotationError( "`total=False` not supported for nested structures." ) else: @@ -265,15 +319,24 @@ def _field_list_from_tuple( field_list = [] children = get_args(f) if Ellipsis in children: - raise _instantiators.UnsupportedTypeAnnotationError( - "For nested structures, only fixed-length tuples are allowed." + return _field_list_from_sequence( + next(iter(set(children) - {Ellipsis})), default_instance ) + # Infer more specific type when tuple annotation isn't subscripted. This generally + # doesn't happen + if len(children) == 0: + if default_instance in MISSING_SINGLETONS: + raise _instantiators.UnsupportedTypeAnnotationError( + "If contained types of a tuple are not specified in the annotation, a" + " default instance must be specified." + ) + else: + assert isinstance(default_instance, tuple) + children = tuple(type(x) for x in default_instance) + if default_instance in MISSING_SINGLETONS: default_instance = (default_instance,) * len(children) - assert len(default_instance) == len( # type: ignore - children - ), "Tuple: length of annotation and type don't match." for i, child in enumerate(children): default_i = default_instance[i] # type: ignore @@ -293,17 +356,89 @@ def _field_list_from_tuple( contains_nested = False for field in field_list: - contains_nested |= is_possibly_nested_type(field.typ, field.default) + contains_nested |= is_nested_type(field.typ, field.default) if not contains_nested: # We could also check for variable length children, which can be populated when # the tuple is interpreted as a nested field but not a directly parsed one. - raise _instantiators.UnsupportedTypeAnnotationError( + raise UnsupportedNestedTypeAnnotationError( "Tuple does not contain any nested structures." ) return field_list +def _field_list_from_sequence( + contained_type: Type, + default_instance: DefaultInstanceT, +) -> List[FieldDefinition]: + # When no default instance is specified: + # If we have List[int] => this can be parsed as a single field. + # If we have List[SomeStruct] => OK. + if default_instance in MISSING_SINGLETONS and not is_nested_type( + contained_type, MISSING_NONPROP + ): + raise UnsupportedNestedTypeAnnotationError( + f"Sequence containing type {contained_type} should be parsed directly!" + ) + + # If we have a default instance: + # [int, int, int] => this can be parsed as a single field. + # [SomeStruct, int, int] => OK. + if isinstance(default_instance, Iterable) and all( + [not is_nested_type(type(x), x) for x in default_instance] + ): + raise UnsupportedNestedTypeAnnotationError( + f"Sequence with default {default_instance} should be parsed directly!" + ) + if default_instance in MISSING_SINGLETONS: + # We use the broader error type to prevent it from being caught by + # is_possibly_nested_type(). This is for sure a bad annotation! + raise _instantiators.UnsupportedTypeAnnotationError( + "For variable-length sequences over nested types, we need a default value" + " to infer length from." + ) + + field_list = [] + for i, default_i in enumerate(default_instance): # type: ignore + field_list.append( + FieldDefinition( + # We'd use an index operator h + name=str(i), + # This will currently break for generics...! + typ=contained_type + if contained_type not in MISSING_SINGLETONS + else type(default_i), + default=default_i, + helptext="", + positional=False, + ) + ) + return field_list + + +def _field_list_from_dict( + f: Callable, + default_instance: DefaultInstanceT, +) -> List[FieldDefinition]: + if default_instance in MISSING_SINGLETONS: + raise UnsupportedNestedTypeAnnotationError( + "Nested dictionary structures must have a default instance specified." + ) + field_list = [] + for k, v in cast(dict, default_instance).items(): + field_list.append( + FieldDefinition( + name=str(k), + # TODO: this will fail for generic types. + typ=type(v), + default=v, + helptext=None, + positional=False, + ) + ) + return field_list + + def _field_list_from_general_callable( f: Callable, cls: Optional[Type], @@ -312,13 +447,14 @@ def _field_list_from_general_callable( ) -> List[FieldDefinition]: # Handle general callables. if default_instance not in MISSING_SINGLETONS: - raise _instantiators.UnsupportedTypeAnnotationError( - "`default_instance` is not supported in this context." + raise UnsupportedNestedTypeAnnotationError( + "`default_instance` is supported only for select types:" + " dataclasses, lists, NamedTuple, TypedDict, etc." ) # Generate field list from function signature. if not callable(f): - raise _instantiators.UnsupportedTypeAnnotationError( + raise UnsupportedNestedTypeAnnotationError( f"Cannot extract annotations from {f}, which is not a callable type." ) params = list(inspect.signature(f).parameters.values()) @@ -328,7 +464,7 @@ def _field_list_from_general_callable( try: return _field_list_from_params(f, cls, params) - except (_instantiators.UnsupportedTypeAnnotationError, TypeError) as e: + except (UnsupportedNestedTypeAnnotationError, TypeError) as e: if not root_field: raise e @@ -387,9 +523,7 @@ def _field_list_from_params( try: hints = get_type_hints(f) except TypeError: - raise _instantiators.UnsupportedTypeAnnotationError( - f"Could not get hints for {f}!" - ) + raise UnsupportedNestedTypeAnnotationError(f"Could not get hints for {f}!") field_list = [] for param in params: @@ -402,7 +536,7 @@ def _field_list_from_params( helptext = _docstrings.get_field_docstring(cls, param.name) if param.name not in hints: - raise _instantiators.UnsupportedTypeAnnotationError( + raise UnsupportedNestedTypeAnnotationError( f"Expected fully type-annotated callable, but {f} with arguments" f" {tuple(map(lambda p: p.name, params))} has no annotation for" f" '{param.name}'." diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 51c4c8c02..195ab2cfd 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -116,10 +116,10 @@ def from_callable( continue else: field = dataclasses.replace(field, typ=type(field.default)) - assert _fields.is_possibly_nested_type(field.typ, field.default) + assert _fields.is_nested_type(field.typ, field.default) # (2) Handle nested callables. - if _fields.is_possibly_nested_type(field.typ, field.default): + if _fields.is_nested_type(field.typ, field.default): nested_parser = ParserSpecification.from_callable( field.typ, description=None, @@ -295,7 +295,7 @@ def from_field( options_no_none = [o for o in options if o != type(None)] # noqa if not all( [ - _fields.is_possibly_nested_type(o, _fields.MISSING_NONPROP) + _fields.is_nested_type(o, _fields.MISSING_NONPROP) for o in options_no_none ] ): diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 61e8e526f..e639e09af 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -17,9 +17,8 @@ def make_field_name(parts: Sequence[str]) -> str: out: List[str] = [] for i, p in enumerate([p for p in parts if len(p) > 0]): if i > 0: - # Delimeter between parts. We use a colon before integers, which can - # currently only come from indices! (since field names can't start with - # digits) + # Delimeter between parts. We use a colon before integers, which are + # typically indices. out.append(":" if p[0].isdigit() else ".") out.append(p) diff --git a/docs/source/examples/03_enums and containers.rst b/docs/source/examples/03_enums_and_containers.rst similarity index 100% rename from docs/source/examples/03_enums and containers.rst rename to docs/source/examples/03_enums_and_containers.rst diff --git a/docs/source/examples/05_hierarchical configs.rst b/docs/source/examples/05_hierarchical_configs.rst similarity index 96% rename from docs/source/examples/05_hierarchical configs.rst rename to docs/source/examples/05_hierarchical_configs.rst index 031133c71..006bd1ab5 100644 --- a/docs/source/examples/05_hierarchical configs.rst +++ b/docs/source/examples/05_hierarchical_configs.rst @@ -16,7 +16,6 @@ objects. This helps with modularity and grouping in larger projects. import dataclasses import enum import pathlib - from typing import Tuple import dcargs @@ -41,7 +40,7 @@ objects. This helps with modularity and grouping in larger projects. @dataclasses.dataclass class ExperimentConfig: # Various configurable options for our optimizer. - optimizer: Tuple[OptimizerConfig, OptimizerConfig, OptimizerConfig] + optimizer: OptimizerConfig # Batch size. batch_size: int = 32 diff --git a/docs/source/examples/06_base configs.rst b/docs/source/examples/06_base_configs.rst similarity index 100% rename from docs/source/examples/06_base configs.rst rename to docs/source/examples/06_base_configs.rst diff --git a/docs/source/examples/07_literals and unions.rst b/docs/source/examples/07_literals_and_unions.rst similarity index 100% rename from docs/source/examples/07_literals and unions.rst rename to docs/source/examples/07_literals_and_unions.rst diff --git a/docs/source/examples/08_positional args.rst b/docs/source/examples/08_positional_args.rst similarity index 100% rename from docs/source/examples/08_positional args.rst rename to docs/source/examples/08_positional_args.rst diff --git a/docs/source/examples/10_multiple subparsers.rst b/docs/source/examples/10_multiple_subparsers.rst similarity index 100% rename from docs/source/examples/10_multiple subparsers.rst rename to docs/source/examples/10_multiple_subparsers.rst diff --git a/docs/source/examples/12_tuples.rst b/docs/source/examples/12_tuples.rst index 70d8342b7..062d2b4d2 100644 --- a/docs/source/examples/12_tuples.rst +++ b/docs/source/examples/12_tuples.rst @@ -12,14 +12,13 @@ Example using ``dcargs.cli()`` to instantiate tuple types. .. code-block:: python :linenos: - import dataclasses from typing import NamedTuple, Tuple import dcargs - @dataclasses.dataclass - class Color: + # Named tuples are interpreted as nested structures. + class Color(NamedTuple): r: int g: int b: int @@ -61,6 +60,6 @@ Example using ``dcargs.cli()`` to instantiate tuple types. .. raw:: html - python 12_tuples.py '--two_colors[1].r' 127 '--two_colors[1].g' 0 '--two_colors[1].b' 0 + python 12_tuples.py --two_colors:1.r 127 --two_colors:1.g 0 --two_colors:1.b 0 -.. program-output:: python ../../examples/12_tuples.py '--two_colors[1].r' 127 '--two_colors[1].g' 0 '--two_colors[1].b' 0 +.. program-output:: python ../../examples/12_tuples.py --two_colors:1.r 127 --two_colors:1.g 0 --two_colors:1.b 0 diff --git a/docs/source/examples/13_standard classes.rst b/docs/source/examples/13_standard_classes.rst similarity index 100% rename from docs/source/examples/13_standard classes.rst rename to docs/source/examples/13_standard_classes.rst diff --git a/docs/source/examples/15_nesting_in_containers.rst b/docs/source/examples/15_nesting_in_containers.rst new file mode 100644 index 000000000..31883dd5a --- /dev/null +++ b/docs/source/examples/15_nesting_in_containers.rst @@ -0,0 +1,74 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +15. Nesting In Containers +========================================== + + +Structures can be nested inside of standard containers. + +Note that lengths must be inferrable, either via a fixed-length tuple annotation or by +parsing default values. + + + +.. code-block:: python + :linenos: + + import dataclasses + from typing import Dict, Tuple + + import dcargs + + + class Color: + pass + + + @dataclasses.dataclass + class RGB(Color): + r: int + g: int + b: int + + + @dataclasses.dataclass + class HSL(Color): + h: int + s: int + l: int + + + @dataclasses.dataclass + class Args: + # Example of specifying nested structures via a fixed-length tuple. + color_tuple: Tuple[RGB, HSL] + + # Examples of nested structures in variable-length containers. These need a default + # provided for length inference -- we don't currently support specifying dynamic + # container lengths directly from the commandline. + color_tuple_alt: Tuple[Color, ...] = ( + RGB(255, 0, 0), + HSL(0, 255, 0), + ) + color_map: Dict[str, RGB] = dataclasses.field( + # Note that we can't use mutable values as defaults directly. + default_factory={ + "red": RGB(255, 0, 0), + "green": RGB(0, 255, 0), + "blue": RGB(0, 0, 255), + }.copy + ) + + + if __name__ == "__main__": + args = dcargs.cli(Args) + print(args) + +------------ + +.. raw:: html + + python 15_nesting_in_containers.py --help + +.. program-output:: python ../../examples/15_nesting_in_containers.py --help diff --git a/docs/update_example_docs.py b/docs/update_example_docs.py index c0267b674..6cc4a19f1 100644 --- a/docs/update_example_docs.py +++ b/docs/update_example_docs.py @@ -5,6 +5,7 @@ import dataclasses import pathlib import shlex +import shutil from typing import Iterable import m2r2 @@ -68,6 +69,10 @@ def main( examples_dir: pathlib.Path = REPO_ROOT / "examples", sphinx_source_dir: pathlib.Path = REPO_ROOT / "docs" / "source", ) -> None: + example_doc_dir = sphinx_source_dir / "examples" + shutil.rmtree(example_doc_dir) + example_doc_dir.mkdir() + for path in get_example_paths(examples_dir): ex = ExampleMetadata.from_path(path) path_for_sphinx = pathlib.Path("..") / ".." / path.relative_to(REPO_ROOT) @@ -100,9 +105,8 @@ def main( ] ( - sphinx_source_dir - / "examples" - / f"{ex.index_with_zero}_{ex.title.lower()}.rst" + example_doc_dir + / f"{ex.index_with_zero}_{ex.title.lower().replace(' ', '_')}.rst" ).write_text( "\n".join( [ diff --git a/examples/05_hierarchical_configs.py b/examples/05_hierarchical_configs.py index 164282634..c0a1be98c 100644 --- a/examples/05_hierarchical_configs.py +++ b/examples/05_hierarchical_configs.py @@ -10,7 +10,6 @@ import dataclasses import enum import pathlib -from typing import Tuple import dcargs @@ -35,7 +34,7 @@ class OptimizerConfig: @dataclasses.dataclass class ExperimentConfig: # Various configurable options for our optimizer. - optimizer: Tuple[OptimizerConfig, OptimizerConfig, OptimizerConfig] + optimizer: OptimizerConfig # Batch size. batch_size: int = 32 diff --git a/examples/12_tuples.py b/examples/12_tuples.py index f9adb5664..ed9005114 100644 --- a/examples/12_tuples.py +++ b/examples/12_tuples.py @@ -3,17 +3,16 @@ Usage: `python ./12_tuples.py --help` `python ./12_tuples.py --color 127 127 127` -`python ./12_tuples.py --two_colors[1].r 127 --two_colors[1].g 0 --two_colors[1].b 0` +`python ./12_tuples.py --two_colors:1.r 127 --two_colors:1.g 0 --two_colors:1.b 0` """ -import dataclasses from typing import NamedTuple, Tuple import dcargs -@dataclasses.dataclass -class Color: +# Named tuples are interpreted as nested structures. +class Color(NamedTuple): r: int g: int b: int diff --git a/examples/15_nesting_in_containers.py b/examples/15_nesting_in_containers.py new file mode 100644 index 000000000..b1378270f --- /dev/null +++ b/examples/15_nesting_in_containers.py @@ -0,0 +1,58 @@ +"""Structures can be nested inside of standard containers. + +Note that lengths must be inferrable, either via a fixed-length tuple annotation or by +parsing default values. + + +Usage: +`python ./15_nesting_in_containers.py.py --help` +""" +import dataclasses +from typing import Dict, Tuple + +import dcargs + + +class Color: + pass + + +@dataclasses.dataclass +class RGB(Color): + r: int + g: int + b: int + + +@dataclasses.dataclass +class HSL(Color): + h: int + s: int + l: int + + +@dataclasses.dataclass +class Args: + # Example of specifying nested structures via a fixed-length tuple. + color_tuple: Tuple[RGB, HSL] + + # Examples of nested structures in variable-length containers. These need a default + # provided for length inference -- we don't currently support specifying dynamic + # container lengths directly from the commandline. + color_tuple_alt: Tuple[Color, ...] = ( + RGB(255, 0, 0), + HSL(0, 255, 0), + ) + color_map: Dict[str, RGB] = dataclasses.field( + # We can't use mutable values as defaults directly. + default_factory={ + "red": RGB(255, 0, 0), + "green": RGB(0, 255, 0), + "blue": RGB(0, 0, 255), + }.copy + ) + + +if __name__ == "__main__": + args = dcargs.cli(Args) + print(args) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index cf44901ea..e01c86b64 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -3,7 +3,18 @@ import dataclasses import enum import pathlib -from typing import Any, AnyStr, Callable, ClassVar, Dict, List, Optional, TypeVar, Union +from typing import ( + Any, + AnyStr, + Callable, + ClassVar, + Dict, + List, + Optional, + Tuple, + TypeVar, + Union, +) import pytest import torch @@ -487,6 +498,15 @@ def test_just_list(): assert dcargs.cli(List[int], args="1 2 3 4".split(" ")) == [1, 2, 3, 4] +def test_just_tuple(): + assert dcargs.cli(Tuple[int, int, int, int], args="1 2 3 4".split(" ")) == ( + 1, + 2, + 3, + 4, + ) + + def test_return_parser(): def main() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() diff --git a/tests/test_errors.py b/tests/test_errors.py index a3fed8b8e..35f64f623 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -68,11 +68,8 @@ class OneStringArg: def main(arg: List[OneStringArg]) -> List[OneStringArg]: # type: ignore return arg - assert dcargs.cli(main, args=["--arg", "0", "1", "2"]) == [ - OneStringArg("0"), - OneStringArg("1"), - OneStringArg("2"), - ] + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli(main, args=["--arg", "0", "1", "2"]) @dataclasses.dataclass class TwoStringArg: diff --git a/tests/test_nested_in_containers.py b/tests/test_nested_in_containers.py new file mode 100644 index 000000000..93697114a --- /dev/null +++ b/tests/test_nested_in_containers.py @@ -0,0 +1,142 @@ +import dataclasses +from typing import Any, Dict, List, Tuple + +import pytest + +import dcargs + + +@dataclasses.dataclass +class Color: + r: int + g: int + b: int + + +def test_nested_tuple_fixed_single(): + def main(x: Tuple[Color]) -> Any: + return x + + assert dcargs.cli(main, args="--x:0.r 255 --x:0.g 127 --x:0.b 5".split(" ")) == ( + Color(255, 127, 5), + ) + + +def test_nested_tuple_fixed_two(): + def main(x: Tuple[Color, Color]) -> Any: + return x + + assert dcargs.cli( + main, + args=( + "--x:0.r 255 --x:0.g 127 --x:0.b 5 --x:1.r 255 --x:1.g 127 --x:1.b 0" + ).split(" "), + ) == ( + Color(255, 127, 5), + Color(255, 127, 0), + ) + + +def test_nested_tuple_fixed_three(): + def main(x: Tuple[Color, int, Color]) -> Any: + return x + + assert dcargs.cli( + main, + args=( + "--x:0.r 255 --x:0.g 127 --x:0.b 5 --x:1 94709 --x:2.r 255 --x:2.g 127" + " --x:2.b 0" + ).split(" "), + ) == ( + Color(255, 127, 5), + 94709, + Color(255, 127, 0), + ) + + +def test_nested_tuple_recursive(): + def main(x: Tuple[Color, Tuple[Color, Color]]) -> Any: + return x + + assert dcargs.cli( + main, + args=( + "--x:0.r 255 --x:0.g 127 --x:0.b 5 --x:1:0.r 255 --x:1:0.g 127 --x:1:0.b 0" + " --x:1:1.r 255 --x:1:1.g 127 --x:1:1.b 0" + ).split(" "), + ) == ( + Color(255, 127, 5), + ( + Color(255, 127, 0), + Color(255, 127, 0), + ), + ) + + +def test_tuple_bad(): + # Unable to infer input length. + def main(x: Tuple[Color, ...]) -> None: + pass + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli(main, args=[]) + + +def test_list_bad(): + # Unable to infer input length. + def main(x: List[Color]) -> None: + pass + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli(main, args=[]) + + +def test_list_ok(): + def main(x: List[Color] = [Color(255, 0, 0)]) -> Any: + return x + + assert dcargs.cli(main, args=[]) == [Color(255, 0, 0)] + assert dcargs.cli(main, args="--x:0.r 127".split(" ")) == [Color(127, 0, 0)] + + +def test_tuple_in_list(): + def main(x: List[Tuple[Color]] = [(Color(255, 0, 0),)]) -> Any: + return x + + assert dcargs.cli(main, args=[]) == [(Color(255, 0, 0),)] + assert dcargs.cli(main, args="--x:0:0.r 127".split(" ")) == [(Color(127, 0, 0),)] + + +def test_tuple_variable(): + def main(x: Tuple[Color, ...] = (Color(255, 0, 0), Color(255, 0, 127))) -> Any: + return x + + assert dcargs.cli(main, args=[]) == (Color(255, 0, 0), Color(255, 0, 127)) + assert dcargs.cli(main, args="--x:0.r 127".split(" ")) == ( + Color(127, 0, 0), + Color(255, 0, 127), + ) + + +def test_dict_bad(): + def main(x: Dict[str, Color]) -> Any: + return x + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli(main, args=[]) + + +def test_dict_ok(): + def main( + x: Dict[str, Color] = { + "red": Color(255, 0, 0), + "green": Color(0, 255, 0), + "blue": Color(0, 0, 255), + } + ) -> Any: + return x + + assert dcargs.cli(main, args=[])["green"] == Color(0, 255, 0) + assert dcargs.cli(main, args="--x.green.g 127".split(" "))["green"] == Color( + 0, 127, 0 + ) From f2dc76f3e99d958e8d9e08019eaa07085a270df7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 12 Aug 2022 23:16:22 -0700 Subject: [PATCH 086/491] Documentation updates --- README.md | 30 ++++++++++++++++--- docs/source/examples/01_functions.rst | 9 ------ docs/source/examples/12_tuples.rst | 4 +-- .../examples/15_nesting_in_containers.rst | 4 +-- docs/source/index.rst | 23 +++++++++++--- examples/01_functions.py | 2 -- examples/12_tuples.py | 2 +- examples/15_nesting_in_containers.py | 2 +- 8 files changed, 51 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 1aa9a0a68..07c04092b 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,11 @@

Our core interface, dcargs.cli(), generates argument parsers from type-annotated -
callables: functions, classes, dataclasses, and nested dataclasses and classes. +
callables: functions, dataclasses, classes, and nested dataclasses and classes.

- This can be used as a drop-in replacement for argparse: + This can be used as a replacement for argparse:

@@ -36,13 +36,13 @@ -
with argparse with dcargs
```python -import argparse +"""Sum two numbers from argparse.""" +import argparse parser = argparse.ArgumentParser() parser.add_argument( "--a", @@ -63,6 +63,9 @@ print(args.a + args.b) ```python +"""Sum two numbers by calling a +function with dcargs.""" + import dcargs def main(a: int, b: int = 3) -> None: @@ -71,6 +74,25 @@ def main(a: int, b: int = 3) -> None: dcargs.cli(main) ``` +--- + +```python +"""Sum two numbers by instantiating +a dataclass with dcargs.""" + +from dataclasses import dataclass + +import dcargs + +@dataclass +class Args: + a: int + b: int = 3 + +args = dcargs.cli(Args) +print(args.a + args.b) +``` +
diff --git a/docs/source/examples/01_functions.rst b/docs/source/examples/01_functions.rst index 2fe87aa42..def4a7c8d 100644 --- a/docs/source/examples/01_functions.rst +++ b/docs/source/examples/01_functions.rst @@ -13,15 +13,6 @@ populated from the CLI. .. code-block:: python :linenos: - In the simplest case, `dcargs.cli()` can be used to run a function with arguments - populated from the CLI. - - Usage: - `python ./01_functions.py --help` - `python ./01_functions.py --field1 hello` - `python ./01_functions.py --field1 hello --field2 10` - """ - import dcargs diff --git a/docs/source/examples/12_tuples.rst b/docs/source/examples/12_tuples.rst index 062d2b4d2..08af30686 100644 --- a/docs/source/examples/12_tuples.rst +++ b/docs/source/examples/12_tuples.rst @@ -60,6 +60,6 @@ Example using ``dcargs.cli()`` to instantiate tuple types. .. raw:: html - python 12_tuples.py --two_colors:1.r 127 --two_colors:1.g 0 --two_colors:1.b 0 + python 12_tuples.py --two-colors:1.r 127 --two-colors:1.g 0 --two-colors:1.b 0 -.. program-output:: python ../../examples/12_tuples.py --two_colors:1.r 127 --two_colors:1.g 0 --two_colors:1.b 0 +.. program-output:: python ../../examples/12_tuples.py --two-colors:1.r 127 --two-colors:1.g 0 --two-colors:1.b 0 diff --git a/docs/source/examples/15_nesting_in_containers.rst b/docs/source/examples/15_nesting_in_containers.rst index 31883dd5a..f669bc8c1 100644 --- a/docs/source/examples/15_nesting_in_containers.rst +++ b/docs/source/examples/15_nesting_in_containers.rst @@ -45,14 +45,14 @@ parsing default values. color_tuple: Tuple[RGB, HSL] # Examples of nested structures in variable-length containers. These need a default - # provided for length inference -- we don't currently support specifying dynamic + # provided for length inference; we don't currently support specifying dynamic # container lengths directly from the commandline. color_tuple_alt: Tuple[Color, ...] = ( RGB(255, 0, 0), HSL(0, 255, 0), ) color_map: Dict[str, RGB] = dataclasses.field( - # Note that we can't use mutable values as defaults directly. + # We can't use mutable values as defaults directly. default_factory={ "red": RGB(255, 0, 0), "green": RGB(0, 255, 0), diff --git a/docs/source/index.rst b/docs/source/index.rst index b8fbb573a..e8eb035ad 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -9,13 +9,12 @@ Our core interface, :func:`dcargs.cli()`, generates argument parsers from type-annotated callables: functions, classes, dataclasses, and *nested* dataclasses and classes. -This can be used as a drop-in replacement for :code:`argparse`: - +This can be used as a replacement for :code:`argparse`: .. code-block:: - # With argparse. + """Sum two numbers with argparse.""" import argparse @@ -29,7 +28,7 @@ This can be used as a drop-in replacement for :code:`argparse`: .. code-block:: - # With dcargs. + """Sum two numbers by calling a function with dcargs.""" import dcargs @@ -39,6 +38,22 @@ This can be used as a drop-in replacement for :code:`argparse`: dcargs.cli(main) +.. code-block:: + + """Sum two numbers by instantiating a dataclass with dcargs.""" + + from dataclasses import dataclass + + import dcargs + + @dataclass + class Args: + a: int + b: int = 3 + + args = dcargs.cli(Args) + print(args.a + args.b) + The broader goal is also a replacement for tools like :code:`hydra`, :code:`gin-config`, and :code:`ml_collections` that's: diff --git a/examples/01_functions.py b/examples/01_functions.py index ab31be917..c51af61f8 100644 --- a/examples/01_functions.py +++ b/examples/01_functions.py @@ -1,5 +1,3 @@ -# PYTHON_ARGCOMPLETE_OK - """In the simplest case, `dcargs.cli()` can be used to run a function with arguments populated from the CLI. diff --git a/examples/12_tuples.py b/examples/12_tuples.py index ed9005114..b7e988fa0 100644 --- a/examples/12_tuples.py +++ b/examples/12_tuples.py @@ -3,7 +3,7 @@ Usage: `python ./12_tuples.py --help` `python ./12_tuples.py --color 127 127 127` -`python ./12_tuples.py --two_colors:1.r 127 --two_colors:1.g 0 --two_colors:1.b 0` +`python ./12_tuples.py --two-colors:1.r 127 --two-colors:1.g 0 --two-colors:1.b 0` """ from typing import NamedTuple, Tuple diff --git a/examples/15_nesting_in_containers.py b/examples/15_nesting_in_containers.py index b1378270f..f5416d05a 100644 --- a/examples/15_nesting_in_containers.py +++ b/examples/15_nesting_in_containers.py @@ -37,7 +37,7 @@ class Args: color_tuple: Tuple[RGB, HSL] # Examples of nested structures in variable-length containers. These need a default - # provided for length inference -- we don't currently support specifying dynamic + # provided for length inference; we don't currently support specifying dynamic # container lengths directly from the commandline. color_tuple_alt: Tuple[Color, ...] = ( RGB(255, 0, 0), From a3b32af8fbd2125e5d3dae2c5acbae5733faeef5 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 12 Aug 2022 23:21:05 -0700 Subject: [PATCH 087/491] Add tests for nesting generics in containers --- dcargs/_fields.py | 3 - tests/test_nested_in_containers.py | 121 ++++++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 4 deletions(-) diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 07d87dbf2..c2575b9b2 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -402,9 +402,7 @@ def _field_list_from_sequence( for i, default_i in enumerate(default_instance): # type: ignore field_list.append( FieldDefinition( - # We'd use an index operator h name=str(i), - # This will currently break for generics...! typ=contained_type if contained_type not in MISSING_SINGLETONS else type(default_i), @@ -429,7 +427,6 @@ def _field_list_from_dict( field_list.append( FieldDefinition( name=str(k), - # TODO: this will fail for generic types. typ=type(v), default=v, helptext=None, diff --git a/tests/test_nested_in_containers.py b/tests/test_nested_in_containers.py index 93697114a..d69696bb9 100644 --- a/tests/test_nested_in_containers.py +++ b/tests/test_nested_in_containers.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, Generic, List, Tuple, TypeVar import pytest @@ -140,3 +140,122 @@ def main( assert dcargs.cli(main, args="--x.green.g 127".split(" "))["green"] == Color( 0, 127, 0 ) + + +def test_dict_nested(): + def main( + x: Dict[str, Tuple[Color, int]] = { + # For each color: RGB and xterm color code. + "red": (Color(255, 0, 0), 9), + "green": (Color(0, 255, 0), 10), + "blue": (Color(0, 0, 255), 12), + } + ) -> Any: + return x + + assert dcargs.cli(main, args=[])["green"] == (Color(0, 255, 0), 10) + assert dcargs.cli(main, args="--x.green:0.g 127 --x.green:1 2".split(" "))[ + "green" + ] == (Color(0, 127, 0), 2) + + +def test_generic_in_tuple(): + ScalarType = TypeVar("ScalarType", int, float) + + @dataclasses.dataclass + class GenericColor(Generic[ScalarType]): + r: ScalarType + g: ScalarType + b: ScalarType + + def main(x: Tuple[GenericColor[float], GenericColor[int]]) -> Any: + return x + + assert dcargs.cli( + main, + args=( + "--x:0.r 0.5 --x:0.g 0.2 --x:0.b 0.3 --x:1.r 25 --x:1.g 2 --x:1.b 3".split( + " " + ) + ), + ) == (GenericColor(0.5, 0.2, 0.3), GenericColor(25, 2, 3)) + + +def test_generic_in_tuple_with_default(): + ScalarType = TypeVar("ScalarType", int, float) + + @dataclasses.dataclass + class GenericColor(Generic[ScalarType]): + r: ScalarType + g: ScalarType + b: ScalarType + + def main( + x: Tuple[GenericColor[float], GenericColor[int]] = ( + GenericColor(0.5, 0.2, 0.3), + GenericColor[int](25, 2, 3), # The subscript should be optional. + ) + ) -> Any: + return x + + assert dcargs.cli( + main, + args=( + "--x:0.r 0.5 --x:0.g 0.2 --x:0.b 0.3 --x:1.r 25 --x:1.g 2 --x:1.b 3".split( + " " + ) + ), + ) == (GenericColor(0.5, 0.2, 0.3), GenericColor(25, 2, 3)) + + +def test_generic_in_variable_tuple_with_default(): + ScalarType = TypeVar("ScalarType", int, float) + + @dataclasses.dataclass + class GenericColor(Generic[ScalarType]): + r: ScalarType + g: ScalarType + b: ScalarType + + def main( + x: Tuple[GenericColor, ...] = ( + GenericColor[float](0.5, 0.2, 0.3), + GenericColor[int](25, 2, 3), + ) + ) -> Any: + return x + + assert dcargs.cli( + main, + args=( + "--x:0.r 0.5 --x:0.g 0.9 --x:0.b 0.3 --x:1.r 25 --x:1.g 2 --x:1.b 3".split( + " " + ) + ), + ) == (GenericColor(0.5, 0.9, 0.3), GenericColor(25, 2, 3)) + + +def test_generic_in_dict_with_default(): + ScalarType = TypeVar("ScalarType", int, float) + + @dataclasses.dataclass + class GenericColor(Generic[ScalarType]): + r: ScalarType + g: ScalarType + b: ScalarType + + def main( + x: Dict[str, GenericColor] = { + "float": GenericColor(0.5, 0.2, 0.3), + "int": GenericColor[int](25, 2, 3), + } + ) -> Any: + return x + + assert dcargs.cli(main, args="--x.float.g 0.1".split(" "),)[ + "float" + ] == GenericColor(0.5, 0.1, 0.3) + assert dcargs.cli( + main, + args="--x.int.g 0".split(" "), + ) == {"float": GenericColor(0.5, 0.2, 0.3), "int": GenericColor(25, 0, 3)} From 568e34ed5f51df8882a75fd255f6c687dc99e87b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 14 Aug 2022 03:20:38 -0700 Subject: [PATCH 088/491] Generalize type narrowing, tests --- dcargs/_calling.py | 1 + dcargs/_docstrings.py | 4 +-- dcargs/_fields.py | 17 ++---------- dcargs/_parsers.py | 8 ++++-- dcargs/_resolver.py | 20 +++++++++++++- tests/test_generics_and_serialization.py | 23 ++++++++++++++++ tests/test_helptext.py | 35 +++++++++++++++++++++++- 7 files changed, 86 insertions(+), 22 deletions(-) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 2581fb099..201d2725c 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -31,6 +31,7 @@ def call_from_args( Returns the output of `f` and a set of used arguments.""" f, type_from_typevar = _resolver.resolve_generic_types(f) + f = _resolver.narrow_type(f, default_instance) args: List[Any] = [] kwargs: Dict[str, Any] = {} diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 8dda6974b..969f3bf92 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -5,7 +5,7 @@ import inspect import io import tokenize -from typing import Callable, Dict, Hashable, List, Optional, Type +from typing import Callable, Dict, Generic, Hashable, List, Optional, Type import docstring_parser from typing_extensions import get_origin, is_typeddict @@ -117,7 +117,7 @@ def get_class_tokenization_with_field( for search_cls in classes_to_search: # Inherited generics seem challenging for now. # https://github.com/python/typing/issues/777 - assert get_origin(search_cls) is None + assert search_cls is Generic or get_origin(search_cls) is None try: tokenization = _ClassTokenization.make(search_cls) # type: ignore diff --git a/dcargs/_fields.py b/dcargs/_fields.py index c2575b9b2..ba1e0ba30 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -146,18 +146,7 @@ def field_list_from_callable( # Unwrap generics. f, type_from_typevar = _resolver.resolve_generic_types(f) - - # Type narrowing: if we annotate as Animal but specify a default instance of Cat, we - # should parse as Cat. - # - # TODO: this will not currently handle generics correctly. We should write tests for - # this. - try: - potential_subclass = type(default_instance) - if issubclass(potential_subclass, cast(Type, f)): - f = potential_subclass # type: ignore - except TypeError: - pass + f = _resolver.narrow_type(f, default_instance) # If `f` is a type: # 1. Set cls to the type. @@ -403,9 +392,7 @@ def _field_list_from_sequence( field_list.append( FieldDefinition( name=str(i), - typ=contained_type - if contained_type not in MISSING_SINGLETONS - else type(default_i), + typ=contained_type, default=default_i, helptext="", positional=False, diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 195ab2cfd..2704aad2c 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -48,6 +48,7 @@ def from_callable( # Resolve generic types. f, type_from_typevar = _resolver.resolve_generic_types(f) + f = _resolver.narrow_type(f, default_instance) if parent_type_from_typevar is not None: for typevar, typ in type_from_typevar.items(): if typ in parent_type_from_typevar: @@ -120,6 +121,9 @@ def from_callable( # (2) Handle nested callables. if _fields.is_nested_type(field.typ, field.default): + field = dataclasses.replace( + field, typ=_resolver.narrow_type(field.typ, field.default) + ) nested_parser = ParserSpecification.from_callable( field.typ, description=None, @@ -141,9 +145,7 @@ def from_callable( else: helptext_from_nested_class_field_name[ field.name - ] = _docstrings.get_callable_description( - _resolver.resolve_generic_types(field.typ)[0] - ) + ] = _docstrings.get_callable_description(field.typ) continue # (3) Handle primitive types. These produce a single argument! diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 0e4719e0d..56b0a121a 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -2,7 +2,7 @@ import copy import dataclasses -from typing import Callable, Dict, List, Tuple, Type, TypeVar, Union +from typing import Any, Callable, Dict, List, Tuple, Type, TypeVar, Union, cast from typing_extensions import get_args, get_origin, get_type_hints @@ -77,3 +77,21 @@ def type_from_typevar_constraints(typ: Union[Type, TypeVar]) -> Union[Type, Type # Try to infer type from TypeVar constraints. return Union.__getitem__(typ.__constraints__) # type: ignore return typ + + +# Be a little bit permissive with types here, since we often blur the lines between +# Callable[..., T] and Type[T]... this could be cleaned up! +TypeT = TypeVar("TypeT", bound=Callable) + + +def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: + """Type narrowing: if we annotate as Animal but specify a default instance of Cat, we + should parse as Cat.""" + try: + potential_subclass = type(default_instance) + superclass = typ + if issubclass(potential_subclass, superclass): # type: ignore + return cast(TypeT, potential_subclass) + except TypeError: + pass + return typ diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 547e2b634..5861250dd 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -347,3 +347,26 @@ class TupleGenericVariableMissing(Generic[ScalarType]): ).xyz[0] is dcargs.MISSING ) + + +def test_generic_inherited_type_narrowing(): + T = TypeVar("T") + + @dataclasses.dataclass + class ActualParentClass(Generic[T]): + x: T # Documentation 1 + + # Documentation 2 + y: T + + z: T = 3 # type: ignore + """Documentation 3""" + + @dataclasses.dataclass + class ChildClass(ActualParentClass[int]): + a: int = 7 + + def main(x: ActualParentClass[int] = ChildClass(5, 5)) -> ActualParentClass: + return x + + assert dcargs.cli(main, args="--x.x 3".split(" ")) == ChildClass(3, 5, 3) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 77007bb00..925375acf 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -4,7 +4,7 @@ import io import pathlib from collections.abc import Callable -from typing import Dict, Generic, List, Optional, Tuple, TypeVar, Union, cast +from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar, Union, cast import pytest from typing_extensions import Literal @@ -82,6 +82,39 @@ class ChildClass(UnrelatedParentClass, ActualParentClass): assert "Documentation 2" in helptext +def test_helptext_inherited_default_override(): + @dataclasses.dataclass + class ParentClass: + """This docstring should __not__ be printed as a description.""" + + x: int # Documentation 1 + + # Documentation 2 + y: int + + # fmt: off + + z: int = 3 + def some_method(self) -> None: # noqa + """Coverage stress test.""" + # fmt: on + + @dataclasses.dataclass + class ChildClass(ParentClass): + """This docstring should be printed as a description.""" + + pass + + def main(x: ParentClass = ChildClass(x=5, y=5)) -> Any: + return x + + helptext = _get_helptext(main) + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + assert "__not__" not in helptext + assert "should be printed" in helptext + + def test_helptext_nested(): """For nested classes, we should pull helptext from the outermost docstring if possible. The class docstring can be used as a fallback.""" From 44ed9a121e9951283cc1128a062e24152397d4ab Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 14 Aug 2022 16:45:31 -0700 Subject: [PATCH 089/491] Improve behavior for optional subparsers --- dcargs/_calling.py | 30 +++--- dcargs/_cli.py | 5 +- dcargs/_parsers.py | 148 +++++++++++++++++------------ examples/09_subparsers.py | 2 +- examples/10_multiple_subparsers.py | 4 +- tests/test_nested.py | 8 +- 6 files changed, 111 insertions(+), 86 deletions(-) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 201d2725c..76a92e7a6 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -119,6 +119,8 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: assert len(parser_definition.subparsers_from_name) > 0 assert field.name in parser_definition.subparsers_from_name + subparser_def = parser_definition.subparsers_from_name[field.name] + subparser_dest = _strings.SUBPARSER_DEST_FMT.format( name=prefixed_field_name ) @@ -126,25 +128,23 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: if subparser_dest in value_from_prefixed_field_name: subparser_name = get_value_from_arg(subparser_dest) else: - default_instance = parser_definition.subparsers_from_name[ - field.name - ].default_instance - assert default_instance is not None + default_instance = subparser_def.default_instance + # assert default_instance is not None subparser_name = None + if subparser_name is None: - # No subparser selected -- this should only happen when we do either - # Optional[Union[A, B, ...]] or Union[A, B, None], or have a + # No subparser selected -- this should only happen when we have a # default/default_factory set. assert ( type(None) in get_args(field_type) - or parser_definition.subparsers_from_name[ - field.name - ].default_instance - is not None + or subparser_def.default_instance is not None ) - value = parser_definition.subparsers_from_name[ - field.name - ].default_instance + value = subparser_def.default_instance + elif subparser_def.can_be_none and subparser_name == "None": + # do either + # Optional[Union[A, B, ...]] or Union[A, B, None], or have a + # default/default_factory set. + value = None else: options = map( lambda x: x if x not in type_from_typevar else type_from_typevar[x], @@ -158,9 +158,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: assert chosen_f is not None value, consumed_keywords_child = call_from_args( chosen_f, - parser_definition.subparsers_from_name[field.name].parser_from_name[ - subparser_name - ], + subparser_def.parser_from_name[subparser_name], field.default if type(field.default) is chosen_f else None, value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, diff --git a/dcargs/_cli.py b/dcargs/_cli.py index c18ec170c..808d9b2ab 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -193,7 +193,10 @@ def _cli_impl( print(e.args[0]) raise SystemExit() - assert len(value_from_prefixed_field_name.keys() - consumed_keywords) == 0 + assert len(value_from_prefixed_field_name.keys() - consumed_keywords) == 0, ( + f"Parsed {value_from_prefixed_field_name.keys()}, but only consumed" + f" {consumed_keywords}" + ) if _return_stage == "f_out": return out diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 2704aad2c..00fdbb4ed 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -216,57 +216,9 @@ def format_group_name(nested_field_name: str) -> str: if len(self.subparsers_from_name) > 0: prev_subparser_tree_nodes = [parser] # Root node. for subparsers in self.subparsers_from_name.values(): - title = "subcommands" - metavar = "{" + ",".join(subparsers.parser_from_name.keys()) + "}" - if not subparsers.required: - title = "optional " + title - metavar = f"[{metavar}]" - - subparser_tree_nodes = [] - for p in prev_subparser_tree_nodes: - # Add subparsers to every node in previous level of the tree. - argparse_subparsers = p.add_subparsers( - dest=_strings.make_field_name( - [ - self.prefix, - _strings.SUBPARSER_DEST_FMT.format( - name=subparsers.name - ), - ] - ), - description=subparsers.description, - required=subparsers.required, - title=termcolor.colored(title, attrs=["bold"]), - metavar=metavar, - ) - for name, subparser_def in subparsers.parser_from_name.items(): - subparser = argparse_subparsers.add_parser( - name, - formatter_class=_argparse_formatter.ArgparseHelpFormatter, - ) - subparser_def.apply(subparser) - - def _get_leaf_subparsers( - node: argparse.ArgumentParser, - ) -> List[argparse.ArgumentParser]: - if node._subparsers is None: - return [node] - else: - # Magic! - return list( - itertools.chain( - *map( - _get_leaf_subparsers, - node._subparsers._actions[ - -1 - ]._name_parser_map.values(), # type: ignore - ) - ) - ) - - subparser_tree_nodes.extend(_get_leaf_subparsers(subparser)) - - prev_subparser_tree_nodes = subparser_tree_nodes + prev_subparser_tree_nodes = subparsers.apply( + self, prev_subparser_tree_nodes + ) @dataclasses.dataclass(frozen=True) @@ -303,6 +255,7 @@ def from_field( ): return None + # Add subparser for each option. parser_from_name: Dict[str, ParserSpecification] = {} for option in options_no_none: subparser_name = _strings.subparser_name_from_type(option) @@ -311,19 +264,18 @@ def from_field( description=None, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, + # Only pass default in if it corresponds to this open. + # TODO: this will break for generics! default_instance=field.default - if isinstance(field.default, _resolver.unwrap_origin(option)) - else None, + if isinstance(field.default, _resolver.unwrap_origin(option)) # type: ignore + else _fields.MISSING_NONPROP, prefix=prefix, avoid_subparsers=avoid_subparsers, ) # Optional if: type hint is Optional[], or a default instance is provided. required = True - can_be_none = options != options_no_none # Optional[] type. - if can_be_none: - required = False - elif field.default not in _fields.MISSING_SINGLETONS: + if field.default not in _fields.MISSING_SINGLETONS: required = False # If there are any required arguments in the default subparser, we should mark @@ -350,7 +302,9 @@ def from_field( if field.helptext is not None: description_parts.append(field.helptext) if not required and field.default not in _fields.MISSING_SINGLETONS: - default = _strings.subparser_name_from_type(type(field.default)) + default = field.default + if default is not None: + default = _strings.subparser_name_from_type(type(default)) description_parts.append(f" (default: {default})") description = ( # We use `None` instead of an empty string to prevent a line break from @@ -368,8 +322,78 @@ def from_field( description=description, parser_from_name=parser_from_name, required=required, - default_instance=field.default - if field.default not in _fields.MISSING_SINGLETONS - else None, - can_be_none=can_be_none, + default_instance=field.default, + # if field.default not in _fields.MISSING_SINGLETONS + # else None, + can_be_none=options != options_no_none, + ) + + def apply( + self, + parent_parser: ParserSpecification, + prev_subparser_tree_nodes: List[argparse.ArgumentParser], + ) -> List[argparse.ArgumentParser]: + title = "subcommands" + metavar = ( + "{" + + ",".join( + (("None",) if self.can_be_none else ()) + + tuple(self.parser_from_name.keys()) + ) + + "}" ) + if not self.required: + title = "optional " + title + metavar = f"[{metavar}]" + + subparser_tree_nodes: List[argparse.ArgumentParser] = [] + for p in prev_subparser_tree_nodes: + # Add subparsers to every node in previous level of the tree. + argparse_subparsers = p.add_subparsers( + dest=_strings.make_field_name( + [ + parent_parser.prefix, + _strings.SUBPARSER_DEST_FMT.format(name=self.name), + ] + ), + description=self.description, + required=self.required, + title=termcolor.colored(title, attrs=["bold"]), + metavar=metavar, + ) + + if self.can_be_none: + subparser = argparse_subparsers.add_parser( + name="None", + formatter_class=_argparse_formatter.ArgparseHelpFormatter, + ) + subparser_tree_nodes.append(subparser) + + for name, subparser_def in self.parser_from_name.items(): + subparser = argparse_subparsers.add_parser( + name, + formatter_class=_argparse_formatter.ArgparseHelpFormatter, + ) + subparser_def.apply(subparser) + + def _get_leaf_subparsers( + node: argparse.ArgumentParser, + ) -> List[argparse.ArgumentParser]: + if node._subparsers is None: + return [node] + else: + # Magic! + return list( + itertools.chain( + *map( + _get_leaf_subparsers, + node._subparsers._actions[ + -1 + ]._name_parser_map.values(), # type: ignore + ) + ) + ) + + subparser_tree_nodes.extend(_get_leaf_subparsers(subparser)) + + return subparser_tree_nodes diff --git a/examples/09_subparsers.py b/examples/09_subparsers.py index 34808114c..f9c4c93e2 100644 --- a/examples/09_subparsers.py +++ b/examples/09_subparsers.py @@ -31,7 +31,7 @@ class Commit: all: bool = False -def main(cmd: Union[Checkout, Commit]) -> None: +def main(cmd: Union[Checkout, Commit] = Checkout(dcargs.MISSING)) -> None: print(cmd) diff --git a/examples/10_multiple_subparsers.py b/examples/10_multiple_subparsers.py index e1ed25244..cadaea696 100644 --- a/examples/10_multiple_subparsers.py +++ b/examples/10_multiple_subparsers.py @@ -46,8 +46,8 @@ class SgdOptimizer: def train( - dataset: Union[MnistDataset, ImageNetDataset] = MnistDataset(), - optimizer: Union[AdamOptimizer, SgdOptimizer] = AdamOptimizer(), + dataset: Union[MnistDataset, ImageNetDataset, None], # = MnistDataset(), + optimizer: Union[AdamOptimizer, SgdOptimizer, None], # = AdamOptimizer(), ) -> None: """Example training script. diff --git a/tests/test_nested.py b/tests/test_nested.py index d588b8fcc..513c5df76 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -113,7 +113,7 @@ class OptionalNestedChild: @dataclasses.dataclass class OptionalNested: x: int - b: Optional[OptionalNestedChild] + b: Optional[OptionalNestedChild] = None assert dcargs.cli(OptionalNested, args=["--x", "1"]) == OptionalNested(x=1, b=None) with pytest.raises(SystemExit): @@ -343,9 +343,9 @@ class OptionalSubparser: assert dcargs.cli( OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--bc.z", "3"] ) == OptionalSubparser(x=1, bc=OptionalSMTPServer(z=3)) - assert dcargs.cli(OptionalSubparser, args=["--x", "1"]) == OptionalSubparser( - x=1, bc=None - ) + assert dcargs.cli( + OptionalSubparser, args=["--x", "1", "None"] + ) == OptionalSubparser(x=1, bc=None) with pytest.raises(SystemExit): # Wrong field. From c63591ef4bd5ced39b3c3df24a9b3335fbceb86e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 14 Aug 2022 17:23:27 -0700 Subject: [PATCH 090/491] More informative names for subcommands, use . for indices --- dcargs/_calling.py | 11 ++- dcargs/_parsers.py | 18 ++-- dcargs/_strings.py | 14 ++-- examples/09_subparsers.py | 10 +-- examples/10_multiple_subparsers.py | 16 ++-- examples/12_tuples.py | 2 +- tests/test_forward_ref.py | 24 ++++-- tests/test_generics_and_serialization.py | 10 ++- tests/test_helptext.py | 4 +- tests/test_nested.py | 100 ++++++++++++----------- tests/test_nested_in_containers.py | 26 +++--- 11 files changed, 134 insertions(+), 101 deletions(-) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 76a92e7a6..51c502891 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -140,7 +140,11 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: or subparser_def.default_instance is not None ) value = subparser_def.default_instance - elif subparser_def.can_be_none and subparser_name == "None": + elif ( + subparser_def.can_be_none + and subparser_name + == _strings.subparser_name_from_type(prefixed_field_name, None) + ): # do either # Optional[Union[A, B, ...]] or Union[A, B, None], or have a # default/default_factory set. @@ -152,7 +156,10 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: ) chosen_f = None for option in options: - if _strings.subparser_name_from_type(option) == subparser_name: + if ( + _strings.subparser_name_from_type(prefixed_field_name, option) + == subparser_name + ): chosen_f = option break assert chosen_f is not None diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 00fdbb4ed..f2cb9eefe 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -228,6 +228,7 @@ class SubparsersSpecification: name: str description: Optional[str] parser_from_name: Dict[str, ParserSpecification] + prefix: str required: bool default_instance: Any can_be_none: bool # If underlying type is Optional[Something]. @@ -258,13 +259,13 @@ def from_field( # Add subparser for each option. parser_from_name: Dict[str, ParserSpecification] = {} for option in options_no_none: - subparser_name = _strings.subparser_name_from_type(option) + subparser_name = _strings.subparser_name_from_type(prefix, option) parser_from_name[subparser_name] = ParserSpecification.from_callable( option, description=None, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, - # Only pass default in if it corresponds to this open. + # Only pass default in if it corresponds to this option. # TODO: this will break for generics! default_instance=field.default if isinstance(field.default, _resolver.unwrap_origin(option)) # type: ignore @@ -285,7 +286,7 @@ def from_field( and field.default not in _fields.MISSING_SINGLETONS ): default_parser = parser_from_name[ - _strings.subparser_name_from_type(type(field.default)) + _strings.subparser_name_from_type(prefix, type(field.default)) ] if any(map(lambda arg: arg.lowered.required, default_parser.args)): required = True @@ -304,7 +305,7 @@ def from_field( if not required and field.default not in _fields.MISSING_SINGLETONS: default = field.default if default is not None: - default = _strings.subparser_name_from_type(type(default)) + default = _strings.subparser_name_from_type(prefix, type(default)) description_parts.append(f" (default: {default})") description = ( # We use `None` instead of an empty string to prevent a line break from @@ -321,6 +322,7 @@ def from_field( # the user to include it in the docstring. description=description, parser_from_name=parser_from_name, + prefix=prefix, required=required, default_instance=field.default, # if field.default not in _fields.MISSING_SINGLETONS @@ -337,7 +339,11 @@ def apply( metavar = ( "{" + ",".join( - (("None",) if self.can_be_none else ()) + ( + (_strings.subparser_name_from_type(self.prefix, None),) + if self.can_be_none + else () + ) + tuple(self.parser_from_name.keys()) ) + "}" @@ -364,7 +370,7 @@ def apply( if self.can_be_none: subparser = argparse_subparsers.add_parser( - name="None", + name=_strings.subparser_name_from_type(self.prefix, None), formatter_class=_argparse_formatter.ArgparseHelpFormatter, ) subparser_tree_nodes.append(subparser) diff --git a/dcargs/_strings.py b/dcargs/_strings.py index e639e09af..846ef8071 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -3,7 +3,7 @@ import functools import re import textwrap -from typing import List, Sequence, Type +from typing import List, Sequence, Type, Union from . import _resolver @@ -17,9 +17,7 @@ def make_field_name(parts: Sequence[str]) -> str: out: List[str] = [] for i, p in enumerate([p for p in parts if len(p) > 0]): if i > 0: - # Delimeter between parts. We use a colon before integers, which are - # typically indices. - out.append(":" if p[0].isdigit() else ".") + out.append(".") out.append(p) @@ -46,7 +44,7 @@ def hyphen_separated_from_camel_case(name: str) -> str: return _camel_separator_pattern().sub(r"-\1", name).lower() -def subparser_name_from_type(cls: Type) -> str: +def _subparser_name_from_type(cls: Type) -> str: cls, type_from_typevar = _resolver.resolve_generic_types(cls) if len(type_from_typevar) == 0: assert hasattr(cls, "__name__") @@ -54,12 +52,16 @@ def subparser_name_from_type(cls: Type) -> str: return "-".join( map( - subparser_name_from_type, + _subparser_name_from_type, [cls] + list(type_from_typevar.values()), ) ) +def subparser_name_from_type(prefix: str, cls: Union[Type, None]) -> str: + return f"{prefix}:{_subparser_name_from_type(cls) if cls is not None else 'None'}" + + @functools.lru_cache(maxsize=None) def _get_ansi_pattern() -> re.Pattern: # https://stackoverflow.com/a/14693789 diff --git a/examples/09_subparsers.py b/examples/09_subparsers.py index f9c4c93e2..9970b0ced 100644 --- a/examples/09_subparsers.py +++ b/examples/09_subparsers.py @@ -2,10 +2,10 @@ Usage: `python ./09_subparsers.py --help` -`python ./09_subparsers.py commit --help` -`python ./09_subparsers.py commit --cmd.message hello --cmd.all` -`python ./09_subparsers.py checkout --help` -`python ./09_subparsers.py checkout --cmd.branch main` +`python ./09_subparsers.py cmd:commit --help` +`python ./09_subparsers.py cmd:commit --cmd.message hello --cmd.all` +`python ./09_subparsers.py cmd:checkout --help` +`python ./09_subparsers.py cmd:checkout --cmd.branch main` """ from __future__ import annotations @@ -31,7 +31,7 @@ class Commit: all: bool = False -def main(cmd: Union[Checkout, Commit] = Checkout(dcargs.MISSING)) -> None: +def main(cmd: Union[Checkout, Commit, None]) -> None: print(cmd) diff --git a/examples/10_multiple_subparsers.py b/examples/10_multiple_subparsers.py index cadaea696..fd474660a 100644 --- a/examples/10_multiple_subparsers.py +++ b/examples/10_multiple_subparsers.py @@ -3,8 +3,8 @@ Usage: `python ./10_multiple_subparsers.py` `python ./10_multiple_subparsers.py --help` -`python ./10_multiple_subparsers.py mnist-dataset --help` -`python ./10_multiple_subparsers.py mnist-dataset adam-optimizer --optimizer.learning-rate 3e-4` +`python ./10_multiple_subparsers.py dataset:mnist --help` +`python ./10_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4` """ from __future__ import annotations @@ -17,13 +17,13 @@ @dataclasses.dataclass -class MnistDataset: +class Mnist: binary: bool = False """Set to load binary version of MNIST dataset.""" @dataclasses.dataclass -class ImageNetDataset: +class ImageNet: subset: Literal[50, 100, 1000] """Choose between ImageNet-50, ImageNet-100, ImageNet-1000, etc.""" @@ -32,13 +32,13 @@ class ImageNetDataset: @dataclasses.dataclass -class AdamOptimizer: +class Adam: learning_rate: float = 1e-3 betas: Tuple[float, float] = (0.9, 0.999) @dataclasses.dataclass -class SgdOptimizer: +class Sgd: learning_rate: float = 3e-4 @@ -46,8 +46,8 @@ class SgdOptimizer: def train( - dataset: Union[MnistDataset, ImageNetDataset, None], # = MnistDataset(), - optimizer: Union[AdamOptimizer, SgdOptimizer, None], # = AdamOptimizer(), + dataset: Union[Mnist, ImageNet] = Mnist(), + optimizer: Union[Adam, Sgd] = Adam(), ) -> None: """Example training script. diff --git a/examples/12_tuples.py b/examples/12_tuples.py index b7e988fa0..f85f4f884 100644 --- a/examples/12_tuples.py +++ b/examples/12_tuples.py @@ -3,7 +3,7 @@ Usage: `python ./12_tuples.py --help` `python ./12_tuples.py --color 127 127 127` -`python ./12_tuples.py --two-colors:1.r 127 --two-colors:1.g 0 --two-colors:1.b 0` +`python ./12_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0` """ from typing import NamedTuple, Tuple diff --git a/tests/test_forward_ref.py b/tests/test_forward_ref.py index fb96e0b08..3bf4230f9 100644 --- a/tests/test_forward_ref.py +++ b/tests/test_forward_ref.py @@ -29,20 +29,28 @@ class C: def test_forward_ref_1(): - assert dcargs.cli(A1, args=["--x", "1", "b", "--bc.y", "3"]) == A1(x=1, bc=B(y=3)) - assert dcargs.cli(A1, args=["--x", "1", "c", "--bc.z", "3"]) == A1(x=1, bc=C(z=3)) + assert dcargs.cli(A1, args=["--x", "1", "bc:b", "--bc.y", "3"]) == A1( + x=1, bc=B(y=3) + ) + assert dcargs.cli(A1, args=["--x", "1", "bc:c", "--bc.z", "3"]) == A1( + x=1, bc=C(z=3) + ) with pytest.raises(SystemExit): - dcargs.cli(A1, args=["--x", "1", "b", "--bc.z", "3"]) + dcargs.cli(A1, args=["--x", "1", "bc:b", "--bc.z", "3"]) with pytest.raises(SystemExit): - dcargs.cli(A1, args=["--x", "1", "c", "--bc.y", "3"]) + dcargs.cli(A1, args=["--x", "1", "bc:c", "--bc.y", "3"]) def test_forward_ref_2(): - assert dcargs.cli(A2, args=["--x", "1", "b", "--bc.y", "3"]) == A2(x=1, bc=B(y=3)) - assert dcargs.cli(A2, args=["--x", "1", "c", "--bc.z", "3"]) == A2(x=1, bc=C(z=3)) + assert dcargs.cli(A2, args=["--x", "1", "bc:b", "--bc.y", "3"]) == A2( + x=1, bc=B(y=3) + ) + assert dcargs.cli(A2, args=["--x", "1", "bc:c", "--bc.z", "3"]) == A2( + x=1, bc=C(z=3) + ) with pytest.raises(SystemExit): - dcargs.cli(A2, args=["--x", "1", "b", "--bc.z", "3"]) + dcargs.cli(A2, args=["--x", "1", "bc:b", "--bc.z", "3"]) with pytest.raises(SystemExit): - dcargs.cli(A2, args=["--x", "1", "c", "--bc.y", "3"]) + dcargs.cli(A2, args=["--x", "1", "bc:c", "--bc.y", "3"]) diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 5861250dd..fa763fe43 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -258,13 +258,15 @@ class Subparser(Generic[T1, T2]): command: Union[T1, T2] parsed_instance = dcargs.cli( - Subparser[CommandOne, CommandTwo], args="command-one --command.a 5".split(" ") + Subparser[CommandOne, CommandTwo], + args="command:command-one --command.a 5".split(" "), ) assert parsed_instance == Subparser(CommandOne(5)) _check_serialization_identity(Subparser[CommandOne, CommandTwo], parsed_instance) parsed_instance = dcargs.cli( - Subparser[CommandOne, CommandTwo], args="command-two --command.b 7".split(" ") + Subparser[CommandOne, CommandTwo], + args="command:command-two --command.b 7".split(" "), ) assert parsed_instance == Subparser(CommandTwo(7)) _check_serialization_identity(Subparser[CommandOne, CommandTwo], parsed_instance) @@ -286,7 +288,7 @@ class Subparser(Generic[T1, T2]): parsed_instance = dcargs.cli( Subparser[Command[int], Command[float]], - args="command-int --command.a 5 3".split(" "), + args="command:command-int --command.a 5 3".split(" "), ) assert parsed_instance == Subparser(Command([5, 3])) and isinstance( parsed_instance.command.a[0], int @@ -297,7 +299,7 @@ class Subparser(Generic[T1, T2]): parsed_instance = dcargs.cli( Subparser[Command[int], Command[float]], - args="command-float --command.a 7 2".split(" "), + args="command:command-float --command.a 7 2".split(" "), ) assert parsed_instance == Subparser(Command([7.0, 2.0])) and isinstance( parsed_instance.command.a[0], float diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 925375acf..a9f9b235b 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -383,13 +383,13 @@ class MultipleSubparsers: assert "Field c description." not in helptext helptext = _get_helptext( - MultipleSubparsers, args=["subcommand1", "subcommand1", "--help"] + MultipleSubparsers, args=["a:subcommand1", "b:subcommand1", "--help"] ) assert "Field a description." not in helptext assert "Field b description." not in helptext assert "Field c description." in helptext - assert "(default: subcommand3)" in helptext + assert "(default: c:subcommand3)" in helptext def test_optional_helptext(): diff --git a/tests/test_nested.py b/tests/test_nested.py index 513c5df76..02278dc6c 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -118,16 +118,16 @@ class OptionalNested: assert dcargs.cli(OptionalNested, args=["--x", "1"]) == OptionalNested(x=1, b=None) with pytest.raises(SystemExit): dcargs.cli( - OptionalNested, args=["--x", "1", "optional-nested-child", "--b.y", "3"] + OptionalNested, args=["--x", "1", "b:optional-nested-child", "--b.y", "3"] ) with pytest.raises(SystemExit): dcargs.cli( - OptionalNested, args=["--x", "1", "optional-nested-child", "--b.z", "3"] + OptionalNested, args=["--x", "1", "b:optional-nested-child", "--b.z", "3"] ) assert dcargs.cli( OptionalNested, - args=["--x", "1", "optional-nested-child", "--b.y", "2", "--b.z", "3"], + args=["--x", "1", "b:optional-nested-child", "--b.y", "2", "--b.z", "3"], ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) @@ -146,10 +146,10 @@ class Subparser: bc: Union[HTTPServer, SMTPServer] assert dcargs.cli( - Subparser, args=["--x", "1", "http-server", "--bc.y", "3"] + Subparser, args=["--x", "1", "bc:http-server", "--bc.y", "3"] ) == Subparser(x=1, bc=HTTPServer(y=3)) assert dcargs.cli( - Subparser, args=["--x", "1", "smtp-server", "--bc.z", "3"] + Subparser, args=["--x", "1", "bc:smtp-server", "--bc.z", "3"] ) == Subparser(x=1, bc=SMTPServer(z=3)) with pytest.raises(SystemExit): @@ -157,10 +157,10 @@ class Subparser: dcargs.cli(Subparser, args=["--x", "1"]) with pytest.raises(SystemExit): # Wrong field. - dcargs.cli(Subparser, args=["--x", "1", "http-server", "--bc.z", "3"]) + dcargs.cli(Subparser, args=["--x", "1", "bc:http-server", "--bc.z", "3"]) with pytest.raises(SystemExit): # Wrong field. - dcargs.cli(Subparser, args=["--x", "1", "smtp-server", "--bc.y", "3"]) + dcargs.cli(Subparser, args=["--x", "1", "bc:smtp-server", "--bc.y", "3"]) def test_subparser_with_default(): @@ -181,17 +181,17 @@ class DefaultSubparser: assert ( dcargs.cli( - DefaultSubparser, args=["--x", "1", "default-http-server", "--bc.y", "5"] + DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "5"] ) == dcargs.cli(DefaultSubparser, args=["--x", "1"]) == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=5)) ) assert dcargs.cli( - DefaultSubparser, args=["--x", "1", "default-smtp-server", "--bc.z", "3"] + DefaultSubparser, args=["--x", "1", "bc:default-smtp-server", "--bc.z", "3"] ) == DefaultSubparser(x=1, bc=DefaultSMTPServer(z=3)) assert ( dcargs.cli( - DefaultSubparser, args=["--x", "1", "default-http-server", "--bc.y", "8"] + DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "8"] ) == dcargs.cli( DefaultSubparser, @@ -224,7 +224,7 @@ class DefaultInstanceSubparser: assert ( dcargs.cli( DefaultInstanceSubparser, - args=["--x", "1", "default-instance-http-server", "--bc.y", "5"], + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], ) == dcargs.cli( DefaultInstanceSubparser, @@ -235,7 +235,7 @@ class DefaultInstanceSubparser: ) == dcargs.cli( DefaultInstanceSubparser, - args=["default-instance-http-server"], + args=["bc:default-instance-http-server"], default_instance=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=5) ), @@ -244,7 +244,7 @@ class DefaultInstanceSubparser: ) assert dcargs.cli( DefaultInstanceSubparser, - args=["default-instance-smtp-server", "--bc.z", "3"], + args=["bc:default-instance-smtp-server", "--bc.z", "3"], default_instance=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=5) ), @@ -252,7 +252,7 @@ class DefaultInstanceSubparser: assert ( dcargs.cli( DefaultInstanceSubparser, - args=["--x", "1", "default-instance-http-server", "--bc.y", "8"], + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], ) == dcargs.cli( DefaultInstanceSubparser, @@ -287,7 +287,7 @@ class DefaultInstanceSubparser: assert ( dcargs.cli( DefaultInstanceSubparser, - args=["--x", "1", "default-instance-http-server", "--bc.y", "5"], + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], ) == dcargs.cli( DefaultInstanceSubparser, @@ -301,7 +301,7 @@ class DefaultInstanceSubparser: ) assert dcargs.cli( DefaultInstanceSubparser, - args=["default-instance-smtp-server", "--bc.z", "3"], + args=["bc:default-instance-smtp-server", "--bc.z", "3"], default_instance=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=5) ), @@ -309,7 +309,7 @@ class DefaultInstanceSubparser: assert ( dcargs.cli( DefaultInstanceSubparser, - args=["--x", "1", "default-instance-http-server", "--bc.y", "8"], + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], ) == dcargs.cli( DefaultInstanceSubparser, @@ -338,24 +338,26 @@ class OptionalSubparser: bc: Optional[Union[OptionalHTTPServer, OptionalSMTPServer]] assert dcargs.cli( - OptionalSubparser, args=["--x", "1", "optional-http-server", "--bc.y", "3"] + OptionalSubparser, args=["--x", "1", "bc:optional-http-server", "--bc.y", "3"] ) == OptionalSubparser(x=1, bc=OptionalHTTPServer(y=3)) assert dcargs.cli( - OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--bc.z", "3"] + OptionalSubparser, args=["--x", "1", "bc:optional-smtp-server", "--bc.z", "3"] ) == OptionalSubparser(x=1, bc=OptionalSMTPServer(z=3)) assert dcargs.cli( - OptionalSubparser, args=["--x", "1", "None"] + OptionalSubparser, args=["--x", "1", "bc:None"] ) == OptionalSubparser(x=1, bc=None) with pytest.raises(SystemExit): # Wrong field. dcargs.cli( - OptionalSubparser, args=["--x", "1", "optional-http-server", "--bc.z", "3"] + OptionalSubparser, + args=["--x", "1", "bc:optional-http-server", "--bc.z", "3"], ) with pytest.raises(SystemExit): # Wrong field. dcargs.cli( - OptionalSubparser, args=["--x", "1", "optional-smtp-server", "--bc.y", "3"] + OptionalSubparser, + args=["--x", "1", "bc:optional-smtp-server", "--bc.y", "3"], ) @@ -410,22 +412,28 @@ class MultipleSubparsers: dcargs.cli(MultipleSubparsers, args=[]) assert dcargs.cli( - MultipleSubparsers, args="subcommand1 subcommand2 subcommand3".split(" ") + MultipleSubparsers, args="a:subcommand1 b:subcommand2 c:subcommand3".split(" ") ) == MultipleSubparsers(Subcommand1(), Subcommand2(), Subcommand3()) assert dcargs.cli( MultipleSubparsers, - args="subcommand1 --a.x 5 subcommand2 --b.y 7 subcommand3 --c.z 3".split(" "), + args="a:subcommand1 --a.x 5 b:subcommand2 --b.y 7 c:subcommand3 --c.z 3".split( + " " + ), ) == MultipleSubparsers(Subcommand1(x=5), Subcommand2(y=7), Subcommand3(z=3)) assert dcargs.cli( MultipleSubparsers, - args="subcommand2 --a.y 5 subcommand1 --b.x 7 subcommand3 --c.z 3".split(" "), + args="a:subcommand2 --a.y 5 b:subcommand1 --b.x 7 c:subcommand3 --c.z 3".split( + " " + ), ) == MultipleSubparsers(Subcommand2(y=5), Subcommand1(x=7), Subcommand3(z=3)) assert dcargs.cli( MultipleSubparsers, - args="subcommand3 --a.z 5 subcommand1 --b.x 7 subcommand3 --c.z 3".split(" "), + args="a:subcommand3 --a.z 5 b:subcommand1 --b.x 7 c:subcommand3 --c.z 3".split( + " " + ), ) == MultipleSubparsers(Subcommand3(z=5), Subcommand1(x=7), Subcommand3(z=3)) @@ -456,12 +464,12 @@ class MultipleSubparsers: assert dcargs.cli( MultipleSubparsers, - args=["subcommand1", "--a.x", "5"], + args=["a:subcommand1", "--a.x", "5"], ) == MultipleSubparsers(Subcommand1(x=5), Subcommand2(y=7), Subcommand3(z=3)) assert dcargs.cli( MultipleSubparsers, - args="subcommand1 --a.x 3".split(" "), + args="a:subcommand1 --a.x 3".split(" "), ) == MultipleSubparsers(Subcommand1(x=3), Subcommand2(y=7), Subcommand3(z=3)) with pytest.raises(SystemExit): @@ -478,7 +486,7 @@ class MultipleSubparsers: dcargs.cli( MultipleSubparsers, args=[ - "subcommand1", + "a:subcommand1", ], default_instance=MultipleSubparsers( Subcommand1(), @@ -489,7 +497,7 @@ class MultipleSubparsers: with pytest.raises(SystemExit): dcargs.cli( MultipleSubparsers, - args=["subcommand1", "subcommand2"], + args=["a:subcommand1", "b:subcommand2"], default_instance=MultipleSubparsers( Subcommand1(), Subcommand2(), @@ -499,7 +507,7 @@ class MultipleSubparsers: with pytest.raises(SystemExit): dcargs.cli( MultipleSubparsers, - args=["subcommand1", "subcommand2", "subcommand3"], + args=["a:subcommand1", "b:subcommand2", "c:subcommand3"], default_instance=MultipleSubparsers( Subcommand1(), Subcommand2(), @@ -508,7 +516,7 @@ class MultipleSubparsers: ) assert dcargs.cli( MultipleSubparsers, - args=["subcommand1", "subcommand2", "subcommand3", "--c.z", "3"], + args=["a:subcommand1", "b:subcommand2", "c:subcommand3", "--c.z", "3"], default_instance=MultipleSubparsers( Subcommand1(), Subcommand2(), @@ -517,7 +525,7 @@ class MultipleSubparsers: ) == MultipleSubparsers(Subcommand1(x=0), Subcommand2(y=1), Subcommand3(z=3)) assert dcargs.cli( MultipleSubparsers, - args=["subcommand1", "subcommand2", "subcommand2"], + args=["a:subcommand1", "b:subcommand2", "c:subcommand2"], default_instance=MultipleSubparsers( Subcommand1(), Subcommand2(), @@ -546,19 +554,19 @@ class MultipleSubparsers: with pytest.raises(SystemExit): dcargs.cli(MultipleSubparsers, args=[]) with pytest.raises(SystemExit): - dcargs.cli(MultipleSubparsers, args=["subcommand2"]) + dcargs.cli(MultipleSubparsers, args=["a:subcommand2"]) assert dcargs.cli( - MultipleSubparsers, args="subcommand1 --a.x 3".split(" ") + MultipleSubparsers, args="a:subcommand1 --a.x 3".split(" ") ) == MultipleSubparsers(Subcommand1(3)) assert dcargs.cli( - MultipleSubparsers, args="subcommand2 subcommand3 --a.y.z 2".split(" ") + MultipleSubparsers, args="a:subcommand2 a.y:subcommand3 --a.y.z 2".split(" ") ) == MultipleSubparsers(Subcommand2(Subcommand3())) assert dcargs.cli( - MultipleSubparsers, args="subcommand2 subcommand3 --a.y.z 7".split(" ") + MultipleSubparsers, args="a:subcommand2 a.y:subcommand3 --a.y.z 7".split(" ") ) == MultipleSubparsers(Subcommand2(Subcommand3(7))) assert dcargs.cli( - MultipleSubparsers, args="subcommand2 subcommand1 --a.y.x 7".split(" ") + MultipleSubparsers, args="a:subcommand2 a.y:subcommand1 --a.y.x 7".split(" ") ) == MultipleSubparsers(Subcommand2(Subcommand1(7))) @@ -583,21 +591,21 @@ class MultipleSubparsers: with pytest.raises(SystemExit): dcargs.cli(MultipleSubparsers, args=[]) assert dcargs.cli( - MultipleSubparsers, args="subcommand1 subcommand1".split(" ") + MultipleSubparsers, args="a:subcommand1 b:subcommand1".split(" ") ) == MultipleSubparsers(Subcommand1(), Subcommand1()) assert dcargs.cli( - MultipleSubparsers, args="subcommand1 subcommand2 subcommand1".split(" ") + MultipleSubparsers, + args="a:subcommand1 b:subcommand2 b.y:subcommand1".split(" "), ) == MultipleSubparsers(Subcommand1(), Subcommand2(Subcommand1())) assert dcargs.cli( MultipleSubparsers, - args="subcommand2 subcommand1 subcommand2 subcommand1".split(" "), + args="a:subcommand2 a.y:subcommand1 b:subcommand2 b.y:subcommand1".split(" "), ) == MultipleSubparsers(Subcommand2(Subcommand1()), Subcommand2(Subcommand1())) assert dcargs.cli( MultipleSubparsers, args=( - "subcommand2 subcommand1 --a.y.x 3 subcommand2 subcommand1 --b.y.x 7".split( - " " - ) + "a:subcommand2 a.y:subcommand1 --a.y.x 3 b:subcommand2 b.y:subcommand1" + " --b.y.x 7".split(" ") ), ) == MultipleSubparsers(Subcommand2(Subcommand1(3)), Subcommand2(Subcommand1(7))) @@ -621,7 +629,7 @@ def main(x: Tuple[Tuple[Color], Location, float]): assert dcargs.cli( main, args=( - "--x:0:0.r 255 --x:0:0.g 0 --x:0:0.b 0 --x:1.x 5.0 --x:1.y 0.0" - " --x:1.z 2.0 --x:2 4.0".split(" ") + "--x.0.0.r 255 --x.0.0.g 0 --x.0.0.b 0 --x.1.x 5.0 --x.1.y 0.0" + " --x.1.z 2.0 --x.2 4.0".split(" ") ), ) == ((Color(255, 0, 0),), Location(5.0, 0.0, 2.0), 4.0) diff --git a/tests/test_nested_in_containers.py b/tests/test_nested_in_containers.py index d69696bb9..6e763b4b7 100644 --- a/tests/test_nested_in_containers.py +++ b/tests/test_nested_in_containers.py @@ -17,7 +17,7 @@ def test_nested_tuple_fixed_single(): def main(x: Tuple[Color]) -> Any: return x - assert dcargs.cli(main, args="--x:0.r 255 --x:0.g 127 --x:0.b 5".split(" ")) == ( + assert dcargs.cli(main, args="--x.0.r 255 --x.0.g 127 --x.0.b 5".split(" ")) == ( Color(255, 127, 5), ) @@ -29,7 +29,7 @@ def main(x: Tuple[Color, Color]) -> Any: assert dcargs.cli( main, args=( - "--x:0.r 255 --x:0.g 127 --x:0.b 5 --x:1.r 255 --x:1.g 127 --x:1.b 0" + "--x.0.r 255 --x.0.g 127 --x.0.b 5 --x.1.r 255 --x.1.g 127 --x.1.b 0" ).split(" "), ) == ( Color(255, 127, 5), @@ -44,8 +44,8 @@ def main(x: Tuple[Color, int, Color]) -> Any: assert dcargs.cli( main, args=( - "--x:0.r 255 --x:0.g 127 --x:0.b 5 --x:1 94709 --x:2.r 255 --x:2.g 127" - " --x:2.b 0" + "--x.0.r 255 --x.0.g 127 --x.0.b 5 --x.1 94709 --x.2.r 255 --x.2.g 127" + " --x.2.b 0" ).split(" "), ) == ( Color(255, 127, 5), @@ -61,8 +61,8 @@ def main(x: Tuple[Color, Tuple[Color, Color]]) -> Any: assert dcargs.cli( main, args=( - "--x:0.r 255 --x:0.g 127 --x:0.b 5 --x:1:0.r 255 --x:1:0.g 127 --x:1:0.b 0" - " --x:1:1.r 255 --x:1:1.g 127 --x:1:1.b 0" + "--x.0.r 255 --x.0.g 127 --x.0.b 5 --x.1.0.r 255 --x.1.0.g 127 --x.1.0.b 0" + " --x.1.1.r 255 --x.1.1.g 127 --x.1.1.b 0" ).split(" "), ) == ( Color(255, 127, 5), @@ -96,7 +96,7 @@ def main(x: List[Color] = [Color(255, 0, 0)]) -> Any: return x assert dcargs.cli(main, args=[]) == [Color(255, 0, 0)] - assert dcargs.cli(main, args="--x:0.r 127".split(" ")) == [Color(127, 0, 0)] + assert dcargs.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] def test_tuple_in_list(): @@ -104,7 +104,7 @@ def main(x: List[Tuple[Color]] = [(Color(255, 0, 0),)]) -> Any: return x assert dcargs.cli(main, args=[]) == [(Color(255, 0, 0),)] - assert dcargs.cli(main, args="--x:0:0.r 127".split(" ")) == [(Color(127, 0, 0),)] + assert dcargs.cli(main, args="--x.0.0.r 127".split(" ")) == [(Color(127, 0, 0),)] def test_tuple_variable(): @@ -112,7 +112,7 @@ def main(x: Tuple[Color, ...] = (Color(255, 0, 0), Color(255, 0, 127))) -> Any: return x assert dcargs.cli(main, args=[]) == (Color(255, 0, 0), Color(255, 0, 127)) - assert dcargs.cli(main, args="--x:0.r 127".split(" ")) == ( + assert dcargs.cli(main, args="--x.0.r 127".split(" ")) == ( Color(127, 0, 0), Color(255, 0, 127), ) @@ -154,7 +154,7 @@ def main( return x assert dcargs.cli(main, args=[])["green"] == (Color(0, 255, 0), 10) - assert dcargs.cli(main, args="--x.green:0.g 127 --x.green:1 2".split(" "))[ + assert dcargs.cli(main, args="--x.green.0.g 127 --x.green.1 2".split(" "))[ "green" ] == (Color(0, 127, 0), 2) @@ -174,7 +174,7 @@ def main(x: Tuple[GenericColor[float], GenericColor[int]]) -> Any: assert dcargs.cli( main, args=( - "--x:0.r 0.5 --x:0.g 0.2 --x:0.b 0.3 --x:1.r 25 --x:1.g 2 --x:1.b 3".split( + "--x.0.r 0.5 --x.0.g 0.2 --x.0.b 0.3 --x.1.r 25 --x.1.g 2 --x.1.b 3".split( " " ) ), @@ -201,7 +201,7 @@ def main( assert dcargs.cli( main, args=( - "--x:0.r 0.5 --x:0.g 0.2 --x:0.b 0.3 --x:1.r 25 --x:1.g 2 --x:1.b 3".split( + "--x.0.r 0.5 --x.0.g 0.2 --x.0.b 0.3 --x.1.r 25 --x.1.g 2 --x.1.b 3".split( " " ) ), @@ -228,7 +228,7 @@ def main( assert dcargs.cli( main, args=( - "--x:0.r 0.5 --x:0.g 0.9 --x:0.b 0.3 --x:1.r 25 --x:1.g 2 --x:1.b 3".split( + "--x.0.r 0.5 --x.0.g 0.9 --x.0.b 0.3 --x.1.r 25 --x.1.g 2 --x.1.b 3".split( " " ) ), From 5fe3467309352e0cd51ec05f8a9a35c65ef60219 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 14 Aug 2022 19:17:02 -0700 Subject: [PATCH 091/491] Minor fixes for TypedDict, generics --- dcargs/_fields.py | 12 +++++- dcargs/_parsers.py | 16 ++++--- docs/source/examples/09_subparsers.rst | 18 ++++---- .../examples/10_multiple_subparsers.rst | 20 ++++----- docs/source/examples/12_tuples.rst | 4 +- docs/source/examples/14_generics.rst | 4 +- examples/14_generics.py | 4 +- tests/test_dict_namedtuple.py | 42 +++++++++++++++++++ tests/test_nested.py | 22 +++++++++- 9 files changed, 107 insertions(+), 35 deletions(-) diff --git a/dcargs/_fields.py b/dcargs/_fields.py index ba1e0ba30..8f255e0a2 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -77,6 +77,8 @@ class ExcludeFromCallType(_Singleton): # nonpropagating missing sentinel, which does not override child defaults. MISSING_PROP = PropagatingMissingType() MISSING_NONPROP = NonpropagatingMissingType() + +# When total=False in a TypedDict, we exclude fields from the constructor by default. EXCLUDE_FROM_CALL = ExcludeFromCallType() # Note that our "public" missing API will always be the propagating missing sentinel. @@ -102,7 +104,9 @@ class ExcludeFromCallType(_Singleton): T = TypeVar("T") -DefaultInstanceT = Union[T, PropagatingMissingType, NonpropagatingMissingType] +DefaultInstanceT = Union[ + T, PropagatingMissingType, NonpropagatingMissingType, ExcludeFromCallType +] _known_parsable_types = set( filter( @@ -324,7 +328,11 @@ def _field_list_from_tuple( assert isinstance(default_instance, tuple) children = tuple(type(x) for x in default_instance) - if default_instance in MISSING_SINGLETONS: + if ( + default_instance in MISSING_SINGLETONS + # EXCLUDE_FROM_CALL indicates we're inside a TypedDict, with total=False. + or default_instance is EXCLUDE_FROM_CALL + ): default_instance = (default_instance,) * len(children) for i, child in enumerate(children): diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index f2cb9eefe..a2ba461cc 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -72,9 +72,10 @@ def from_callable( helptext_from_nested_class_field_name = {} subparsers_from_name = {} - for field in _fields.field_list_from_callable( + field_list = _fields.field_list_from_callable( f=f, default_instance=default_instance, root_field=True - ): + ) + for field in field_list: field = dataclasses.replace( field, typ=_resolver.type_from_typevar_constraints( @@ -265,10 +266,8 @@ def from_field( description=None, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, - # Only pass default in if it corresponds to this option. - # TODO: this will break for generics! default_instance=field.default - if isinstance(field.default, _resolver.unwrap_origin(option)) # type: ignore + if type(field.default) == _resolver.unwrap_origin(option) # type: ignore else _fields.MISSING_NONPROP, prefix=prefix, avoid_subparsers=avoid_subparsers, @@ -285,6 +284,13 @@ def from_field( field.default is not None and field.default not in _fields.MISSING_SINGLETONS ): + # It's really hard to concretize a generic type at runtime, so we just... + # don't. :-) + if hasattr(type(field.default), "__parameters__"): + raise _instantiators.UnsupportedTypeAnnotationError( + "Default values for generic subparsers are not supported." + ) + default_parser = parser_from_name[ _strings.subparser_name_from_type(prefix, type(field.default)) ] diff --git a/docs/source/examples/09_subparsers.rst b/docs/source/examples/09_subparsers.rst index c6fb8c5da..6299c1d91 100644 --- a/docs/source/examples/09_subparsers.rst +++ b/docs/source/examples/09_subparsers.rst @@ -35,7 +35,7 @@ Unions over nested types (classes or dataclasses) are populated using subparsers all: bool = False - def main(cmd: Union[Checkout, Commit]) -> None: + def main(cmd: Union[Checkout, Commit, None]) -> None: print(cmd) @@ -54,30 +54,30 @@ Unions over nested types (classes or dataclasses) are populated using subparsers .. raw:: html - python 09_subparsers.py commit --help + python 09_subparsers.py cmd:commit --help -.. program-output:: python ../../examples/09_subparsers.py commit --help +.. program-output:: python ../../examples/09_subparsers.py cmd:commit --help ------------ .. raw:: html - python 09_subparsers.py commit --cmd.message hello --cmd.all + python 09_subparsers.py cmd:commit --cmd.message hello --cmd.all -.. program-output:: python ../../examples/09_subparsers.py commit --cmd.message hello --cmd.all +.. program-output:: python ../../examples/09_subparsers.py cmd:commit --cmd.message hello --cmd.all ------------ .. raw:: html - python 09_subparsers.py checkout --help + python 09_subparsers.py cmd:checkout --help -.. program-output:: python ../../examples/09_subparsers.py checkout --help +.. program-output:: python ../../examples/09_subparsers.py cmd:checkout --help ------------ .. raw:: html - python 09_subparsers.py checkout --cmd.branch main + python 09_subparsers.py cmd:checkout --cmd.branch main -.. program-output:: python ../../examples/09_subparsers.py checkout --cmd.branch main +.. program-output:: python ../../examples/09_subparsers.py cmd:checkout --cmd.branch main diff --git a/docs/source/examples/10_multiple_subparsers.rst b/docs/source/examples/10_multiple_subparsers.rst index 88ef9e2fd..ee2fd7330 100644 --- a/docs/source/examples/10_multiple_subparsers.rst +++ b/docs/source/examples/10_multiple_subparsers.rst @@ -23,13 +23,13 @@ Multiple unions over nested types are populated using a series of subparsers. @dataclasses.dataclass - class MnistDataset: + class Mnist: binary: bool = False """Set to load binary version of MNIST dataset.""" @dataclasses.dataclass - class ImageNetDataset: + class ImageNet: subset: Literal[50, 100, 1000] """Choose between ImageNet-50, ImageNet-100, ImageNet-1000, etc.""" @@ -38,13 +38,13 @@ Multiple unions over nested types are populated using a series of subparsers. @dataclasses.dataclass - class AdamOptimizer: + class Adam: learning_rate: float = 1e-3 betas: Tuple[float, float] = (0.9, 0.999) @dataclasses.dataclass - class SgdOptimizer: + class Sgd: learning_rate: float = 3e-4 @@ -52,8 +52,8 @@ Multiple unions over nested types are populated using a series of subparsers. def train( - dataset: Union[MnistDataset, ImageNetDataset] = MnistDataset(), - optimizer: Union[AdamOptimizer, SgdOptimizer] = AdamOptimizer(), + dataset: Union[Mnist, ImageNet] = Mnist(), + optimizer: Union[Adam, Sgd] = Adam(), ) -> None: """Example training script. @@ -91,14 +91,14 @@ Multiple unions over nested types are populated using a series of subparsers. .. raw:: html - python 10_multiple_subparsers.py mnist-dataset --help + python 10_multiple_subparsers.py dataset:mnist --help -.. program-output:: python ../../examples/10_multiple_subparsers.py mnist-dataset --help +.. program-output:: python ../../examples/10_multiple_subparsers.py dataset:mnist --help ------------ .. raw:: html - python 10_multiple_subparsers.py mnist-dataset adam-optimizer --optimizer.learning-rate 3e-4 + python 10_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 -.. program-output:: python ../../examples/10_multiple_subparsers.py mnist-dataset adam-optimizer --optimizer.learning-rate 3e-4 +.. program-output:: python ../../examples/10_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 diff --git a/docs/source/examples/12_tuples.rst b/docs/source/examples/12_tuples.rst index 08af30686..b3303daf2 100644 --- a/docs/source/examples/12_tuples.rst +++ b/docs/source/examples/12_tuples.rst @@ -60,6 +60,6 @@ Example using ``dcargs.cli()`` to instantiate tuple types. .. raw:: html - python 12_tuples.py --two-colors:1.r 127 --two-colors:1.g 0 --two-colors:1.b 0 + python 12_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0 -.. program-output:: python ../../examples/12_tuples.py --two-colors:1.r 127 --two-colors:1.g 0 --two-colors:1.b 0 +.. program-output:: python ../../examples/12_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0 diff --git a/docs/source/examples/14_generics.rst b/docs/source/examples/14_generics.rst index a65f6232b..7216c5f98 100644 --- a/docs/source/examples/14_generics.rst +++ b/docs/source/examples/14_generics.rst @@ -17,7 +17,7 @@ Example of parsing for generic dataclasses. import dcargs - ScalarType = TypeVar("ScalarType") + ScalarType = TypeVar("ScalarType", int, float) ShapeType = TypeVar("ShapeType") @@ -38,8 +38,6 @@ Example of parsing for generic dataclasses. @dataclasses.dataclass(frozen=True) class Args(Generic[ShapeType]): - point_continuous: Point3[float] - point_discrete: Point3[int] shape: ShapeType diff --git a/examples/14_generics.py b/examples/14_generics.py index c377e06ef..2450cd7d5 100644 --- a/examples/14_generics.py +++ b/examples/14_generics.py @@ -9,7 +9,7 @@ import dcargs -ScalarType = TypeVar("ScalarType") +ScalarType = TypeVar("ScalarType", int, float) ShapeType = TypeVar("ShapeType") @@ -30,8 +30,6 @@ class Triangle: @dataclasses.dataclass(frozen=True) class Args(Generic[ShapeType]): - point_continuous: Point3[float] - point_discrete: Point3[int] shape: ShapeType diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index cfe791b06..0c33b90de 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -1,4 +1,5 @@ import contextlib +import dataclasses import io import pathlib from typing import Any, Dict, Mapping, NamedTuple, Tuple, Union, cast @@ -114,6 +115,47 @@ class ParentTypedDict(TypedDict, total=False): ) +def test_total_false_typeddict_with_nested(): + @dataclasses.dataclass + class Inner: + j: float + + class ManyTypesTypedDict(TypedDict, total=False): + i: int + s: Inner + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli( + ManyTypesTypedDict, + args="".split(" "), + ) + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli( + ManyTypesTypedDict, + args="--x.i 5 --x.s 5 5".split(" "), + ) + + +def test_total_false_typeddict_with_tuple(): + class ManyTypesTypedDict(TypedDict, total=False): + i: int + s: Tuple[str, str] + + assert ( + dcargs.cli( + ManyTypesTypedDict, + args=[], + ) + == dict() + ) + + assert dcargs.cli( + ManyTypesTypedDict, + args="--i 5 --s 5 5".split(" "), + ) == dict(i=5, s=("5", "5")) + + def test_nested_typeddict(): class ChildTypedDict(TypedDict): y: int diff --git a/tests/test_nested.py b/tests/test_nested.py index 02278dc6c..3cbb02c75 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Optional, Tuple, Union +from typing import Any, Generic, Optional, Tuple, TypeVar, Union import pytest @@ -633,3 +633,23 @@ def main(x: Tuple[Tuple[Color], Location, float]): " --x.1.z 2.0 --x.2 4.0".split(" ") ), ) == ((Color(255, 0, 0),), Location(5.0, 0.0, 2.0), 4.0) + + +def test_generic_subparsers(): + T = TypeVar("T") + + @dataclasses.dataclass + class A(Generic[T]): + x: T + + def main(x: Union[A[int], A[float]]) -> Any: + return x + + assert dcargs.cli(main, args="x:a-float --x.x 3.2".split(" ")) == A(3.2) + assert dcargs.cli(main, args="x:a-int --x.x 3".split(" ")) == A(3) + + def main_with_default(x: Union[A[int], A[float]] = A(5)) -> Any: + return x + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli(main_with_default, args=[]) From 63628dff4d9b1b0f216bba55917012b106a66b66 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 16 Aug 2022 23:05:31 -0700 Subject: [PATCH 092/491] Clean up field generation logic --- dcargs/_fields.py | 280 +++++++++++++++++++++++----------------------- 1 file changed, 141 insertions(+), 139 deletions(-) diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 8f255e0a2..07a23f0a9 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -1,24 +1,15 @@ """Abstractions for pulling out 'field' definitions, which specify inputs, types, and defaults, from general callables.""" +from __future__ import annotations + import collections import dataclasses import inspect import itertools import typing import warnings -from typing import ( - Any, - Callable, - Hashable, - Iterable, - List, - Optional, - Type, - TypeVar, - Union, - cast, -) +from typing import Any, Callable, Hashable, Iterable, List, Optional, Type, Union, cast import docstring_parser import typing_extensions @@ -27,15 +18,6 @@ from . import _docstrings, _instantiators, _resolver -class UnsupportedNestedTypeAnnotationError( - _instantiators.UnsupportedTypeAnnotationError -): - """Narrower version of UnsupportedNestedTypeAnnotationError, which signifies that a type - cannot correspond to a nested field. It may still be parsed directly.""" - - pass - - @dataclasses.dataclass(frozen=True) class FieldDefinition: name: str @@ -102,52 +84,67 @@ class ExcludeFromCallType(_Singleton): except ImportError: pass -T = TypeVar("T") - -DefaultInstanceT = Union[ - T, PropagatingMissingType, NonpropagatingMissingType, ExcludeFromCallType -] -_known_parsable_types = set( - filter( - lambda x: isinstance(x, Hashable), # type: ignore - itertools.chain( - __builtins__.values(), # type: ignore - vars(typing).values(), - vars(typing_extensions).values(), - vars(collections.abc).values(), - ), - ) -) +@dataclasses.dataclass(frozen=True) +class UnsupportedNestedTypeMessage: + """Reason why a callable cannot be treated as a nested type.""" + message: str -def is_nested_type(typ: Any, default_instance: Any) -> bool: - """Determine whether a type should be treated as a 'nested type', where a single field - has multiple corresponding arguments (eg for nested dataclasses or classes).""" - try: - # This implementation will result in some computational redundancy, but is nice - # for reusing logic. Could be revisited. - field_list_from_callable(typ, default_instance, root_field=False) - return True - except UnsupportedNestedTypeAnnotationError: - return False +def is_nested_type(typ: Type, default_instance: _DefaultInstance) -> bool: + """Determine whether a type should be treated as a 'nested type', where a single + type can be broken down into multiple fields (eg for nested dataclasses or + classes).""" + return not isinstance( + _try_field_list_from_callable(typ, default_instance, root_field=False), + UnsupportedNestedTypeMessage, + ) def field_list_from_callable( - f: Callable[..., T], - default_instance: DefaultInstanceT, + f: Union[Callable, Type], + default_instance: _DefaultInstance, root_field: bool = False, ) -> List[FieldDefinition]: """Generate a list of generic 'field' objects corresponding to the inputs of some annotated callable. - `f` can be from a dataclass type, regular class type, or function. - If `root_field` is set to True, we treat `int`, `torch.device`, etc as nested fields. This is to make sure that these types can be passed directly into dcargs.cli(); the logic can likely be refactored.""" + out = _try_field_list_from_callable(f, default_instance, root_field) + + if isinstance(out, UnsupportedNestedTypeMessage): + raise _instantiators.UnsupportedTypeAnnotationError(out.message) + return out + +# Implementation details below. + + +_DefaultInstance = Union[ + Any, PropagatingMissingType, NonpropagatingMissingType, ExcludeFromCallType +] + +_known_parsable_types = set( + filter( + lambda x: isinstance(x, Hashable), # type: ignore + itertools.chain( + __builtins__.values(), # type: ignore + vars(typing).values(), + vars(typing_extensions).values(), + vars(collections.abc).values(), + ), + ) +) + + +def _try_field_list_from_callable( + f: Union[Callable, Type], + default_instance: _DefaultInstance, + root_field: bool, +) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Unwrap generics. f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) @@ -164,66 +161,71 @@ def field_list_from_callable( # Try special cases. if cls is not None and is_typeddict(cls): - return _field_list_from_typeddict(cls, default_instance) + return _try_field_list_from_typeddict(cls, default_instance) elif cls is not None and _resolver.is_namedtuple(cls): - return _field_list_from_namedtuple(cls, default_instance) + return _try_field_list_from_namedtuple(cls, default_instance) elif cls is not None and _resolver.is_dataclass(cls): - return _field_list_from_dataclass(cls, default_instance) + return _try_field_list_from_dataclass(cls, default_instance) # Standard container types. These are special because they can be nested structures - # when - try: - # Note that f_origin will be populated if we annotate as `Tuple[..]`, and cls will - # be populated if we annotated as just `tuple`. - if f_origin is tuple or cls is tuple: - out = _field_list_from_tuple(f, default_instance) - return out - - elif f_origin in (list, set, typing.Sequence) or cls in ( - list, - set, - typing.Sequence, - ): - if len(get_args(f)) == 0: - if default_instance in MISSING_SINGLETONS: - raise _instantiators.UnsupportedTypeAnnotationError( - f"Sequence type {cls} needs either an explicit type or a" - " default to infer from." - ) - assert isinstance(default_instance, Iterable) - contained_type = MISSING_NONPROP - else: - (contained_type,) = get_args(f) - f_origin = list if f_origin is typing.Sequence else f_origin # type: ignore - return _field_list_from_sequence( - contained_type, # type: ignore - default_instance, - ) - elif f_origin is dict: - return _field_list_from_dict(f, default_instance) - except UnsupportedNestedTypeAnnotationError as e: - # For the root field case, we can try again in the general case. - if not root_field: - raise e + # if they contain other nested types (eg Tuple[Struct, Struct]), or treated as + # single arguments otherwise (eg Tuple[int, int]). + # + # Note that f_origin will be populated if we annotate as `Tuple[..]`, and cls will + # be populated if we annotated as just `tuple`. + container_fields = None + if f_origin is tuple or cls is tuple: + container_fields = _field_list_from_tuple(f, default_instance) + elif f_origin in (list, set, typing.Sequence) or cls in ( + list, + set, + typing.Sequence, + ): + contained_type: Any + if len(get_args(f)) == 0: + if default_instance in MISSING_SINGLETONS: + raise _instantiators.UnsupportedTypeAnnotationError( + f"Sequence type {cls} needs either an explicit type or a" + " default to infer from." + ) + assert isinstance(default_instance, Iterable) + else: + (contained_type,) = get_args(f) + f_origin = list if f_origin is typing.Sequence else f_origin # type: ignore + + container_fields = _try_field_list_from_sequence( + contained_type, default_instance + ) + elif f_origin is dict: + container_fields = _try_field_list_from_dict(f, default_instance) + + # Check if one of the container types matched. + if container_fields is not None: + # Found fields! + if not isinstance(container_fields, UnsupportedNestedTypeMessage): + return container_fields + # Got an error -> give up if not the root field. + elif not root_field: + assert isinstance(container_fields, UnsupportedNestedTypeMessage) + return container_fields # General cases. if not root_field and ( (cls is not None and cls in _known_parsable_types) or _resolver.unwrap_origin(f) in _known_parsable_types ): - raise UnsupportedNestedTypeAnnotationError(f"{f} should be parsed directly!") - + return UnsupportedNestedTypeMessage(f"{f} should be parsed directly!") else: - return _field_list_from_general_callable( + return _try_field_list_from_general_callable( f, cls, default_instance, root_field=root_field ) -def _field_list_from_typeddict( - cls: Type[T], default_instance: DefaultInstanceT -) -> List[FieldDefinition]: +def _try_field_list_from_typeddict( + cls: Type, default_instance: _DefaultInstance +) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: field_list = [] valid_default_instance = ( default_instance not in MISSING_SINGLETONS @@ -236,7 +238,7 @@ def _field_list_from_typeddict( elif getattr(cls, "__total__") is False: default = EXCLUDE_FROM_CALL if is_nested_type(typ, MISSING_NONPROP): - raise UnsupportedNestedTypeAnnotationError( + return UnsupportedNestedTypeMessage( "`total=False` not supported for nested structures." ) else: @@ -254,9 +256,9 @@ def _field_list_from_typeddict( return field_list -def _field_list_from_namedtuple( - cls: Type[T], default_instance: DefaultInstanceT -) -> List[FieldDefinition]: +def _try_field_list_from_namedtuple( + cls: Type, default_instance: _DefaultInstance +) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Handle NamedTuples. # # TODO: in terms of helptext, we currently do display the default NamedTuple @@ -286,9 +288,9 @@ def _field_list_from_namedtuple( return field_list -def _field_list_from_dataclass( - cls: Type[T], default_instance: DefaultInstanceT -) -> List[FieldDefinition]: +def _try_field_list_from_dataclass( + cls: Type, default_instance: _DefaultInstance +) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Handle dataclasses. field_list = [] for dc_field in filter(lambda field: field.init, _resolver.resolved_fields(cls)): @@ -306,13 +308,13 @@ def _field_list_from_dataclass( def _field_list_from_tuple( - f: Callable, default_instance: DefaultInstanceT -) -> List[FieldDefinition]: + f: Union[Callable, Type], default_instance: _DefaultInstance +) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Fixed-length tuples. field_list = [] children = get_args(f) if Ellipsis in children: - return _field_list_from_sequence( + return _try_field_list_from_sequence( next(iter(set(children) - {Ellipsis})), default_instance ) @@ -357,24 +359,24 @@ def _field_list_from_tuple( if not contains_nested: # We could also check for variable length children, which can be populated when # the tuple is interpreted as a nested field but not a directly parsed one. - raise UnsupportedNestedTypeAnnotationError( + return UnsupportedNestedTypeMessage( "Tuple does not contain any nested structures." ) return field_list -def _field_list_from_sequence( +def _try_field_list_from_sequence( contained_type: Type, - default_instance: DefaultInstanceT, -) -> List[FieldDefinition]: + default_instance: _DefaultInstance, +) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # When no default instance is specified: # If we have List[int] => this can be parsed as a single field. # If we have List[SomeStruct] => OK. if default_instance in MISSING_SINGLETONS and not is_nested_type( contained_type, MISSING_NONPROP ): - raise UnsupportedNestedTypeAnnotationError( + return UnsupportedNestedTypeMessage( f"Sequence containing type {contained_type} should be parsed directly!" ) @@ -384,7 +386,7 @@ def _field_list_from_sequence( if isinstance(default_instance, Iterable) and all( [not is_nested_type(type(x), x) for x in default_instance] ): - raise UnsupportedNestedTypeAnnotationError( + return UnsupportedNestedTypeMessage( f"Sequence with default {default_instance} should be parsed directly!" ) if default_instance in MISSING_SINGLETONS: @@ -409,12 +411,12 @@ def _field_list_from_sequence( return field_list -def _field_list_from_dict( - f: Callable, - default_instance: DefaultInstanceT, -) -> List[FieldDefinition]: +def _try_field_list_from_dict( + f: Union[Callable, Type], + default_instance: _DefaultInstance, +) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: if default_instance in MISSING_SINGLETONS: - raise UnsupportedNestedTypeAnnotationError( + return UnsupportedNestedTypeMessage( "Nested dictionary structures must have a default instance specified." ) field_list = [] @@ -431,22 +433,22 @@ def _field_list_from_dict( return field_list -def _field_list_from_general_callable( - f: Callable, +def _try_field_list_from_general_callable( + f: Union[Callable, Type], cls: Optional[Type], - default_instance: DefaultInstanceT, + default_instance: _DefaultInstance, root_field: bool, -) -> List[FieldDefinition]: +) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Handle general callables. if default_instance not in MISSING_SINGLETONS: - raise UnsupportedNestedTypeAnnotationError( + return UnsupportedNestedTypeMessage( "`default_instance` is supported only for select types:" " dataclasses, lists, NamedTuple, TypedDict, etc." ) # Generate field list from function signature. if not callable(f): - raise UnsupportedNestedTypeAnnotationError( + return UnsupportedNestedTypeMessage( f"Cannot extract annotations from {f}, which is not a callable type." ) params = list(inspect.signature(f).parameters.values()) @@ -454,15 +456,14 @@ def _field_list_from_general_callable( # Ignore self parameter. params = params[1:] - try: - return _field_list_from_params(f, cls, params) - except (UnsupportedNestedTypeAnnotationError, TypeError) as e: - if not root_field: - raise e - - # Try to support passing things like int, str, Dict[K,V], torch.device - # directly into dcargs.cli(). These aren't "type-annotated callables" but - # this a nice-to-have. + out = _field_list_from_params(f, cls, params) + if not isinstance(out, UnsupportedNestedTypeMessage): + return out + + # Try to support passing things like int, str, Dict[K,V], torch.device + # directly into dcargs.cli(). These aren't "type-annotated callables" but + # this a nice-to-have. + if root_field: param_count = 0 has_kw_only = False has_var_positional = False @@ -486,7 +487,7 @@ def _field_list_from_general_callable( ): # Things look ok! if cls is not None: - f = cls + f = cls # type: ignore return [ FieldDefinition( name=_resolver.unwrap_origin(f).__name__, @@ -496,13 +497,15 @@ def _field_list_from_general_callable( positional=True, ) ] - else: - raise e + + # Return error message. + assert isinstance(out, UnsupportedNestedTypeMessage) + return out def _field_list_from_params( - f: Callable, cls: Optional[Type], params: List[inspect.Parameter] -) -> List[FieldDefinition]: + f: Union[Callable, Type], cls: Optional[Type], params: List[inspect.Parameter] +) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Get type annotations, docstrings. docstring = inspect.getdoc(f) docstring_from_arg_name = {} @@ -515,7 +518,7 @@ def _field_list_from_params( try: hints = get_type_hints(f) except TypeError: - raise UnsupportedNestedTypeAnnotationError(f"Could not get hints for {f}!") + return UnsupportedNestedTypeMessage(f"Could not get hints for {f}!") field_list = [] for param in params: @@ -528,7 +531,7 @@ def _field_list_from_params( helptext = _docstrings.get_field_docstring(cls, param.name) if param.name not in hints: - raise UnsupportedNestedTypeAnnotationError( + return UnsupportedNestedTypeMessage( f"Expected fully type-annotated callable, but {f} with arguments" f" {tuple(map(lambda p: p.name, params))} has no annotation for" f" '{param.name}'." @@ -600,8 +603,7 @@ def _get_dataclass_field_default( # Special case to ignore default_factory if we write: # `field: Dataclass = dataclasses.field(default_factory=Dataclass)`. # - # In other words, treat it the same way as: - # `field: Dataclass`. + # In other words, treat it the same way as: `field: Dataclass`. # # The only time this matters is when we our dataclass has a `__post_init__` # function that mutates the dataclass. We choose here to use the default values From 3a40774530472675d8fe425d45768cff28af10e7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 16 Aug 2022 23:35:57 -0700 Subject: [PATCH 093/491] Tests for nesting in containers + fixed values --- dcargs/_arguments.py | 8 ++++---- tests/test_helptext.py | 21 +++++++++++++++++++++ tests/test_nested_in_containers.py | 29 +++++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 83fb08863..59ba622aa 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -179,9 +179,7 @@ def _rule_recursive_instantiator_from_type( # available. return dataclasses.replace( lowered, - metavar=termcolor.colored( - "{" + str(arg.field.default) + "}", color="red" - ), + metavar=termcolor.colored("{fixed}", color="red"), required=False, default=_fields.MISSING_PROP, ) @@ -253,7 +251,9 @@ def _rule_generate_helptext( # because the types of all arguments are set to strings, which will cause the # default to be casted to a string and introduce extra quotation marks. if lowered.instantiator is None: - default_text = "(fixed)" + # Intentionally not quoted via shlex, since this can't actually be passed + # in via the commandline. + default_text = f"(fixed to: {str(arg.field.default)})" elif lowered.action == "store_true": default_text = f"(sets: {arg.field.name}=True)" elif lowered.action == "store_false": diff --git a/tests/test_helptext.py b/tests/test_helptext.py index a9f9b235b..929e2c9a1 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -7,6 +7,7 @@ from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar, Union, cast import pytest +import torch.nn as nn from typing_extensions import Literal import dcargs @@ -511,3 +512,23 @@ class Something( helptext = _get_helptext(Something) assert "This text should not" not in helptext assert "But this text should!" in helptext + + +def test_unparsable(): + @dataclasses.dataclass + class Struct: + a: int = 5 + b: str = "7" + + def main(x: Any = Struct()): + pass + + helptext = _get_helptext(main) + assert "--x {fixed}" in helptext + + def main(x: Callable = nn.ReLU): + pass + + helptext = _get_helptext(main) + assert "--x {fixed}" in helptext + assert "(fixed to: None: dcargs.cli(main, args=[]) +def test_set_bad(): + # Unable to infer input length. + def main(x: Set[Color]) -> None: + pass + + with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + dcargs.cli(main, args=[]) + + +def test_set_ok(): + def main(x: Set[Color] = {Color(255, 0, 0)}) -> Any: + return x + + assert dcargs.cli(main, args=[]) == {Color(255, 0, 0)} + assert dcargs.cli(main, args="--x.0.r 127".split(" ")) == {Color(127, 0, 0)} + + def test_list_bad(): # Unable to infer input length. def main(x: List[Color]) -> None: @@ -99,6 +116,14 @@ def main(x: List[Color] = [Color(255, 0, 0)]) -> Any: assert dcargs.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] +def test_list_object(): + def main(x: List[object] = [Color(255, 0, 0)]) -> Any: + return x + + assert dcargs.cli(main, args=[]) == [Color(255, 0, 0)] + assert dcargs.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] + + def test_tuple_in_list(): def main(x: List[Tuple[Color]] = [(Color(255, 0, 0),)]) -> Any: return x From febee9393d443eaad2fa06e1f9b6f72d600cda86 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 16 Aug 2022 23:56:29 -0700 Subject: [PATCH 094/491] Support dictionaries with non-string keys --- dcargs/_calling.py | 9 +++++++- dcargs/_fields.py | 10 ++++++++- tests/test_nested_in_containers.py | 36 ++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 51c502891..cc8edda32 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -177,7 +177,9 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: if field.positional: args.append(value) else: - kwargs[field.name] = value + kwargs[ + field.name if field.name_override is None else field.name_override + ] = value unwrapped_f = _resolver.unwrap_origin(f) unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore @@ -191,5 +193,10 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: # single set of positional arguments. assert len(args) == 1 return unwrapped_f(args[0]), consumed_keywords # type: ignore + elif unwrapped_f is dict: + for arg in args: + assert isinstance(arg, dict) + kwargs.update(arg) + return kwargs, consumed_keywords # type: ignore else: return unwrapped_f(*args, **kwargs), consumed_keywords # type: ignore diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 07a23f0a9..66e8363a6 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -5,6 +5,7 @@ import collections import dataclasses +import enum import inspect import itertools import typing @@ -26,6 +27,11 @@ class FieldDefinition: helptext: Optional[str] positional: bool + # Override the name in our kwargs. Currently only used for dictionary types when + # the key values aren't strings, but in the future could be used whenever the + # user-facing argument name doesn't match the keyword expected by our callable. + name_override: Optional[Any] = None + class _Singleton: # Singleton pattern. @@ -423,11 +429,13 @@ def _try_field_list_from_dict( for k, v in cast(dict, default_instance).items(): field_list.append( FieldDefinition( - name=str(k), + name=str(k) if not isinstance(k, enum.Enum) else k.name, typ=type(v), default=v, helptext=None, positional=False, + # Dictionary specific key: + name_override=k, ) ) return field_list diff --git a/tests/test_nested_in_containers.py b/tests/test_nested_in_containers.py index 8e9b3f9d1..c914ef35d 100644 --- a/tests/test_nested_in_containers.py +++ b/tests/test_nested_in_containers.py @@ -1,4 +1,5 @@ import dataclasses +import enum from typing import Any, Dict, Generic, List, Set, Tuple, TypeVar import pytest @@ -167,6 +168,41 @@ def main( ) +def test_dict_key_int(): + def main( + x: Dict[int, Color] = { + 0: Color(255, 0, 0), + 1: Color(0, 255, 0), + 2: Color(0, 0, 255), + } + ) -> Any: + return x + + assert dcargs.cli(main, args=[])[1] == Color(0, 255, 0) + assert dcargs.cli(main, args="--x.1.g 127".split(" "))[1] == Color(0, 127, 0) + + +def test_dict_key_enum(): + class ColorType(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + def main( + x: Dict[ColorType, Color] = { + ColorType.RED: Color(255, 0, 0), + ColorType.GREEN: Color(0, 255, 0), + ColorType.BLUE: Color(0, 0, 255), + } + ) -> Any: + return x + + assert dcargs.cli(main, args=[])[ColorType.GREEN] == Color(0, 255, 0) + assert dcargs.cli(main, args="--x.GREEN.g 127".split(" "))[ + ColorType.GREEN + ] == Color(0, 127, 0) + + def test_dict_nested(): def main( x: Dict[str, Tuple[Color, int]] = { From de2148d42303c4f3345989f06c900f835013ce91 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 21 Aug 2022 12:54:45 -0700 Subject: [PATCH 095/491] Generalize logic for direct type inputs --- dcargs/_arguments.py | 8 +++- dcargs/_calling.py | 8 +--- dcargs/_cli.py | 28 +++++++++++++- dcargs/_fields.py | 81 ++++++++++----------------------------- dcargs/_parsers.py | 4 +- dcargs/_strings.py | 18 +++++++-- examples/09_subparsers.py | 5 ++- tests/test_nested.py | 19 +++++++++ 8 files changed, 93 insertions(+), 78 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 59ba622aa..d10df3c6a 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -51,6 +51,8 @@ def add_argument( kwargs.pop("instantiator") kwargs = {k: v for k, v in kwargs.items() if v is not None} name_or_flag = kwargs.pop("name_or_flag") + if len(name_or_flag) == 0: + name_or_flag = _strings.dummy_field_name # We're actually going to skip the default field: if an argument is unset, the # MISSING value will be detected in _calling.py and the field default will @@ -236,7 +238,9 @@ def _rule_generate_helptext( # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string docstring_help = docstring_help.replace("%", "%%") help_parts.append(docstring_help) - elif arg.field.positional: + elif arg.field.positional and arg.field.name != _strings.dummy_field_name: + # Place the type in the helptext. Note that we skip this for dummy fields, which + # will sitll have the type in the metavar. help_parts.append(str(lowered.metavar)) default = lowered.default @@ -330,6 +334,6 @@ def _rule_positional_special_handling( lowered, dest=None, required=None, # Can't be passed in for positionals. - metavar=metavar, + metavar=metavar if len(metavar) > 0 else lowered.metavar, nargs=nargs, ) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index cc8edda32..d81f8e0f5 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -50,9 +50,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: ] = arg for field in _fields.field_list_from_callable( - f, - default_instance=default_instance, - root_field=True, + f, default_instance=default_instance ): # type: ignore value: Any prefixed_field_name = _strings.make_field_name([field_name_prefix, field.name]) @@ -121,9 +119,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: subparser_def = parser_definition.subparsers_from_name[field.name] - subparser_dest = _strings.SUBPARSER_DEST_FMT.format( - name=prefixed_field_name - ) + subparser_dest = _strings.make_subparser_dest(name=prefixed_field_name) consumed_keywords.add(subparser_dest) if subparser_dest in value_from_prefixed_field_name: subparser_name = get_value_from_arg(subparser_dest) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 808d9b2ab..11222cc80 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -1,11 +1,12 @@ """Core public API.""" import argparse -from typing import Callable, Optional, Sequence, TypeVar, Union, overload +import dataclasses +from typing import Callable, Optional, Sequence, Type, TypeVar, Union, cast, overload from typing_extensions import Literal, assert_never -from . import _argparse_formatter, _calling, _fields, _parsers +from . import _argparse_formatter, _calling, _fields, _parsers, _strings T = TypeVar("T") @@ -155,6 +156,20 @@ def _cli_impl( _fields.MISSING_NONPROP if default_instance is None else default_instance ) + if not _fields.is_nested_type(cast(Type, f), default_instance_internal): + dummy_field = dataclasses.field( + default=default_instance + if default_instance is not None + else dataclasses.MISSING + ) + f = dataclasses.make_dataclass( + cls_name="", + fields=[(_strings.dummy_field_name, cast(Type, f), dummy_field)], + ) + dummy_wrapped = True + else: + dummy_wrapped = False + # Map a callable to the relevant CLI arguments + subparsers. parser_definition = _parsers.ParserSpecification.from_callable( f, @@ -176,6 +191,12 @@ def _cli_impl( return parser value_from_prefixed_field_name = vars(parser.parse_args(args=args)) + if dummy_wrapped: + value_from_prefixed_field_name = { + k.replace(_strings.dummy_field_name, ""): v + for k, v in value_from_prefixed_field_name.items() + } + try: # Attempt to call `f` using whatever was passed in. out, consumed_keywords = _calling.call_from_args( @@ -197,6 +218,9 @@ def _cli_impl( f"Parsed {value_from_prefixed_field_name.keys()}, but only consumed" f" {consumed_keywords}" ) + + if dummy_wrapped: + out = getattr(out, _strings.dummy_field_name) if _return_stage == "f_out": return out diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 66e8363a6..896e8f4fd 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -16,7 +16,7 @@ import typing_extensions from typing_extensions import get_args, get_type_hints, is_typeddict -from . import _docstrings, _instantiators, _resolver +from . import _docstrings, _instantiators, _resolver, _strings @dataclasses.dataclass(frozen=True) @@ -103,7 +103,7 @@ def is_nested_type(typ: Type, default_instance: _DefaultInstance) -> bool: type can be broken down into multiple fields (eg for nested dataclasses or classes).""" return not isinstance( - _try_field_list_from_callable(typ, default_instance, root_field=False), + _try_field_list_from_callable(typ, default_instance), UnsupportedNestedTypeMessage, ) @@ -111,15 +111,10 @@ def is_nested_type(typ: Type, default_instance: _DefaultInstance) -> bool: def field_list_from_callable( f: Union[Callable, Type], default_instance: _DefaultInstance, - root_field: bool = False, ) -> List[FieldDefinition]: """Generate a list of generic 'field' objects corresponding to the inputs of some - annotated callable. - - If `root_field` is set to True, we treat `int`, `torch.device`, etc as nested - fields. This is to make sure that these types can be passed directly into - dcargs.cli(); the logic can likely be refactored.""" - out = _try_field_list_from_callable(f, default_instance, root_field) + annotated callable.""" + out = _try_field_list_from_callable(f, default_instance) if isinstance(out, UnsupportedNestedTypeMessage): raise _instantiators.UnsupportedTypeAnnotationError(out.message) @@ -149,7 +144,6 @@ def field_list_from_callable( def _try_field_list_from_callable( f: Union[Callable, Type], default_instance: _DefaultInstance, - root_field: bool, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Unwrap generics. f, type_from_typevar = _resolver.resolve_generic_types(f) @@ -212,21 +206,18 @@ def _try_field_list_from_callable( # Found fields! if not isinstance(container_fields, UnsupportedNestedTypeMessage): return container_fields - # Got an error -> give up if not the root field. - elif not root_field: + # Got an error, + else: assert isinstance(container_fields, UnsupportedNestedTypeMessage) return container_fields # General cases. - if not root_field and ( - (cls is not None and cls in _known_parsable_types) - or _resolver.unwrap_origin(f) in _known_parsable_types - ): + if (cls is not None and cls in _known_parsable_types) or _resolver.unwrap_origin( + f + ) in _known_parsable_types: return UnsupportedNestedTypeMessage(f"{f} should be parsed directly!") else: - return _try_field_list_from_general_callable( - f, cls, default_instance, root_field=root_field - ) + return _try_field_list_from_general_callable(f, cls, default_instance) def _try_field_list_from_typeddict( @@ -244,7 +235,7 @@ def _try_field_list_from_typeddict( elif getattr(cls, "__total__") is False: default = EXCLUDE_FROM_CALL if is_nested_type(typ, MISSING_NONPROP): - return UnsupportedNestedTypeMessage( + raise _instantiators.UnsupportedTypeAnnotationError( "`total=False` not supported for nested structures." ) else: @@ -307,7 +298,9 @@ def _try_field_list_from_dataclass( typ=dc_field.type, default=default, helptext=_docstrings.get_field_docstring(cls, dc_field.name), - positional=False, + # Only mark positional if using a dummy field, for taking single types + # directly as input. + positional=dc_field.name == _strings.dummy_field_name, ) ) return field_list @@ -445,7 +438,6 @@ def _try_field_list_from_general_callable( f: Union[Callable, Type], cls: Optional[Type], default_instance: _DefaultInstance, - root_field: bool, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Handle general callables. if default_instance not in MISSING_SINGLETONS: @@ -468,44 +460,6 @@ def _try_field_list_from_general_callable( if not isinstance(out, UnsupportedNestedTypeMessage): return out - # Try to support passing things like int, str, Dict[K,V], torch.device - # directly into dcargs.cli(). These aren't "type-annotated callables" but - # this a nice-to-have. - if root_field: - param_count = 0 - has_kw_only = False - has_var_positional = False - for param in params: - if ( - param.kind - in ( - inspect.Parameter.POSITIONAL_ONLY, - inspect.Parameter.POSITIONAL_OR_KEYWORD, - ) - and param.default is inspect.Parameter.empty - ): - param_count += 1 - elif param.kind is inspect.Parameter.KEYWORD_ONLY: - has_kw_only = True - elif param.kind is inspect.Parameter.VAR_POSITIONAL: - has_var_positional = True - - if not has_kw_only and ( - param_count == 1 or (param_count == 0 and has_var_positional) - ): - # Things look ok! - if cls is not None: - f = cls # type: ignore - return [ - FieldDefinition( - name=_resolver.unwrap_origin(f).__name__, - typ=cast(Type, f), - default=MISSING_NONPROP, - helptext=None, - positional=True, - ) - ] - # Return error message. assert isinstance(out, UnsupportedNestedTypeMessage) return out @@ -539,11 +493,16 @@ def _field_list_from_params( helptext = _docstrings.get_field_docstring(cls, param.name) if param.name not in hints: - return UnsupportedNestedTypeMessage( + out = UnsupportedNestedTypeMessage( f"Expected fully type-annotated callable, but {f} with arguments" f" {tuple(map(lambda p: p.name, params))} has no annotation for" f" '{param.name}'." ) + if param.kind is param.KEYWORD_ONLY: + # If keyword only: this can't possibly be an instantiator function + # either, so we escalate to an error. + raise _instantiators.UnsupportedTypeAnnotationError(out.message) + return out field_list.append( FieldDefinition( diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index a2ba461cc..58b3919e2 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -73,7 +73,7 @@ def from_callable( subparsers_from_name = {} field_list = _fields.field_list_from_callable( - f=f, default_instance=default_instance, root_field=True + f=f, default_instance=default_instance ) for field in field_list: field = dataclasses.replace( @@ -365,7 +365,7 @@ def apply( dest=_strings.make_field_name( [ parent_parser.prefix, - _strings.SUBPARSER_DEST_FMT.format(name=self.name), + _strings.make_subparser_dest(name=self.name), ] ), description=self.description, diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 846ef8071..e9a13a573 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -3,10 +3,16 @@ import functools import re import textwrap -from typing import List, Sequence, Type, Union +from typing import Iterable, List, Sequence, Type, Union from . import _resolver +dummy_field_name = "__dcargs_dummy_field_name__" + + +def _strip_dummy_field_names(parts: Iterable[str]) -> Iterable[str]: + return filter(lambda name: len(name) > 0 and name != dummy_field_name, parts) + def make_field_name(parts: Sequence[str]) -> str: """Join parts of a field name together. Used for nesting. @@ -15,7 +21,7 @@ def make_field_name(parts: Sequence[str]) -> str: ('parents', '1', 'child') => 'parents:1.child' """ out: List[str] = [] - for i, p in enumerate([p for p in parts if len(p) > 0]): + for i, p in enumerate(_strip_dummy_field_names(parts)): if i > 0: out.append(".") @@ -24,7 +30,8 @@ def make_field_name(parts: Sequence[str]) -> str: return "".join(out) -SUBPARSER_DEST_FMT: str = "{name} (positional)" +def make_subparser_dest(name: str) -> str: + return f"{name} (positional)" def dedent(text: str) -> str: @@ -59,7 +66,10 @@ def _subparser_name_from_type(cls: Type) -> str: def subparser_name_from_type(prefix: str, cls: Union[Type, None]) -> str: - return f"{prefix}:{_subparser_name_from_type(cls) if cls is not None else 'None'}" + suffix = _subparser_name_from_type(cls) if cls is not None else "None" + if len(prefix) == 0: + return suffix + return f"{prefix}:{suffix}" @functools.lru_cache(maxsize=None) diff --git a/examples/09_subparsers.py b/examples/09_subparsers.py index 9970b0ced..aa0ec8838 100644 --- a/examples/09_subparsers.py +++ b/examples/09_subparsers.py @@ -31,9 +31,12 @@ class Commit: all: bool = False -def main(cmd: Union[Checkout, Commit, None]) -> None: +def main(cmd: Union[Checkout, Commit]) -> None: print(cmd) if __name__ == "__main__": + # Note that we can also pass `Union[Checkout, Command]` directly into + # `dcargs.cli()`; this is understood by dcargs and pyright, but unfortunately not by + # mypy. dcargs.cli(main) diff --git a/tests/test_nested.py b/tests/test_nested.py index 3cbb02c75..00d926d9d 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -163,6 +163,25 @@ class Subparser: dcargs.cli(Subparser, args=["--x", "1", "bc:smtp-server", "--bc.y", "3"]) +def test_subparser_root(): + @dataclasses.dataclass + class HTTPServer: + y: int + + @dataclasses.dataclass + class SMTPServer: + z: int + + @dataclasses.dataclass + class Subparser: + x: int + bc: Union[HTTPServer, SMTPServer] + + assert dcargs.cli( + Union[HTTPServer, SMTPServer], args=["http-server", "--y", "3"] # type: ignore + ) == HTTPServer(y=3) + + def test_subparser_with_default(): @dataclasses.dataclass class DefaultHTTPServer: From e0d99a17e4c8523284ff377dca48ba2c4ff6c98d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 21 Aug 2022 13:36:58 -0700 Subject: [PATCH 096/491] Docs update, version bump --- .../_static/css/compact_table_header.css | 22 +++++ docs/source/alternatives.md | 24 ----- docs/source/conf.py | 1 + docs/source/examples/09_subparsers.rst | 5 +- docs/source/goals_and_alternatives.md | 99 +++++++++++++++++++ docs/source/index.rst | 2 +- pyproject.toml | 2 +- 7 files changed, 128 insertions(+), 27 deletions(-) create mode 100644 docs/source/_static/css/compact_table_header.css delete mode 100644 docs/source/alternatives.md create mode 100644 docs/source/goals_and_alternatives.md diff --git a/docs/source/_static/css/compact_table_header.css b/docs/source/_static/css/compact_table_header.css new file mode 100644 index 000000000..37af2d7c2 --- /dev/null +++ b/docs/source/_static/css/compact_table_header.css @@ -0,0 +1,22 @@ +/* This stylesheet is currently overfit to the comparison table. Should be + * revisited if we add more tables to the documentation. */ +table.docutils thead, +table.docutils tr, +table.docutils th { + background: none; +} +table.docutils thead th { + position: relative; + height: 12em; + text-align: left; + border: 0; +} +table.docutils thead th p { + transform: rotate(300deg); + width: 1em; + overflow: visible; + white-space: nowrap; + position: absolute; + bottom: 0; + z-index: 1; +} diff --git a/docs/source/alternatives.md b/docs/source/alternatives.md deleted file mode 100644 index 600aa62e6..000000000 --- a/docs/source/alternatives.md +++ /dev/null @@ -1,24 +0,0 @@ -# Alternative tools - -The core functionality of `dcargs` — generating argument parsers from type -annotations — can be found as a subset of the features offered by many other -libraries. A summary of some distinguishing features: - -| | Literals | Generics | Docstrings as helptext | Nesting | Subparsers | Containers | -| -------------------------------------------------------------------------------------------------------- | -------------------------------------------------------- | -------- | ---------------------- | ------- | ---------- | ---------- | -| **dcargs** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| [datargs](https://github.com/roee30/datargs) | ✓ | | | | ✓ | ✓ | -| [tap](https://github.com/swansonk14/typed-argument-parser) | ✓ | | ✓ | | ✓ | ✓ | -| [simple-parsing](https://github.com/lebrice/SimpleParsing) | [soon](https://github.com/lebrice/SimpleParsing/pull/86) | | ✓ | ✓ | ✓ | ✓ | -| [argparse-dataclass](https://pypi.org/project/argparse-dataclass/) | | | | | | | -| [argparse-dataclasses](https://pypi.org/project/argparse-dataclasses/) | | | | | | | -| [dataclass-cli](https://github.com/malte-soe/dataclass-cli) | | | | | | | -| [clout](https://pypi.org/project/clout/) | | | | ✓ | | | -| [hf_argparser](https://github.com/huggingface/transformers/blob/master/src/transformers/hf_argparser.py) | | | | | | ✓ | -| [pyrallis](https://github.com/eladrich/pyrallis/) | | | ✓ | ✓ | | ✓ | - -Note that most of these other libraries are generally aimed specifically at -_dataclasses_ rather than general typed callables, but offer other features that -you might find useful, such as registration for custom types (`pyrallis`), -different approaches for serialization and config files (`tap`, `pyrallis`), -simultaneous parsing of multiple dataclasses (`simple-parsing`), etc. diff --git a/docs/source/conf.py b/docs/source/conf.py index 3aa5e3222..0d1df0fb4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -390,3 +390,4 @@ def docstring(app, what, name, obj, options, lines): def setup(app): app.connect("autodoc-process-docstring", docstring) + app.add_stylesheet("css/compact_table_header.css") diff --git a/docs/source/examples/09_subparsers.rst b/docs/source/examples/09_subparsers.rst index 6299c1d91..f41c03a43 100644 --- a/docs/source/examples/09_subparsers.rst +++ b/docs/source/examples/09_subparsers.rst @@ -35,11 +35,14 @@ Unions over nested types (classes or dataclasses) are populated using subparsers all: bool = False - def main(cmd: Union[Checkout, Commit, None]) -> None: + def main(cmd: Union[Checkout, Commit]) -> None: print(cmd) if __name__ == "__main__": + # Note that we can also pass `Union[Checkout, Command]` directly into + # `dcargs.cli()`; this is understood by dcargs and pyright, but unfortunately not by + # mypy. dcargs.cli(main) ------------ diff --git a/docs/source/goals_and_alternatives.md b/docs/source/goals_and_alternatives.md new file mode 100644 index 000000000..1f9e3f032 --- /dev/null +++ b/docs/source/goals_and_alternatives.md @@ -0,0 +1,99 @@ +# Goals and alternatives + +## Design goals + +The core functionality of `dcargs` — generating argument parsers from type +annotations — overlaps significantly with features offered by other libraries. +Usage distinctions are the result of two API goals: + +- **One uninvasive function.** Whenever possible, learning to use `dcargs` + should reduce to learning to write (type-annotated) Python. For example, types + are specified using standard annotations, helptext using docstrings, choices + using the standard `typing.Literal` type, subparsers with `typing.Union` of + nested types, and positional arguments with `/`. + - In contrast, similar libraries have more expansive APIs (sometimes spanning + dozens of specialized class and functions), and often require + library-specific structures, decorators, or metadata formats for configuring + parsing behavior. +- **Strict typing.** Any type that can be annotated and unambiguously parsed + with an `argparse`-style CLI interface should work out-of-the-box; any public + API that isn't statically analyzable shouldn't be implemented. + - In contrast, many similar libraries implement features that depend on + dynamic argparse-style namespaces, or string-based accessors that can't be + statically checked. + +More concretely, we can also compare specific features. A noncomprehensive set: + +| | Dataclasses | Functions | Literals | Generics | Docstrings as helptext | Nested structures | Subparsers | Dictionaries | Lists, tuples | +| -------------------------------------------- | ----------- | --------- | -------------------- | -------- | ---------------------- | ----------------- | ------------------- | ------------ | -------------------- | +| **dcargs** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| [argparse-dataclass][argparse-dataclass] | ✓ | | | | | | | | | +| [argparse-dataclasses][argparse-dataclasses] | ✓ | | | | | | | | | +| [datargs][datargs] | ✓ | | ✓[^datargs_literals] | | | | ✓ | | ✓ | +| [tap][tap] | | | ✓ | | ✓ | | ✓[^tap_subparsers] | | ✓ | +| [simple-parsing][simple-parsing] | ✓ | | ✓[^simp_literals] | | ✓ | ✓ | ✓[^simp_subparsers] | ✓ | ✓ | +| [dataclass-cli][dataclass-cli] | ✓ | | | | | | | | | +| [clout][clout] | ✓ | | | | | ✓ | | | | +| [hf_argparser][hf_argparser] | ✓ | | | | | | | ✓ | ✓ | +| [typer][typer] | | ✓ | | | | | ✓ | | ~[^typer_containers] | +| [pyrallis][pyrallis] | ✓ | | | | ✓ | ✓ | | | ✓ | +| [yahp][yahp] | ✓ | | | | ~[^yahp_docstrings] | ✓ | ✓[^yahp_subparsers] | | ✓ | +| [omegaconf][omegaconf] | ✓ | | | | | ✓ | | ✓ | ✓ | + + + +[datargs]: https://github.com/roee30/datargs +[tap]: https://github.com/swansonk14/typed-argument-parser +[simple-parsing]: https://github.com/lebrice/SimpleParsing +[argparse-dataclass]: https://pypi.org/project/argparse-dataclass/ +[argparse-dataclasses]: https://pypi.org/project/argparse-dataclasses/ +[dataclass-cli]: https://github.com/malte-soe/dataclass-cli +[clout]: https://pypi.org/project/clout/ +[hf_argparser]: https://github.com/huggingface/transformers/blob/master/src/transformers/hf_argparser.py +[pyrallis]: https://github.com/eladrich/pyrallis/ +[typer]: https://typer.tiangolo.com/ +[yahp]: https://github.com/mosaicml/yahp +[omegaconf]: https://omegaconf.readthedocs.io/en/2.1_branch/structured_config.html + +[^tap_subparsers]: Supported but not strongly typed. +[^simp_literals]: Not supported for mixed (eg `Literal[5, "five"]`) or in container (eg `List[Literal[1, 2]]`) types. +[^simp_subparsers]: Supported but not strongly typed. +[^yahp_subparsers]: Supported but not strongly typed. +[^datargs_literals]: Not supported for mixed types (eg `Literal[5, "five"]`). +[^typer_containers]: `typer` limit itself to positional arguments, which can be more readable in certain cases but means that only one variable-length argument (such as `List[int]`) is supported per argument parser. +[^yahp_docstrings]: Via the `hp.auto()` function, which can parse docstrings from external classes. Usage is different from the more direct parsing that `dcargs`, `tap`, and `simple-parsing`/`pyrallis` support. + + + +Note that most of these other libraries are generally aimed specifically at only +one of _dataclasses_ (`datargs`, `simple-parsing`, `argparse-dataclass`, +`argparse-dataclasses`, `dataclass-cli`, `clout`, `hf_argparser`, `pyrallis`, +`yahp`), _custom structures_ (`tap`), or _functions_ (`typer`) rather than +general typed callables, but offer other features that you might find useful, +such as registration for custom types (`pyrallis`), built-in approaches for +serialization and config files (`tap`, `pyrallis`, `yahp`) and, opportunities +for integration with fields generated using standard argparse definitions +(`simple-parsing`). + +## Note on configuration systems + +Our API is designed to be used to configure general applications and +computational experiments written in Python, but intentionally tries to avoid +building a full configuration framework (eg `hydra`). These frameworks can +typically be broken into three components with varying levels of integration, +which include syntax and logic for **(1)** defining configurable fields, **(2)** +saving and choosing between a set of base configurations, and **(3)** overriding +configurable values at the command-line. + +We take advantage of modern Python features for **(1)** and focus completely on +**(3)** in a way that's agnostic to a project's preferred approach for **(2)**. +**(2)** is left as an exercise to the user; it tends to be the most open-ended +and varied in terms of project and personal preferences, but is typically +straightforward to implement given **(1)**. + +In contrast, popular libraries oriented toward configuration often strive to be +more "batteries-included", which is convenient but requires prescribing things +like specific formats, processes, or even directory structures for working with +saved configurations. This requires a lot more moving parts, which can be +limiting if any one of them is insufficient for a particular use case. (or if +you just don't like working with YAML formats) diff --git a/docs/source/index.rst b/docs/source/index.rst index e8eb035ad..afa9ad104 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -105,8 +105,8 @@ The broader goal is also a replacement for tools like :code:`hydra`, :hidden: :glob: + goals_and_alternatives serialization - alternatives .. toctree:: diff --git a/pyproject.toml b/pyproject.toml index 792fc0d04..0a415e6a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.1.11" +version = "0.1.12" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From cd6411928a3e098242dd5d9c8a6ee04426fb146f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 23 Aug 2022 13:24:32 -0700 Subject: [PATCH 097/491] Support inherited generics, docs tweaks --- dcargs/_parsers.py | 9 +----- dcargs/_resolver.py | 19 ++++++++++-- docs/source/goals_and_alternatives.md | 38 ++++++++++++------------ tests/test_errors.py | 30 +------------------ tests/test_generics_and_serialization.py | 26 ---------------- tests/test_nested.py | 25 ++++++++++++++++ 6 files changed, 62 insertions(+), 85 deletions(-) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 58b3919e2..014e1d339 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -86,15 +86,8 @@ def from_callable( ), ) if isinstance(field.typ, TypeVar): - # Found an unbound TypeVar. This could be because inheriting from - # generics is currently not implemented. It's unclear whether this is - # feasible, because generics are lost in the mro: - # https://github.com/python/typing/issues/777 raise _instantiators.UnsupportedTypeAnnotationError( - f"Field {field.name} has an unbound TypeVar: {field.typ}. Note" - " that inheriting from generics is currently not implemented." - " It's unclear whether this is feasible, because generics are" - " lost in the mro: https://github.com/python/typing/issues/777" + f"Field {field.name} has an unbound TypeVar: {field.typ}." ) # (1) Handle Unions over callables; these result in subparsers. diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 56b0a121a..8d7e69adc 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -30,13 +30,26 @@ def resolve_generic_types( class, and a mapping from typevars to concrete types.""" origin_cls = get_origin(cls) + + type_from_typevars = {} if origin_cls is not None and hasattr(origin_cls, "__parameters__"): typevars = origin_cls.__parameters__ typevar_values = get_args(cls) assert len(typevars) == len(typevar_values) - return origin_cls, dict(zip(typevars, typevar_values)) - else: - return cls, {} + cls = origin_cls + type_from_typevars.update(dict(zip(typevars, typevar_values))) + + if hasattr(cls, "__orig_bases__"): + bases = getattr(cls, "__orig_bases__") + for base in bases: + origin_base = unwrap_origin(base) + if origin_base is base or not hasattr(origin_base, "__parameters__"): + continue + typevars = origin_base.__parameters__ + typevar_values = get_args(base) + type_from_typevars.update(dict(zip(typevars, typevar_values))) + + return cls, type_from_typevars def resolved_fields(cls: Type) -> List[dataclasses.Field]: diff --git a/docs/source/goals_and_alternatives.md b/docs/source/goals_and_alternatives.md index 1f9e3f032..06bbd55ce 100644 --- a/docs/source/goals_and_alternatives.md +++ b/docs/source/goals_and_alternatives.md @@ -60,17 +60,17 @@ More concretely, we can also compare specific features. A noncomprehensive set: [^simp_subparsers]: Supported but not strongly typed. [^yahp_subparsers]: Supported but not strongly typed. [^datargs_literals]: Not supported for mixed types (eg `Literal[5, "five"]`). -[^typer_containers]: `typer` limit itself to positional arguments, which can be more readable in certain cases but means that only one variable-length argument (such as `List[int]`) is supported per argument parser. +[^typer_containers]: `typer` uses positional arguments for all required fields, which means that only one variable-length argument (such as `List[int]`) without a default is supported per argument parser. [^yahp_docstrings]: Via the `hp.auto()` function, which can parse docstrings from external classes. Usage is different from the more direct parsing that `dcargs`, `tap`, and `simple-parsing`/`pyrallis` support. Note that most of these other libraries are generally aimed specifically at only -one of _dataclasses_ (`datargs`, `simple-parsing`, `argparse-dataclass`, +one of dataclasses (`datargs`, `simple-parsing`, `argparse-dataclass`, `argparse-dataclasses`, `dataclass-cli`, `clout`, `hf_argparser`, `pyrallis`, -`yahp`), _custom structures_ (`tap`), or _functions_ (`typer`) rather than -general typed callables, but offer other features that you might find useful, -such as registration for custom types (`pyrallis`), built-in approaches for +`yahp`), custom structures (`tap`), or functions (`typer`) rather than general +typed callables, but offer other features that you might find useful, such as +registration for custom types (`pyrallis`), built-in approaches for serialization and config files (`tap`, `pyrallis`, `yahp`) and, opportunities for integration with fields generated using standard argparse definitions (`simple-parsing`). @@ -79,21 +79,21 @@ for integration with fields generated using standard argparse definitions Our API is designed to be used to configure general applications and computational experiments written in Python, but intentionally tries to avoid -building a full configuration framework (eg `hydra`). These frameworks can -typically be broken into three components with varying levels of integration, -which include syntax and logic for **(1)** defining configurable fields, **(2)** -saving and choosing between a set of base configurations, and **(3)** overriding -configurable values at the command-line. +building a full configuration framework (for example, `hydra`). These frameworks +can typically be broken into three components with varying levels of +integration, which include syntax and logic for **(1)** defining configurable +fields, **(2)** saving and choosing between a set of base configurations, and +**(3)** overriding configurable values at the command-line. -We take advantage of modern Python features for **(1)** and focus completely on -**(3)** in a way that's agnostic to a project's preferred approach for **(2)**. -**(2)** is left as an exercise to the user; it tends to be the most open-ended -and varied in terms of project and personal preferences, but is typically -straightforward to implement given **(1)**. +`dcargs` is meant to take advantage of modern Python features for **(1)** and +focus completely on **(3)** in a way that's agnostic to a project's preferred +approach for **(2)**. **(2)** is left as an exercise to the user; it tends to be +the most open-ended and varied in terms of project and personal preferences, but +is typically straightforward to implement given **(1)**. In contrast, popular libraries oriented toward configuration often strive to be more "batteries-included", which is convenient but requires prescribing things -like specific formats, processes, or even directory structures for working with -saved configurations. This requires a lot more moving parts, which can be -limiting if any one of them is insufficient for a particular use case. (or if -you just don't like working with YAML formats) +like specific formats, processes, or directory structures for working with saved +configurations. This requires more moving parts, which can be limiting if any +one of them is insufficient for a particular use case. (or if you just don't +like working with YAML formats) diff --git a/tests/test_errors.py b/tests/test_errors.py index 35f64f623..4d71aa2c2 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Generic, List, Tuple, TypeVar, Union +from typing import List, Tuple, Union import pytest @@ -83,34 +83,6 @@ def main(arg: List[TwoStringArg]) -> None: dcargs.cli(main, args=[]) -def test_generic_inherited(): - """Inheriting from generics is currently not implemented. It's unclear whether this - is feasible, because generics are lost in the mro: - https://github.com/python/typing/issues/777""" - - class UnrelatedParentClass: - pass - - T = TypeVar("T") - - @dataclasses.dataclass - class ActualParentClass(Generic[T]): - x: T # Documentation 1 - - # Documentation 2 - y: T - - z: T = 3 # type: ignore - """Documentation 3""" - - @dataclasses.dataclass - class ChildClass(UnrelatedParentClass, ActualParentClass[int]): - pass - - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(ChildClass, args=["--x", "1", "--y", "2", "--z", "3"]) - - def test_missing_annotation_1(): def main(a, b) -> None: pass diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index fa763fe43..0ac23f7cf 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -309,32 +309,6 @@ class Subparser(Generic[T1, T2]): ) -def test_generic_inherited(): - class UnrelatedParentClass: - pass - - T = TypeVar("T") - - @dataclasses.dataclass - class ActualParentClass(Generic[T]): - x: T # Documentation 1 - - # Documentation 2 - y: T - - z: T = 3 # type: ignore - """Documentation 3""" - - @dataclasses.dataclass - class ChildClass(UnrelatedParentClass, ActualParentClass[int]): - pass - - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - assert dcargs.cli( - ChildClass, args=["--x", "1", "--y", "2", "--z", "3"] - ) == ChildClass(1, 2, 3) - - def test_serialize_missing(): @dataclasses.dataclass class TupleGenericVariableMissing(Generic[ScalarType]): diff --git a/tests/test_nested.py b/tests/test_nested.py index 00d926d9d..08d8265b6 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -672,3 +672,28 @@ def main_with_default(x: Union[A[int], A[float]] = A(5)) -> Any: with pytest.raises(dcargs.UnsupportedTypeAnnotationError): dcargs.cli(main_with_default, args=[]) + + +def test_generic_inherited(): + class UnrelatedParentClass: + pass + + T = TypeVar("T") + + @dataclasses.dataclass + class ActualParentClass(Generic[T]): + x: T # Documentation 1 + + # Documentation 2 + y: T + + z: T = 3 # type: ignore + """Documentation 3""" + + @dataclasses.dataclass + class ChildClass(UnrelatedParentClass, ActualParentClass[int]): + pass + + assert dcargs.cli( + ChildClass, args=["--x", "1", "--y", "2", "--z", "3"] + ) == ChildClass(x=1, y=2, z=3) From f46dd96b9de5d961e84d9cca2b6aff3f8332d869 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 23 Aug 2022 15:46:39 -0700 Subject: [PATCH 098/491] Version fix --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0a415e6a1..1853b867e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.1.12" +version = "0.2.0" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From 93f5baa51e91590e16f34600fec35843247904f3 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 23 Aug 2022 16:02:12 -0700 Subject: [PATCH 099/491] Update publish.yml --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 62aa21212..2291c4d68 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -27,4 +27,4 @@ jobs: PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - poetry publish --username $PYPI_USERNAME --password $PYPI_PASSWORD + poetry publish --build --username $PYPI_USERNAME --password $PYPI_PASSWORD From 819cc741f9c83bce21e6563a59edfa59b029b649 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 23 Aug 2022 16:33:35 -0700 Subject: [PATCH 100/491] Add Poetry links --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1853b867e..570b91984 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,13 @@ [tool.poetry] name = "dcargs" -version = "0.2.0" +version = "0.2.1" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] +readme = "README.md" +repository = "https://github.com/brentyi/dcargs" +homepage = "https://github.com/brentyi/dcargs" +documentation = "https://brentyi.github.io/dcargs/" [tool.poetry.dependencies] python = "^3.7" From 29255761f73cca6189400404ac6d3473b2c24a1f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 24 Aug 2022 12:59:28 -0700 Subject: [PATCH 101/491] Subparser fixes: nesting, Optional[] --- dcargs/_calling.py | 5 +++-- dcargs/_parsers.py | 19 ++++++++----------- pyproject.toml | 2 +- tests/test_nested.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index d81f8e0f5..49e967413 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -115,15 +115,16 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: else: # Unions over dataclasses (subparsers). This is the only other option. assert len(parser_definition.subparsers_from_name) > 0 - assert field.name in parser_definition.subparsers_from_name + assert prefixed_field_name in parser_definition.subparsers_from_name - subparser_def = parser_definition.subparsers_from_name[field.name] + subparser_def = parser_definition.subparsers_from_name[prefixed_field_name] subparser_dest = _strings.make_subparser_dest(name=prefixed_field_name) consumed_keywords.add(subparser_dest) if subparser_dest in value_from_prefixed_field_name: subparser_name = get_value_from_arg(subparser_dest) else: + assert subparser_def.default_instance not in _fields.MISSING_SINGLETONS default_instance = subparser_def.default_instance # assert default_instance is not None subparser_name = None diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 014e1d339..5f430595f 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -103,15 +103,13 @@ def from_callable( not avoid_subparsers # Required subparsers => must create a subparser. or subparsers_attempt.required - # If the output can be None, the only way to specify whether this is - # or isn't None is with a subparser. - or subparsers_attempt.can_be_none ): - subparsers_from_name[subparsers_attempt.name] = subparsers_attempt + subparsers_from_name[ + _strings.make_field_name([prefix, subparsers_attempt.name]) + ] = subparsers_attempt continue else: field = dataclasses.replace(field, typ=type(field.default)) - assert _fields.is_nested_type(field.typ, field.default) # (2) Handle nested callables. if _fields.is_nested_type(field.typ, field.default): @@ -129,6 +127,10 @@ def from_callable( ) args.extend(nested_parser.args) + # Include nested subparsers. + subparsers_from_name.update(nested_parser.subparsers_from_name) + + # Include nested strings. for k, v in nested_parser.helptext_from_nested_class_field_name.items(): helptext_from_nested_class_field_name[ _strings.make_field_name([field.name, k]) @@ -355,12 +357,7 @@ def apply( for p in prev_subparser_tree_nodes: # Add subparsers to every node in previous level of the tree. argparse_subparsers = p.add_subparsers( - dest=_strings.make_field_name( - [ - parent_parser.prefix, - _strings.make_subparser_dest(name=self.name), - ] - ), + dest=_strings.make_subparser_dest(self.prefix), description=self.description, required=self.required, title=termcolor.colored(title, attrs=["bold"]), diff --git a/pyproject.toml b/pyproject.toml index 570b91984..dbb57d57e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.2.1" +version = "0.2.2" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] diff --git a/tests/test_nested.py b/tests/test_nested.py index 08d8265b6..6663de44d 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -697,3 +697,34 @@ class ChildClass(UnrelatedParentClass, ActualParentClass[int]): assert dcargs.cli( ChildClass, args=["--x", "1", "--y", "2", "--z", "3"] ) == ChildClass(x=1, y=2, z=3) + + +def test_subparser_in_nested(): + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + + @dataclasses.dataclass + class Nested2: + subcommand: Union[A, B] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2 + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:a --nested1.nested2.subcommand.a 3".split(" "), + ) == Parent(Nested1(Nested2(A(3)))) + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:b --nested1.nested2.subcommand.b 7".split(" "), + ) == Parent(Nested1(Nested2(B(7)))) From e517015331fbd0bee2d0782b140bb8f6ef8b9ed3 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 24 Aug 2022 19:31:34 -0700 Subject: [PATCH 102/491] Tests, more intuitive behavior for type narrowing --- dcargs/_fields.py | 2 +- dcargs/_resolver.py | 2 +- pyproject.toml | 2 +- tests/test_collections.py | 41 ++++++++++++++++++++++- tests/test_helptext.py | 1 - tests/test_is_nested_type.py | 52 ++++++++++++++++++++++++++++++ tests/test_nested_in_containers.py | 51 +++++++++++++++++++++++++++++ 7 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 tests/test_is_nested_type.py diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 896e8f4fd..4032f2f6a 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -198,7 +198,7 @@ def _try_field_list_from_callable( container_fields = _try_field_list_from_sequence( contained_type, default_instance ) - elif f_origin is dict: + elif f_origin is dict or cls is dict: container_fields = _try_field_list_from_dict(f, default_instance) # Check if one of the container types matched. diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 8d7e69adc..f6598cf92 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -103,7 +103,7 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: try: potential_subclass = type(default_instance) superclass = typ - if issubclass(potential_subclass, superclass): # type: ignore + if superclass is Any or issubclass(potential_subclass, superclass): # type: ignore return cast(TypeT, potential_subclass) except TypeError: pass diff --git a/pyproject.toml b/pyproject.toml index dbb57d57e..4626b66fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.2.2" +version = "0.2.3" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] diff --git a/tests/test_collections.py b/tests/test_collections.py index c60c47d46..b6970cbfc 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -1,7 +1,18 @@ import collections import dataclasses import enum -from typing import Any, Deque, FrozenSet, List, Optional, Sequence, Set, Tuple, Union +from typing import ( + Any, + Deque, + Dict, + FrozenSet, + List, + Optional, + Sequence, + Set, + Tuple, + Union, +) import pytest from typing_extensions import Literal @@ -382,3 +393,31 @@ def main( ] with pytest.raises(SystemExit): dcargs.cli(main, args=["--help"]) + + +def test_dict_no_annotation(): + def main(x: Dict[str, Any] = {"int": 5, "str": "5"}): + return x + + assert dcargs.cli(main, args=[]) == {"int": 5, "str": "5"} + assert dcargs.cli(main, args="--x.int 3 --x.str 7".split(" ")) == { + "int": 3, + "str": "7", + } + + +def test_double_dict_no_annotation(): + def main( + x: Dict[str, Any] = { + "wow": {"int": 5, "str": "5"}, + } + ): + return x + + assert dcargs.cli(main, args=[]) == {"wow": {"int": 5, "str": "5"}} + assert dcargs.cli(main, args="--x.wow.int 3 --x.wow.str 7".split(" ")) == { + "wow": { + "int": 3, + "str": "7", + } + } diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 929e2c9a1..2160c40b9 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -515,7 +515,6 @@ class Something( def test_unparsable(): - @dataclasses.dataclass class Struct: a: int = 5 b: str = "7" diff --git a/tests/test_is_nested_type.py b/tests/test_is_nested_type.py new file mode 100644 index 000000000..6960d9a9f --- /dev/null +++ b/tests/test_is_nested_type.py @@ -0,0 +1,52 @@ +import dataclasses +import pathlib +from typing import Any, Dict, List, Tuple + +from dcargs._fields import MISSING_NONPROP, is_nested_type + + +def test_is_nested_type_simple(): + assert not is_nested_type(int, MISSING_NONPROP) + assert not is_nested_type(bool, MISSING_NONPROP) + assert not is_nested_type(str, MISSING_NONPROP) + assert not is_nested_type(pathlib.Path, MISSING_NONPROP) + + +def test_is_nested_type_containers(): + assert not is_nested_type(List[int], MISSING_NONPROP) + assert not is_nested_type(List[bool], MISSING_NONPROP) + assert not is_nested_type(List[str], MISSING_NONPROP) + assert not is_nested_type(List[pathlib.Path], MISSING_NONPROP) + + +@dataclasses.dataclass +class Color: + r: int + g: int + b: int + + +def test_is_nested_type_actually_nested(): + assert is_nested_type(Color, Color(255, 0, 0)) + + +def test_is_nested_type_actually_nested_narrowing(): + assert is_nested_type(Any, Color(255, 0, 0)) + assert is_nested_type(object, Color(255, 0, 0)) + assert not is_nested_type(int, Color(255, 0, 0)) + + +def test_is_nested_type_actually_nested_in_container(): + assert is_nested_type(Tuple[Color, Color], MISSING_NONPROP) + assert is_nested_type(Tuple[object, ...], (Color(255, 0, 0),)) + assert is_nested_type(Tuple[Any, ...], (Color(255, 0, 0),)) + assert is_nested_type(tuple, (Color(255, 0, 0),)) + assert not is_nested_type(tuple, (1, 2, 3)) + assert is_nested_type(tuple, (1, Color(255, 0, 0), 3)) + assert is_nested_type(List[Any], [Color(255, 0, 0)]) + + +def test_nested_dict(): + assert is_nested_type(Dict[str, int], {"x": 5}) + assert is_nested_type(dict, {"x": 5}) + assert is_nested_type(Any, {"x": 5}) diff --git a/tests/test_nested_in_containers.py b/tests/test_nested_in_containers.py index c914ef35d..56c3dfded 100644 --- a/tests/test_nested_in_containers.py +++ b/tests/test_nested_in_containers.py @@ -125,6 +125,14 @@ def main(x: List[object] = [Color(255, 0, 0)]) -> Any: assert dcargs.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] +def test_list_any(): + def main(x: List[Any] = [Color(255, 0, 0)]) -> Any: + return x + + assert dcargs.cli(main, args=[]) == [Color(255, 0, 0)] + assert dcargs.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] + + def test_tuple_in_list(): def main(x: List[Tuple[Color]] = [(Color(255, 0, 0),)]) -> Any: return x @@ -320,3 +328,46 @@ def main( main, args="--x.int.g 0".split(" "), ) == {"float": GenericColor(0.5, 0.2, 0.3), "int": GenericColor(25, 0, 3)} + + +def test_generic_in_double_nested_dict_with_default(): + ScalarType = TypeVar("ScalarType", int, float) + + @dataclasses.dataclass + class GenericColor(Generic[ScalarType]): + r: ScalarType + g: ScalarType + b: ScalarType + + def main( + x: Dict[str, Dict[str, GenericColor]] = { + "hello": { + "float": GenericColor(0.5, 0.2, 0.3), + "int": GenericColor[int](25, 2, 3), + } + } + ) -> Any: + return x + + assert dcargs.cli(main, args="--x.hello.float.g 0.1".split(" "),)["hello"][ + "float" + ] == GenericColor(0.5, 0.1, 0.3) + assert dcargs.cli(main, args="--x.hello.int.g 0".split(" "),) == { + "hello": {"float": GenericColor(0.5, 0.2, 0.3), "int": GenericColor(25, 0, 3)} + } + + +def test_double_nested_dict_with_inferred_type(): + def main( + x: Dict[str, Any] = { + "hello": { + "a": Color(5, 2, 3), + "b": Color(25, 2, 3), + } + } + ) -> Any: + return x + + assert dcargs.cli(main, args="--x.hello.a.g 1".split(" "),)["hello"][ + "a" + ] == Color(5, 1, 3) From c9b0203b455809a3408136f25143a748ae0069d0 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 24 Aug 2022 22:10:29 -0700 Subject: [PATCH 103/491] Tests and fixes for general mappings, frozendict --- dcargs/_calling.py | 1 + dcargs/_docstrings.py | 10 +- dcargs/_fields.py | 19 +++- docs/source/examples/11_dictionaries.rst | 16 ++- examples/11_dictionaries.py | 16 ++- poetry.lock | 122 +++++++++++++++-------- pyproject.toml | 3 +- tests/test_dict_namedtuple.py | 10 +- tests/test_nested.py | 19 +++- 9 files changed, 151 insertions(+), 65 deletions(-) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 49e967413..958b587ff 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -180,6 +180,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: unwrapped_f = _resolver.unwrap_origin(f) unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore + unwrapped_f = _resolver.narrow_type(unwrapped_f, default_instance) if unwrapped_f in (tuple, list, set): if len(args) == 0: # When tuples are used as nested structures (eg Tuple[SomeDataclass]), we diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 969f3bf92..dcca98ade 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -1,9 +1,11 @@ """Helpers for parsing docstrings. Used for helptext generation.""" +import collections.abc import dataclasses import functools import inspect import io +import itertools import tokenize from typing import Callable, Dict, Generic, Hashable, List, Optional, Type @@ -236,10 +238,10 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: return None -_builtins = set( +_callable_description_blocklist = set( filter( lambda x: isinstance(x, Hashable), # type: ignore - __builtins__.values(), # type: ignore + itertools.chain(__builtins__.values(), vars(collections.abc).values()), # type: ignore ) ) @@ -250,9 +252,9 @@ def get_callable_description(f: Callable) -> str: Note that the `dataclasses.dataclass` will automatically populate __doc__ based on the fields of the class if a docstring is not specified; this helper will ignore these docstrings.""" - f, _unused = _resolver.resolve_generic_types(f) - if _resolver.unwrap_origin(f) in _builtins: + f, _unused = _resolver.resolve_generic_types(f) + if _resolver.unwrap_origin(f) in _callable_description_blocklist: return "" # Note inspect.getdoc() causes some corner cases with TypedDicts. diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 4032f2f6a..9ad5574aa 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -4,13 +4,25 @@ from __future__ import annotations import collections +import collections.abc import dataclasses import enum import inspect import itertools import typing import warnings -from typing import Any, Callable, Hashable, Iterable, List, Optional, Type, Union, cast +from typing import ( + Any, + Callable, + Hashable, + Iterable, + List, + Mapping, + Optional, + Type, + Union, + cast, +) import docstring_parser import typing_extensions @@ -198,7 +210,10 @@ def _try_field_list_from_callable( container_fields = _try_field_list_from_sequence( contained_type, default_instance ) - elif f_origin is dict or cls is dict: + elif f_origin in (collections.abc.Mapping, dict) or cls in ( + collections.abc.Mapping, + dict, + ): container_fields = _try_field_list_from_dict(f, default_instance) # Check if one of the container types matched. diff --git a/docs/source/examples/11_dictionaries.rst b/docs/source/examples/11_dictionaries.rst index a2b681f8f..cd3400151 100644 --- a/docs/source/examples/11_dictionaries.rst +++ b/docs/source/examples/11_dictionaries.rst @@ -13,7 +13,9 @@ or a ``TypedDict`` subclass. .. code-block:: python :linenos: - from typing import Dict, Tuple, TypedDict + from typing import Dict, Mapping, Tuple, TypedDict + + from frozendict import frozendict # type: ignore import dcargs @@ -34,11 +36,19 @@ or a ``TypedDict`` subclass. "beta1": 0.9, "beta2": 0.999, }, + frozen_dict: Mapping[str, float] = frozendict( + { + "num_epochs": 20, + "batch_size": 64, + } + ), ) -> None: - assert isinstance(standard_dict, dict) assert isinstance(typed_dict, dict) - print("Standard dict:", standard_dict) + assert isinstance(standard_dict, dict) + assert isinstance(frozen_dict, frozendict) print("Typed dict:", typed_dict) + print("Standard dict:", standard_dict) + print("Frozen dict:", frozen_dict) if __name__ == "__main__": diff --git a/examples/11_dictionaries.py b/examples/11_dictionaries.py index 0f91a0f31..617622c09 100644 --- a/examples/11_dictionaries.py +++ b/examples/11_dictionaries.py @@ -7,7 +7,9 @@ `python ./11_dictionaries.py --typed-dict.betas 0.9 0.999` """ -from typing import Dict, Tuple, TypedDict +from typing import Dict, Mapping, Tuple, TypedDict + +from frozendict import frozendict # type: ignore import dcargs @@ -28,11 +30,19 @@ def main( "beta1": 0.9, "beta2": 0.999, }, + frozen_dict: Mapping[str, float] = frozendict( + { + "num_epochs": 20, + "batch_size": 64, + } + ), ) -> None: - assert isinstance(standard_dict, dict) assert isinstance(typed_dict, dict) - print("Standard dict:", standard_dict) + assert isinstance(standard_dict, dict) + assert isinstance(frozen_dict, frozendict) print("Typed dict:", typed_dict) + print("Standard dict:", standard_dict) + print("Frozen dict:", frozen_dict) if __name__ == "__main__": diff --git a/poetry.lock b/poetry.lock index f55188f57..2ee845526 100644 --- a/poetry.lock +++ b/poetry.lock @@ -46,7 +46,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "coverage" -version = "6.4.3" +version = "6.4.4" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -66,6 +66,14 @@ category = "main" optional = false python-versions = ">=3.6,<4.0" +[[package]] +name = "frozendict" +version = "2.3.4" +description = "A simple immutable dictionary" +category = "main" +optional = false +python-versions = ">=3.6" + [[package]] name = "importlib-metadata" version = "4.12.0" @@ -310,7 +318,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "277f1e54b73327250e08fd0cf5b127efab41d324617bf7368ea8249d1c2ffa02" +content-hash = "897e78d65e9a00cd292034a23eb35922e934052dbba5b927795e39cf21b3f449" [metadata.files] antlr4-python3-runtime = [ @@ -332,52 +340,80 @@ colorama = [ {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] coverage = [ - {file = "coverage-6.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f50d3a822947572496ea922ee7825becd8e3ae6fbd2400cd8236b7d64b17f285"}, - {file = "coverage-6.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d5191d53afbe5b6059895fa7f58223d3751c42b8101fb3ce767e1a0b1a1d8f87"}, - {file = "coverage-6.4.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04010af3c06ce2bfeb3b1e4e05d136f88d88c25f76cd4faff5d1fd84d11581ea"}, - {file = "coverage-6.4.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6630d8d943644ea62132789940ca97d05fac83f73186eaf0930ffa715fbdab6b"}, - {file = "coverage-6.4.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05de0762c1caed4a162b3e305f36cf20a548ff4da0be6766ad5c870704be3660"}, - {file = "coverage-6.4.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0e3a41aad5919613483aad9ebd53336905cab1bd6788afd3995c2a972d89d795"}, - {file = "coverage-6.4.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a2738ba1ee544d6f294278cfb6de2dc1f9a737a780469b5366e662a218f806c3"}, - {file = "coverage-6.4.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a0d2df4227f645a879010461df2cea6b7e3fb5a97d7eafa210f7fb60345af9e8"}, - {file = "coverage-6.4.3-cp310-cp310-win32.whl", hash = "sha256:73a10939dc345460ca0655356a470dd3de9759919186a82383c87b6eb315faf2"}, - {file = "coverage-6.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:53c8edd3b83a4ddba3d8c506f1359401e7770b30f2188f15c17a338adf5a14db"}, - {file = "coverage-6.4.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f1eda5cae434282712e40b42aaf590b773382afc3642786ac3ed39053973f61f"}, - {file = "coverage-6.4.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59fc88bc13e30f25167e807b8cad3c41b7218ef4473a20c86fd98a7968733083"}, - {file = "coverage-6.4.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75314b00825d70e1e34b07396e23f47ed1d4feedc0122748f9f6bd31a544840"}, - {file = "coverage-6.4.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52f8b9fcf3c5e427d51bbab1fb92b575a9a9235d516f175b24712bcd4b5be917"}, - {file = "coverage-6.4.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5a559aab40c716de80c7212295d0dc96bc1b6c719371c20dd18c5187c3155518"}, - {file = "coverage-6.4.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:306788fd019bb90e9cbb83d3f3c6becad1c048dd432af24f8320cf38ac085684"}, - {file = "coverage-6.4.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:920a734fe3d311ca01883b4a19aa386c97b82b69fbc023458899cff0a0d621b9"}, - {file = "coverage-6.4.3-cp37-cp37m-win32.whl", hash = "sha256:ab9ef0187d6c62b09dec83a84a3b94f71f9690784c84fd762fb3cf2d2b44c914"}, - {file = "coverage-6.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:39ebd8e120cb77a06ee3d5fc26f9732670d1c397d7cd3acf02f6f62693b89b80"}, - {file = "coverage-6.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc698580216050b5f4a34d2cdd2838b429c53314f1c4835fab7338200a8396f2"}, - {file = "coverage-6.4.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:877ee5478fd78e100362aed56db47ccc5f23f6e7bb035a8896855f4c3e49bc9b"}, - {file = "coverage-6.4.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:555a498999c44f5287cc95500486cd0d4f021af9162982cbe504d4cb388f73b5"}, - {file = "coverage-6.4.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eff095a5aac7011fdb51a2c82a8fae9ec5211577f4b764e1e59cfa27ceeb1b59"}, - {file = "coverage-6.4.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5de1e9335e2569974e20df0ce31493d315a830d7987e71a24a2a335a8d8459d3"}, - {file = "coverage-6.4.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7856ea39059d75f822ff0df3a51ea6d76307c897048bdec3aad1377e4e9dca20"}, - {file = "coverage-6.4.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:411fdd9f4203afd93b056c0868c8f9e5e16813e765de962f27e4e5798356a052"}, - {file = "coverage-6.4.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cdf7b83f04a313a21afb1f8730fe4dd09577fefc53bbdfececf78b2006f4268e"}, - {file = "coverage-6.4.3-cp38-cp38-win32.whl", hash = "sha256:ab2b1a89d2bc7647622e9eaf06128a5b5451dccf7c242deaa31420b055716481"}, - {file = "coverage-6.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:0e34247274bde982bbc613894d33f9e36358179db2ed231dd101c48dd298e7b0"}, - {file = "coverage-6.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b104b6b1827d6a22483c469e3983a204bcf9c6bf7544bf90362c4654ebc2edf3"}, - {file = "coverage-6.4.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adf1a0d272633b21d645dd6e02e3293429c1141c7d65a58e4cbcd592d53b8e01"}, - {file = "coverage-6.4.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff9832434a9193fbd716fbe05f9276484e18d26cc4cf850853594bb322807ac3"}, - {file = "coverage-6.4.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:923f9084d7e1d31b5f74c92396b05b18921ed01ee5350402b561a79dce3ea48d"}, - {file = "coverage-6.4.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4d64304acf79766e650f7acb81d263a3ea6e2d0d04c5172b7189180ff2c023c"}, - {file = "coverage-6.4.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:fc294de50941d3da66a09dca06e206297709332050973eca17040278cb0918ff"}, - {file = "coverage-6.4.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a42eaaae772f14a5194f181740a67bfd48e8806394b8c67aa4399e09d0d6b5db"}, - {file = "coverage-6.4.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4822327b35cb032ff16af3bec27f73985448f08e874146b5b101e0e558b613dd"}, - {file = "coverage-6.4.3-cp39-cp39-win32.whl", hash = "sha256:f217850ac0e046ede611312703423767ca032a7b952b5257efac963942c055de"}, - {file = "coverage-6.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:0a84376e4fd13cebce2c0ef8c2f037929c8307fb94af1e5dbe50272a1c651b5d"}, - {file = "coverage-6.4.3-pp36.pp37.pp38-none-any.whl", hash = "sha256:068d6f2a893af838291b8809c876973d885543411ea460f3e6886ac0ee941732"}, - {file = "coverage-6.4.3.tar.gz", hash = "sha256:ec2ae1f398e5aca655b7084392d23e80efb31f7a660d2eecf569fb9f79b3fb94"}, + {file = "coverage-6.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e7b4da9bafad21ea45a714d3ea6f3e1679099e420c8741c74905b92ee9bfa7cc"}, + {file = "coverage-6.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fde17bc42e0716c94bf19d92e4c9f5a00c5feb401f5bc01101fdf2a8b7cacf60"}, + {file = "coverage-6.4.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdbb0d89923c80dbd435b9cf8bba0ff55585a3cdb28cbec65f376c041472c60d"}, + {file = "coverage-6.4.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:67f9346aeebea54e845d29b487eb38ec95f2ecf3558a3cffb26ee3f0dcc3e760"}, + {file = "coverage-6.4.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42c499c14efd858b98c4e03595bf914089b98400d30789511577aa44607a1b74"}, + {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c35cca192ba700979d20ac43024a82b9b32a60da2f983bec6c0f5b84aead635c"}, + {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9cc4f107009bca5a81caef2fca843dbec4215c05e917a59dec0c8db5cff1d2aa"}, + {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5f444627b3664b80d078c05fe6a850dd711beeb90d26731f11d492dcbadb6973"}, + {file = "coverage-6.4.4-cp310-cp310-win32.whl", hash = "sha256:66e6df3ac4659a435677d8cd40e8eb1ac7219345d27c41145991ee9bf4b806a0"}, + {file = "coverage-6.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:35ef1f8d8a7a275aa7410d2f2c60fa6443f4a64fae9be671ec0696a68525b875"}, + {file = "coverage-6.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c1328d0c2f194ffda30a45f11058c02410e679456276bfa0bbe0b0ee87225fac"}, + {file = "coverage-6.4.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61b993f3998ee384935ee423c3d40894e93277f12482f6e777642a0141f55782"}, + {file = "coverage-6.4.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d5dd4b8e9cd0deb60e6fcc7b0647cbc1da6c33b9e786f9c79721fd303994832f"}, + {file = "coverage-6.4.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7026f5afe0d1a933685d8f2169d7c2d2e624f6255fb584ca99ccca8c0e966fd7"}, + {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9c7b9b498eb0c0d48b4c2abc0e10c2d78912203f972e0e63e3c9dc21f15abdaa"}, + {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ee2b2fb6eb4ace35805f434e0f6409444e1466a47f620d1d5763a22600f0f892"}, + {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ab066f5ab67059d1f1000b5e1aa8bbd75b6ed1fc0014559aea41a9eb66fc2ce0"}, + {file = "coverage-6.4.4-cp311-cp311-win32.whl", hash = "sha256:9d6e1f3185cbfd3d91ac77ea065d85d5215d3dfa45b191d14ddfcd952fa53796"}, + {file = "coverage-6.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:e3d3c4cc38b2882f9a15bafd30aec079582b819bec1b8afdbde8f7797008108a"}, + {file = "coverage-6.4.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a095aa0a996ea08b10580908e88fbaf81ecf798e923bbe64fb98d1807db3d68a"}, + {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef6f44409ab02e202b31a05dd6666797f9de2aa2b4b3534e9d450e42dea5e817"}, + {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b7101938584d67e6f45f0015b60e24a95bf8dea19836b1709a80342e01b472f"}, + {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14a32ec68d721c3d714d9b105c7acf8e0f8a4f4734c811eda75ff3718570b5e3"}, + {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6a864733b22d3081749450466ac80698fe39c91cb6849b2ef8752fd7482011f3"}, + {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:08002f9251f51afdcc5e3adf5d5d66bb490ae893d9e21359b085f0e03390a820"}, + {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a3b2752de32c455f2521a51bd3ffb53c5b3ae92736afde67ce83477f5c1dd928"}, + {file = "coverage-6.4.4-cp37-cp37m-win32.whl", hash = "sha256:f855b39e4f75abd0dfbcf74a82e84ae3fc260d523fcb3532786bcbbcb158322c"}, + {file = "coverage-6.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ee6ae6bbcac0786807295e9687169fba80cb0617852b2fa118a99667e8e6815d"}, + {file = "coverage-6.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:564cd0f5b5470094df06fab676c6d77547abfdcb09b6c29c8a97c41ad03b103c"}, + {file = "coverage-6.4.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cbbb0e4cd8ddcd5ef47641cfac97d8473ab6b132dd9a46bacb18872828031685"}, + {file = "coverage-6.4.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6113e4df2fa73b80f77663445be6d567913fb3b82a86ceb64e44ae0e4b695de1"}, + {file = "coverage-6.4.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d032bfc562a52318ae05047a6eb801ff31ccee172dc0d2504614e911d8fa83e"}, + {file = "coverage-6.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e431e305a1f3126477abe9a184624a85308da8edf8486a863601d58419d26ffa"}, + {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cf2afe83a53f77aec067033199797832617890e15bed42f4a1a93ea24794ae3e"}, + {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:783bc7c4ee524039ca13b6d9b4186a67f8e63d91342c713e88c1865a38d0892a"}, + {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ff934ced84054b9018665ca3967fc48e1ac99e811f6cc99ea65978e1d384454b"}, + {file = "coverage-6.4.4-cp38-cp38-win32.whl", hash = "sha256:e1fabd473566fce2cf18ea41171d92814e4ef1495e04471786cbc943b89a3781"}, + {file = "coverage-6.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:4179502f210ebed3ccfe2f78bf8e2d59e50b297b598b100d6c6e3341053066a2"}, + {file = "coverage-6.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:98c0b9e9b572893cdb0a00e66cf961a238f8d870d4e1dc8e679eb8bdc2eb1b86"}, + {file = "coverage-6.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc600f6ec19b273da1d85817eda339fb46ce9eef3e89f220055d8696e0a06908"}, + {file = "coverage-6.4.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a98d6bf6d4ca5c07a600c7b4e0c5350cd483c85c736c522b786be90ea5bac4f"}, + {file = "coverage-6.4.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01778769097dbd705a24e221f42be885c544bb91251747a8a3efdec6eb4788f2"}, + {file = "coverage-6.4.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfa0b97eb904255e2ab24166071b27408f1f69c8fbda58e9c0972804851e0558"}, + {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:fcbe3d9a53e013f8ab88734d7e517eb2cd06b7e689bedf22c0eb68db5e4a0a19"}, + {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:15e38d853ee224e92ccc9a851457fb1e1f12d7a5df5ae44544ce7863691c7a0d"}, + {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6913dddee2deff8ab2512639c5168c3e80b3ebb0f818fed22048ee46f735351a"}, + {file = "coverage-6.4.4-cp39-cp39-win32.whl", hash = "sha256:354df19fefd03b9a13132fa6643527ef7905712109d9c1c1903f2133d3a4e145"}, + {file = "coverage-6.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:1238b08f3576201ebf41f7c20bf59baa0d05da941b123c6656e42cdb668e9827"}, + {file = "coverage-6.4.4-pp36.pp37.pp38-none-any.whl", hash = "sha256:f67cf9f406cf0d2f08a3515ce2db5b82625a7257f88aad87904674def6ddaec1"}, + {file = "coverage-6.4.4.tar.gz", hash = "sha256:e16c45b726acb780e1e6f88b286d3c10b3914ab03438f32117c4aa52d7f30d58"}, ] docstring-parser = [ {file = "docstring_parser-0.14.1-py3-none-any.whl", hash = "sha256:14ac6ec1f1ba6905c4d8cb90fd0bc55394f5678183752c90e44812bf28d7a515"}, {file = "docstring_parser-0.14.1.tar.gz", hash = "sha256:2c77522e31b7c88b1ab457a1f3c9ae38947ad719732260ba77ee8a3deb58622a"}, ] +frozendict = [ + {file = "frozendict-2.3.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4a3b32d47282ae0098b9239a6d53ec539da720258bd762d62191b46f2f87c5fc"}, + {file = "frozendict-2.3.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84c9887179a245a66a50f52afa08d4d92ae0f269839fab82285c70a0fa0dd782"}, + {file = "frozendict-2.3.4-cp310-cp310-win_amd64.whl", hash = "sha256:b98a0d65a59af6da03f794f90b0c3085a7ee14e7bf8f0ef36b079ee8aa992439"}, + {file = "frozendict-2.3.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:3d8042b7dab5e992e30889c9b71b781d5feef19b372d47d735e4d7d45846fd4a"}, + {file = "frozendict-2.3.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25a6d2e8b7cf6b6e5677a1a4b53b4073e5d9ec640d1db30dc679627668d25e90"}, + {file = "frozendict-2.3.4-cp36-cp36m-win_amd64.whl", hash = "sha256:dbbe1339ac2646523e0bb00d1896085d1f70de23780e4927ca82b36ab8a044d3"}, + {file = "frozendict-2.3.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95bac22f7f09d81f378f2b3f672b7a50a974ca180feae1507f5e21bc147e8bc8"}, + {file = "frozendict-2.3.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dae686722c144b333c4dbdc16323a5de11406d26b76d2be1cc175f90afacb5ba"}, + {file = "frozendict-2.3.4-cp37-cp37m-win_amd64.whl", hash = "sha256:389f395a74eb16992217ac1521e689c1dea2d70113bcb18714669ace1ed623b9"}, + {file = "frozendict-2.3.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ccb6450a416c9cc9acef7683e637e28356e3ceeabf83521f74cc2718883076b7"}, + {file = "frozendict-2.3.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aca59108b77cadc13ba7dfea7e8f50811208c7652a13dc6c7f92d7782a24d299"}, + {file = "frozendict-2.3.4-cp38-cp38-win_amd64.whl", hash = "sha256:3ec86ebf143dd685184215c27ec416c36e0ba1b80d81b1b9482f7d380c049b4e"}, + {file = "frozendict-2.3.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5809e6ff6b7257043a486f7a3b73a7da71cf69a38980b4171e4741291d0d9eb3"}, + {file = "frozendict-2.3.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c550ed7fdf1962984bec21630c584d722b3ee5d5f57a0ae2527a0121dc0414a"}, + {file = "frozendict-2.3.4-cp39-cp39-win_amd64.whl", hash = "sha256:3e93aebc6e69a8ef329bbe9afb8342bd33c7b5c7a0c480cb9f7e60b0cbe48072"}, + {file = "frozendict-2.3.4-py3-none-any.whl", hash = "sha256:d722f3d89db6ae35ef35ecc243c40c800eb344848c83dba4798353312cd37b15"}, + {file = "frozendict-2.3.4.tar.gz", hash = "sha256:15b4b18346259392b0d27598f240e9390fafbff882137a9c48a1e0104fb17f78"}, +] importlib-metadata = [ {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, diff --git a/pyproject.toml b/pyproject.toml index 4626b66fb..047848b6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.2.3" +version = "0.2.4" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] @@ -17,6 +17,7 @@ PyYAML = "^6.0" termcolor = "^1.1.0" "backports.cached-property" = { version = "^1.0.2", python = "~3.7" } colorama = {version = "^0.4.0", platform = "win32"} +frozendict = "^2.3.4" [tool.poetry.dev-dependencies] pytest = "^7.1.2" diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index 0c33b90de..a9ebf83ec 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -36,18 +36,12 @@ def main(params: Mapping[Literal[1, 3, 5, 7], bool] = {5: False, 1: True}) -> An return params assert dcargs.cli(main, args=[]) == {5: False, 1: True} - assert dcargs.cli(main, args="--params 5 True 3 False".split(" ")) == { + assert dcargs.cli(main, args="--params.5 --params.no-1".split(" ")) == { 5: True, - 3: False, + 1: False, } with pytest.raises(SystemExit): dcargs.cli(main, args="--params".split(" ")) - with pytest.raises(SystemExit): - dcargs.cli(main, args="--params 5 Tru 3 False".split(" ")) - with pytest.raises(SystemExit): - dcargs.cli(main, args="--params 5 Tru 3 False".split(" ")) - with pytest.raises(SystemExit): - dcargs.cli(main, args="--params 4 Tru 3 False".split(" ")) def test_tuple_in_dict(): diff --git a/tests/test_nested.py b/tests/test_nested.py index 6663de44d..eae4840da 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -1,7 +1,8 @@ import dataclasses -from typing import Any, Generic, Optional, Tuple, TypeVar, Union +from typing import Any, Generic, Mapping, Optional, Tuple, TypeVar, Union import pytest +from frozendict import frozendict # type: ignore import dcargs @@ -728,3 +729,19 @@ class Parent: Parent, args="nested1.nested2.subcommand:b --nested1.nested2.subcommand.b 7".split(" "), ) == Parent(Nested1(Nested2(B(7)))) + + +def test_frozen_dict(): + def main( + x: Mapping[str, float] = frozendict( + { + "num_epochs": 20, + "batch_size": 64, + } + ) + ): + return x + + assert hash(dcargs.cli(main, args="--x.num-epochs 10".split(" "))) == hash( + frozendict({"num_epochs": 10, "batch_size": 64}) + ) From 3c3167f4c424ee3ed074fd1122815ee4a8698f8b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 25 Aug 2022 17:05:11 -0700 Subject: [PATCH 104/491] Sync docs changes --- docs/source/conf.py | 2 +- docs/source/goals_and_alternatives.md | 38 ++++++++++++++------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 0d1df0fb4..5f3eaee4f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -390,4 +390,4 @@ def docstring(app, what, name, obj, options, lines): def setup(app): app.connect("autodoc-process-docstring", docstring) - app.add_stylesheet("css/compact_table_header.css") + app.add_css_file("css/compact_table_header.css") diff --git a/docs/source/goals_and_alternatives.md b/docs/source/goals_and_alternatives.md index 06bbd55ce..445903b95 100644 --- a/docs/source/goals_and_alternatives.md +++ b/docs/source/goals_and_alternatives.md @@ -24,21 +24,21 @@ Usage distinctions are the result of two API goals: More concretely, we can also compare specific features. A noncomprehensive set: -| | Dataclasses | Functions | Literals | Generics | Docstrings as helptext | Nested structures | Subparsers | Dictionaries | Lists, tuples | -| -------------------------------------------- | ----------- | --------- | -------------------- | -------- | ---------------------- | ----------------- | ------------------- | ------------ | -------------------- | -| **dcargs** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| [argparse-dataclass][argparse-dataclass] | ✓ | | | | | | | | | -| [argparse-dataclasses][argparse-dataclasses] | ✓ | | | | | | | | | -| [datargs][datargs] | ✓ | | ✓[^datargs_literals] | | | | ✓ | | ✓ | -| [tap][tap] | | | ✓ | | ✓ | | ✓[^tap_subparsers] | | ✓ | -| [simple-parsing][simple-parsing] | ✓ | | ✓[^simp_literals] | | ✓ | ✓ | ✓[^simp_subparsers] | ✓ | ✓ | -| [dataclass-cli][dataclass-cli] | ✓ | | | | | | | | | -| [clout][clout] | ✓ | | | | | ✓ | | | | -| [hf_argparser][hf_argparser] | ✓ | | | | | | | ✓ | ✓ | -| [typer][typer] | | ✓ | | | | | ✓ | | ~[^typer_containers] | -| [pyrallis][pyrallis] | ✓ | | | | ✓ | ✓ | | | ✓ | -| [yahp][yahp] | ✓ | | | | ~[^yahp_docstrings] | ✓ | ✓[^yahp_subparsers] | | ✓ | -| [omegaconf][omegaconf] | ✓ | | | | | ✓ | | ✓ | ✓ | +| | Dataclasses | Functions | Literals | Docstrings as helptext | Nested structures | Unions over primitives | Unions over nested types | Lists, tuples | Dictionaries | Generics | +| -------------------------------------------- | ----------- | --------- | -------------------- | ---------------------- | ----------------- | ---------------------- | ------------------------- | -------------------- | ------------ | -------- | +| [argparse-dataclass][argparse-dataclass] | ✓ | | | | | | | | | | +| [argparse-dataclasses][argparse-dataclasses] | ✓ | | | | | | | | | | +| [datargs][datargs] | ✓ | | ✓[^datargs_literals] | | | | ✓[^datargs_unions_nested] | ✓ | | | +| [tap][tap] | | | ✓ | ✓ | | ✓ | ~[^tap_unions_nested] | ✓ | | | +| [simple-parsing][simple-parsing] | ✓ | | ✓[^simp_literals] | ✓ | ✓ | ✓ | ✓[^simp_unions_nested] | ✓ | ✓ | | +| [dataclass-cli][dataclass-cli] | ✓ | | | | | | | | | | +| [clout][clout] | ✓ | | | | ✓ | | | | | | +| [hf_argparser][hf_argparser] | ✓ | | | | | | | ✓ | ✓ | | +| [typer][typer] | | ✓ | | | | | ~[^typer_unions_nested] | ~[^typer_containers] | | | +| [pyrallis][pyrallis] | ✓ | | | ✓ | ✓ | | | ✓ | | | +| [yahp][yahp] | ✓ | | | ~[^yahp_docstrings] | ✓ | ✓ | ~[^yahp_unions_nested] | ✓ | | | +| [omegaconf][omegaconf] | ✓ | | | | ✓ | | | ✓ | ✓ | | +| **dcargs** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | @@ -55,10 +55,12 @@ More concretely, we can also compare specific features. A noncomprehensive set: [yahp]: https://github.com/mosaicml/yahp [omegaconf]: https://omegaconf.readthedocs.io/en/2.1_branch/structured_config.html -[^tap_subparsers]: Supported but not strongly typed. +[^datargs_unions_nested]: One allowed per class. +[^tap_unions_nested]: Not supported, but API exists for creating subparsers that accomplish a similar goal. +[^simp_unions_nested]: One allowed per class. +[^yahp_unions_nested]: Not supported, but similar functionality available via ["registries"](https://docs.mosaicml.com/projects/yahp/en/stable/examples/registry.html). +[^typer_unions_nested]: Not supported, but API exists for creating subparsers that accomplish a similar goal. [^simp_literals]: Not supported for mixed (eg `Literal[5, "five"]`) or in container (eg `List[Literal[1, 2]]`) types. -[^simp_subparsers]: Supported but not strongly typed. -[^yahp_subparsers]: Supported but not strongly typed. [^datargs_literals]: Not supported for mixed types (eg `Literal[5, "five"]`). [^typer_containers]: `typer` uses positional arguments for all required fields, which means that only one variable-length argument (such as `List[int]`) without a default is supported per argument parser. [^yahp_docstrings]: Via the `hp.auto()` function, which can parse docstrings from external classes. Usage is different from the more direct parsing that `dcargs`, `tap`, and `simple-parsing`/`pyrallis` support. From e4da3fc98ad076c1350f5522c62e55205d247ef8 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 25 Aug 2022 20:20:15 -0700 Subject: [PATCH 105/491] Improve formatting for long helptext --- dcargs/_argparse_formatter.py | 88 ++++++++++++++++++++++++++++++++--- dcargs/_instantiators.py | 18 ++++--- dcargs/_strings.py | 14 ++++++ tests/test_dict_namedtuple.py | 6 ++- 4 files changed, 108 insertions(+), 18 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index b88251f72..dc5ddebc2 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -1,10 +1,21 @@ import argparse import contextlib +import itertools +import re as _re +from gettext import gettext as _ +from gettext import ngettext from typing import Any, ContextManager, Generator from . import _strings +def monkeypatch_len(obj: Any) -> int: + if isinstance(obj, str): + return len(_strings.strip_ansi_sequences(obj)) + else: + return len(obj) + + def ansi_context() -> ContextManager[None]: """Context for working with ANSI codes + argparse: - Applies a temporary monkey patch for making argparse ignore ANSI codes when @@ -14,14 +25,9 @@ def ansi_context() -> ContextManager[None]: @contextlib.contextmanager def inner() -> Generator[None, None, None]: - def monkeypatched_len(obj: Any) -> int: - if isinstance(obj, str): - return len(_strings.strip_ansi_sequences(obj)) - else: - return len(obj) if not hasattr(argparse, "len"): - argparse.len = monkeypatched_len # type: ignore + argparse.len = monkeypatch_len # type: ignore try: # Use Colorama to support coloring in Windows shells. import colorama # type: ignore @@ -54,8 +60,78 @@ def monkeypatched_len(obj: Any) -> int: class ArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): + def __init__( + self, + prog, + indent_increment=2, + max_help_position=64, # Usually 24 + width=None, + ): + super().__init__(prog, indent_increment, max_help_position, width) + def _format_args(self, action, default_metavar): """Override _format_args() to ignore nargs and always expect single string metavars.""" get_metavar = self._metavar_formatter(action, default_metavar) return get_metavar(1)[0] + + def _format_action(self, action): + # determine the required width and the entry label + help_position = min(self._action_max_length + 2, self._max_help_position) + help_width = max(self._width - help_position, 11) + action_width = help_position - self._current_indent - 2 + action_header = self._format_action_invocation(action) + + # no help; start on same line and add a final newline + if not action.help: + tup = self._current_indent, "", action_header + action_header = "%*s%s\n" % tup + + # short action name; start on the same line and pad two spaces + elif monkeypatch_len(action_header) <= action_width: + # Original: + # tup = self._current_indent, "", action_width, action_header + # action_header = "%*s%-*s " % tup + # + action_header = ( + " " * self._current_indent + + action_header + + " " * (action_width - monkeypatch_len(action_header)) + ) + # + indent_first = 0 + + # long action name; start on the next line + else: + tup = self._current_indent, "", action_header + action_header = "%*s%s\n" % tup + indent_first = help_position + + # collect the pieces of the action help + parts = [action_header] + + # if there was help for the action, add lines of help text + if action.help: + help_text = self._expand_help(action) + # + # Respect existing line breaks. + help_lines = tuple( + itertools.chain( + *(self._split_lines(h, help_width) for h in help_text.split("\n")) + ) + ) + # + parts.append("%*s%s\n" % (indent_first, "", help_lines[0])) + for line in help_lines[1:]: + parts.append("%*s%s\n" % (help_position, "", line)) + + # or add a newline if the description doesn't end with one + elif not action_header.endswith("\n"): + parts.append("\n") + + # if there are any sub-actions, add their help as well + for subaction in self._iter_indented_subactions(action): + parts.append(self._format_action(subaction)) + + # return a single string + return self._join_parts(parts) diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index b2482cb4c..72db1b5f1 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -56,6 +56,8 @@ import termcolor from typing_extensions import Annotated, Final, Literal, get_args, get_origin +from . import _strings + _StandardInstantiator = Callable[[List[str]], Any] # Special case: the only time that argparse doesn't give us a string is when the # argument action is set to `store_true` or `store_false`. In this case, we get @@ -94,10 +96,6 @@ class UnsupportedTypeAnnotationError(Exception): ) -def _format_metavar(x: str) -> str: - return termcolor.colored(x, attrs=["bold"]) - - def instantiator_from_type( typ: Type, type_from_typevar: Dict[TypeVar, Type] ) -> Tuple[Instantiator, InstantiatorMetadata]: @@ -131,7 +129,7 @@ def instantiator(strings: List[str]) -> None: return instantiator, InstantiatorMetadata( nargs=1, - metavar="{" + _format_metavar("None") + "}", + metavar="{" + _strings.format_metavar("None") + "}", choices=("None",), ) @@ -218,9 +216,9 @@ def instantiator_base_case(strings: List[str]) -> Any: return instantiator_base_case, InstantiatorMetadata( nargs=1, - metavar=_format_metavar(typ.__name__.upper()) + metavar=_strings.format_metavar(typ.__name__.upper()) if auto_choices is None - else "{" + ",".join(map(_format_metavar, map(str, auto_choices))) + "}", + else "{" + ",".join(map(_strings.format_metavar, map(str, auto_choices))) + "}", choices=auto_choices, ) @@ -519,7 +517,7 @@ def dict_instantiator(strings: List[str]) -> Any: pair_metavar = f"{key_meta.metavar} {val_meta.metavar}" return dict_instantiator, InstantiatorMetadata( nargs="+", - metavar=f"{pair_metavar} [{pair_metavar} ...]", + metavar=_strings.multi_metavar_from_single(pair_metavar), choices=None, ) @@ -563,7 +561,7 @@ def sequence_instantiator(strings: List[str]) -> Any: return sequence_instantiator, InstantiatorMetadata( nargs="+", - metavar=f"{inner_meta.metavar} [{inner_meta.metavar} ...]", + metavar=_strings.multi_metavar_from_single(inner_meta.metavar), choices=inner_meta.choices, ) @@ -579,7 +577,7 @@ def _instantiator_from_literal( lambda strings: choices[str_choices.index(strings[0])], InstantiatorMetadata( nargs=1, - metavar="{" + ",".join(map(_format_metavar, str_choices)) + "}", + metavar="{" + ",".join(map(_strings.format_metavar, str_choices)) + "}", choices=str_choices, ), ) diff --git a/dcargs/_strings.py b/dcargs/_strings.py index e9a13a573..7ac05c46d 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -5,6 +5,8 @@ import textwrap from typing import Iterable, List, Sequence, Type, Union +import termcolor + from . import _resolver dummy_field_name = "__dcargs_dummy_field_name__" @@ -80,3 +82,15 @@ def _get_ansi_pattern() -> re.Pattern: def strip_ansi_sequences(x: str): return _get_ansi_pattern().sub("", x) + + +def format_metavar(x: str) -> str: + return termcolor.colored(x, attrs=["bold"]) + + +def multi_metavar_from_single(single: str) -> str: + if len(strip_ansi_sequences(single)) >= 32: + # Shorten long metavars + return f"{single} [...]" + else: + return f"{single} [{single} ...]" diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index a9ebf83ec..d1898e183 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -246,9 +246,11 @@ class HelptextNamedTupleDefault(NamedTuple): dcargs.cli(HelptextNamedTupleDefault, args=["--help"]) helptext = dcargs._strings.strip_ansi_sequences(f.getvalue()) assert cast(str, HelptextNamedTupleDefault.__doc__) in helptext - assert "--x INT Documentation 1 (required)\n" in helptext - assert "--y INT Documentation 2 (required)\n" in helptext + assert "--x INT" in helptext + assert "--y INT" in helptext assert "--z INT" in helptext + assert "Documentation 1 (required)\n" in helptext + assert "Documentation 2 (required)\n" in helptext assert "Documentation 3 (default: 3)\n" in helptext From 7f3e76071a07f8cf3b10a8ff26f388e33a713776 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 25 Aug 2022 20:31:59 -0700 Subject: [PATCH 106/491] Fix spacing bug --- dcargs/_argparse_formatter.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index dc5ddebc2..08a28a5f8 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -96,7 +96,7 @@ def _format_action(self, action): action_header = ( " " * self._current_indent + action_header - + " " * (action_width - monkeypatch_len(action_header)) + + " " * (action_width - monkeypatch_len(action_header) + 2) ) # indent_first = 0 diff --git a/pyproject.toml b/pyproject.toml index 047848b6e..2cd9b4ca6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.2.4" +version = "0.2.5" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From 19a29789ffed2d860fdf59f5a87d9a2e7f4a5b0d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 26 Aug 2022 19:28:36 -0700 Subject: [PATCH 107/491] Responsive helptext --- dcargs/_argparse_formatter.py | 41 ++++++++++++++++++++++++----------- dcargs/_cli.py | 5 ++++- dcargs/_fields.py | 14 +----------- dcargs/_instantiators.py | 1 - dcargs/_parsers.py | 6 +++-- 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index 08a28a5f8..be67d18af 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -1,10 +1,9 @@ import argparse import contextlib +import functools import itertools -import re as _re -from gettext import gettext as _ -from gettext import ngettext -from typing import Any, ContextManager, Generator +import shutil +from typing import Any, ContextManager, Generator, Type from . import _strings @@ -25,7 +24,6 @@ def ansi_context() -> ContextManager[None]: @contextlib.contextmanager def inner() -> Generator[None, None, None]: - if not hasattr(argparse, "len"): argparse.len = monkeypatch_len # type: ignore try: @@ -59,14 +57,20 @@ def inner() -> Generator[None, None, None]: return inner() -class ArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): - def __init__( - self, - prog, - indent_increment=2, - max_help_position=64, # Usually 24 - width=None, - ): +def make_formatter_class(field_count: int) -> Any: + return functools.partial(_ArgparseHelpFormatter, field_count=field_count) + + +class _ArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): + def __init__(self, prog, *, field_count: int): + indent_increment = 2 + width = shutil.get_terminal_size().columns - 2 + max_help_position = min(36, width // 3) # Usual is 24. + + # Try to make helptext more concise when we have a lot of fields! + if field_count > 16 and width >= 100: # pragma: no cover + max_help_position = min(96, width // 2) # Usual is 24. + super().__init__(prog, indent_increment, max_help_position, width) def _format_args(self, action, default_metavar): @@ -75,6 +79,17 @@ def _format_args(self, action, default_metavar): get_metavar = self._metavar_formatter(action, default_metavar) return get_metavar(1)[0] + def add_argument(self, action): # pragma: no cover + # Patch to avoid super long arguments from shifting the helptext of all of the + # fields. + prev_max_length = self._action_max_length + super().add_argument(action) + if ( + self._action_max_length >= 40 + and self._action_max_length > self._max_help_position + 2 + ): + self._action_max_length = prev_max_length + def _format_action(self, action): # determine the required width and the entry label help_position = min(self._action_max_length + 2, self._max_help_position) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 11222cc80..a7faa05b9 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -184,7 +184,10 @@ def _cli_impl( # Generate parser! with _argparse_formatter.ansi_context(): parser = argparse.ArgumentParser( - prog=prog, formatter_class=_argparse_formatter.ArgparseHelpFormatter + prog=prog, + formatter_class=_argparse_formatter.make_formatter_class( + len(parser_definition.args) + ), ) parser_definition.apply(parser) if _return_stage == "parser": diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 9ad5574aa..4d930c1d9 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -11,18 +11,7 @@ import itertools import typing import warnings -from typing import ( - Any, - Callable, - Hashable, - Iterable, - List, - Mapping, - Optional, - Type, - Union, - cast, -) +from typing import Any, Callable, Hashable, Iterable, List, Optional, Type, Union, cast import docstring_parser import typing_extensions @@ -87,7 +76,6 @@ class ExcludeFromCallType(_Singleton): `default_instance` for `dcargs.cli()` as required.""" -MISSING_TYPE = Union[PropagatingMissingType, NonpropagatingMissingType] MISSING_SINGLETONS = [ dataclasses.MISSING, MISSING_PROP, diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index 72db1b5f1..4a0fd20b4 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -53,7 +53,6 @@ overload, ) -import termcolor from typing_extensions import Annotated, Final, Literal, get_args, get_origin from . import _strings diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 5f430595f..46d6965c1 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -367,14 +367,16 @@ def apply( if self.can_be_none: subparser = argparse_subparsers.add_parser( name=_strings.subparser_name_from_type(self.prefix, None), - formatter_class=_argparse_formatter.ArgparseHelpFormatter, + formatter_class=_argparse_formatter.make_formatter_class(0), ) subparser_tree_nodes.append(subparser) for name, subparser_def in self.parser_from_name.items(): subparser = argparse_subparsers.add_parser( name, - formatter_class=_argparse_formatter.ArgparseHelpFormatter, + formatter_class=_argparse_formatter.make_formatter_class( + len(subparser_def.args) + ), ) subparser_def.apply(subparser) From f5c28656c432a4aad89658fd05ac85738494edcf Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 27 Aug 2022 19:27:04 -0700 Subject: [PATCH 108/491] Annotation improvements for pyright --- dcargs/_argparse_formatter.py | 2 +- dcargs/_cli.py | 137 ++++++++++++++++++++++++---------- dcargs/_fields.py | 4 +- pyproject.toml | 2 +- 4 files changed, 104 insertions(+), 41 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index be67d18af..3a59b9b67 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -3,7 +3,7 @@ import functools import itertools import shutil -from typing import Any, ContextManager, Generator, Type +from typing import Any, ContextManager, Generator from . import _strings diff --git a/dcargs/_cli.py b/dcargs/_cli.py index a7faa05b9..ee40c5308 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -8,47 +8,52 @@ from . import _argparse_formatter, _calling, _fields, _parsers, _strings -T = TypeVar("T") +OutT = TypeVar("OutT") -def generate_parser( - f: Callable[..., T], +# Overload notes: +# 1. Type[T] is almost a subtype of Callable[..., T]; the difference is types like +# Union[T1, T2] which fall under the former but not the latter. +# 2. We really shouldn't need an overload here. But as of 1.1.268, it seems like it's +# needed for pyright to understand that Union types are OK to pass in directly. +# Hopefully we can just switch to a Union[Type[...], Callable[...]] in the future. + + +@overload +def cli( + f: Type[OutT], *, prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[T] = None, + default_instance: Optional[OutT] = None, avoid_subparsers: bool = False, -) -> argparse.ArgumentParser: - """Returns the argparse parser that would be used under-the-hood if `dcargs.cli()` - was called with the same arguments. +) -> OutT: + ... - This can be useful for libraries like argcomplete, pyzshcomplete, or shtab, which - enable autocompletion for argparse parsers.""" - # Potentially we could do some caching in the future, to reduce the redundant work - # done when both `generate_parser()` and `cli()` are called. - out = _cli_impl( - "parser", - f, - prog=prog, - description=description, - args=args, - default_instance=default_instance, - avoid_subparsers=avoid_subparsers, - ) - assert isinstance(out, argparse.ArgumentParser) - return out + +@overload +def cli( + f: Callable[..., OutT], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + default_instance: Optional[OutT] = None, + avoid_subparsers: bool = False, +) -> OutT: + ... def cli( - f: Callable[..., T], + f: Union[Type[OutT], Callable[..., OutT]], *, prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[T] = None, + default_instance: Optional[OutT] = None, avoid_subparsers: bool = False, -) -> T: +) -> OutT: """Call `f(...)`, with arguments populated from an automatically generated CLI interface. @@ -114,15 +119,68 @@ def cli( return out +@overload +def generate_parser( + f: Type[OutT], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + default_instance: Optional[OutT] = None, + avoid_subparsers: bool = False, +) -> argparse.ArgumentParser: + ... + + +@overload +def generate_parser( + f: Callable[..., OutT], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + default_instance: Optional[OutT] = None, + avoid_subparsers: bool = False, +) -> argparse.ArgumentParser: + ... + + +def generate_parser( + f: Union[Type[OutT], Callable[..., OutT]], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + default_instance: Optional[OutT] = None, + avoid_subparsers: bool = False, +) -> argparse.ArgumentParser: + """Returns the argparse parser that would be used under-the-hood if `dcargs.cli()` + was called with the same arguments. + + This can be useful for libraries like argcomplete, pyzshcomplete, or shtab, which + enable autocompletion for argparse parsers.""" + out = _cli_impl( + "parser", + f, + prog=prog, + description=description, + args=args, + default_instance=default_instance, + avoid_subparsers=avoid_subparsers, + ) + assert isinstance(out, argparse.ArgumentParser) + return out + + @overload def _cli_impl( _return_stage: Literal["parser"], - f: Callable[..., T], + f: Callable[..., OutT], *, prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[T] = None, + default_instance: Optional[OutT] = None, avoid_subparsers: bool = False, ) -> argparse.ArgumentParser: ... @@ -131,36 +189,39 @@ def _cli_impl( @overload def _cli_impl( _return_stage: Literal["f_out"], - f: Callable[..., T], + f: Callable[..., OutT], *, prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[T] = None, + default_instance: Optional[OutT] = None, avoid_subparsers: bool = False, -) -> T: +) -> OutT: ... def _cli_impl( _return_stage: Literal["parser", "f_out"], - f: Callable[..., T], + f: Callable[..., OutT], *, prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[T] = None, + default_instance: Optional[OutT] = None, avoid_subparsers: bool = False, -) -> Union[T, argparse.ArgumentParser]: - default_instance_internal: Union[_fields.NonpropagatingMissingType, T] = ( +) -> Union[OutT, argparse.ArgumentParser]: + default_instance_internal: Union[_fields.NonpropagatingMissingType, OutT] = ( _fields.MISSING_NONPROP if default_instance is None else default_instance ) if not _fields.is_nested_type(cast(Type, f), default_instance_internal): - dummy_field = dataclasses.field( - default=default_instance - if default_instance is not None - else dataclasses.MISSING + dummy_field = cast( + dataclasses.Field, + dataclasses.field( + default=default_instance + if default_instance is not None + else dataclasses.MISSING + ), ) f = dataclasses.make_dataclass( cls_name="", diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 4d930c1d9..412e84b37 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -343,7 +343,9 @@ def _field_list_from_tuple( default_i = default_instance[i] # type: ignore field_list.append( FieldDefinition( - # We'd use an index operator h + # Ideally we'd have --tuple[0] instead of --tuple.0 as the command-line + # argument, but in practice the brackets are annoying because they + # require escaping. name=str(i), typ=child, default=default_i, diff --git a/pyproject.toml b/pyproject.toml index 2cd9b4ca6..7dd66399e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.2.5" +version = "0.2.6" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From bd63936d9baf364287516947b87ba89e3fac974e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 30 Aug 2022 06:15:08 -0700 Subject: [PATCH 109/491] Initial pass at implementing #4 --- dcargs/__init__.py | 3 +- dcargs/_argparse_formatter.py | 2 +- dcargs/_calling.py | 8 +- dcargs/_fields.py | 17 ++++- dcargs/_parsers.py | 44 ++++++++--- dcargs/_resolver.py | 58 ++++++++++++-- dcargs/_strings.py | 11 +++ dcargs/extras/__init__.py | 6 +- dcargs/extras/_base_configs.py | 75 +++++++++++++++++++ dcargs/metadata/__init__.py | 16 ++++ dcargs/metadata/_subcommands.py | 55 ++++++++++++++ ..._unions.rst => 06_literals_and_unions.rst} | 6 +- ...tional_args.rst => 07_positional_args.rst} | 10 +-- .../{09_subparsers.rst => 08_subparsers.rst} | 22 +++--- ...parsers.rst => 09_multiple_subparsers.rst} | 18 ++--- ...6_base_configs.rst => 10_base_configs.rst} | 55 +++++--------- ...nd_unions.py => 06_literals_and_unions.py} | 2 +- ...sitional_args.py => 07_positional_args.py} | 4 +- .../{09_subparsers.py => 08_subparsers.py} | 10 +-- ...ubparsers.py => 09_multiple_subparsers.py} | 8 +- ...{06_base_configs.py => 10_base_configs.py} | 43 +++-------- tests/test_helptext.py | 8 +- tests/test_nested.py | 16 ++++ 23 files changed, 357 insertions(+), 140 deletions(-) create mode 100644 dcargs/extras/_base_configs.py create mode 100644 dcargs/metadata/__init__.py create mode 100644 dcargs/metadata/_subcommands.py rename docs/source/examples/{07_literals_and_unions.rst => 06_literals_and_unions.rst} (92%) rename docs/source/examples/{08_positional_args.rst => 07_positional_args.rst} (88%) rename docs/source/examples/{09_subparsers.rst => 08_subparsers.rst} (68%) rename docs/source/examples/{10_multiple_subparsers.rst => 09_multiple_subparsers.rst} (79%) rename docs/source/examples/{06_base_configs.rst => 10_base_configs.rst} (65%) rename examples/{07_literals_and_unions.py => 06_literals_and_unions.py} (96%) rename examples/{08_positional_args.py => 07_positional_args.py} (93%) rename examples/{09_subparsers.py => 08_subparsers.py} (73%) rename examples/{10_multiple_subparsers.py => 09_multiple_subparsers.py} (84%) rename examples/{06_base_configs.py => 10_base_configs.py} (66%) diff --git a/dcargs/__init__.py b/dcargs/__init__.py index 4022ff6a4..0b317d4f1 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,10 +1,11 @@ -from . import extras +from . import extras, metadata from ._cli import cli, generate_parser from ._fields import MISSING_PUBLIC as MISSING from ._instantiators import UnsupportedTypeAnnotationError __all__ = [ "extras", + "metadata", "cli", "generate_parser", "MISSING", diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index 3a59b9b67..bfc302926 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -136,7 +136,7 @@ def _format_action(self, action): ) ) # - parts.append("%*s%s\n" % (indent_first, "", help_lines[0])) + parts.append("%*s%s\n" % (indent_first, "", help_lines[0])) # type: ignore for line in help_lines[1:]: parts.append("%*s%s\n" % (help_position, "", line)) diff --git a/dcargs/_calling.py b/dcargs/_calling.py index 958b587ff..cfab0e48e 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -56,11 +56,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: prefixed_field_name = _strings.make_field_name([field_name_prefix, field.name]) # Resolve field type. - field_type = ( - type_from_typevar[field.typ] # type: ignore - if field.typ in type_from_typevar - else field.typ - ) + field_type = type_from_typevar.get(field.typ, field.typ) # type: ignore if prefixed_field_name in arg_from_prefixed_field_name: assert prefixed_field_name not in consumed_keywords @@ -148,7 +144,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: value = None else: options = map( - lambda x: x if x not in type_from_typevar else type_from_typevar[x], + lambda x: type_from_typevar.get(x, x), get_args(field_type), ) chosen_f = None diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 412e84b37..a61de72d0 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -145,6 +145,14 @@ def _try_field_list_from_callable( f: Union[Callable, Type], default_instance: _DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: + from . import metadata as _metadata + + f, subcommand_config = _resolver.unwrap_annotated( + f, _metadata._subcommands._SubcommandConfiguration + ) + if subcommand_config is not None: + default_instance = subcommand_config.default + # Unwrap generics. f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) @@ -156,7 +164,7 @@ def _try_field_list_from_callable( if isinstance(f, type): cls = f f = cls.__init__ # type: ignore - f_origin: Callable = cls + f_origin: Callable = cls # type: ignore f_origin = _resolver.unwrap_origin(f) # Try special cases. @@ -191,6 +199,7 @@ def _try_field_list_from_callable( " default to infer from." ) assert isinstance(default_instance, Iterable) + contained_type = next(iter(default_instance)) else: (contained_type,) = get_args(f) f_origin = list if f_origin is typing.Sequence else f_origin # type: ignore @@ -232,7 +241,7 @@ def _try_field_list_from_typeddict( and default_instance is not EXCLUDE_FROM_CALL ) assert not valid_default_instance or isinstance(default_instance, dict) - for name, typ in get_type_hints(cls).items(): + for name, typ in get_type_hints(cls, include_extras=True).items(): if valid_default_instance: default = default_instance.get(name, MISSING_PROP) # type: ignore elif getattr(cls, "__total__") is False: @@ -268,7 +277,7 @@ def _try_field_list_from_namedtuple( field_defaults = getattr(cls, "_field_defaults") # Note that _field_types is removed in Python 3.9. - for name, typ in _resolver.get_type_hints(cls).items(): + for name, typ in get_type_hints(cls, include_extras=True).items(): # Get default, with priority for `default_instance`. default = field_defaults.get(name, MISSING_NONPROP) if hasattr(default_instance, name): @@ -483,7 +492,7 @@ def _field_list_from_params( # This will throw a type error for torch.device, typing.Dict, etc. try: - hints = get_type_hints(f) + hints = get_type_hints(f, include_extras=True) except TypeError: return UnsupportedNestedTypeMessage(f"Could not get hints for {f}!") diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 46d6965c1..5277d4a38 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -18,6 +18,7 @@ _resolver, _strings, ) +from . import metadata as _metadata T = TypeVar("T") @@ -255,18 +256,38 @@ def from_field( # Add subparser for each option. parser_from_name: Dict[str, ParserSpecification] = {} for option in options_no_none: - subparser_name = _strings.subparser_name_from_type(prefix, option) - parser_from_name[subparser_name] = ParserSpecification.from_callable( + name = _strings.subparser_name_from_type(prefix, option) + option, subcommand_config = _resolver.unwrap_annotated( + option, _metadata._subcommands._SubcommandConfiguration + ) + if subcommand_config is None: + subcommand_config = _metadata._subcommands._SubcommandConfiguration( + "unused", + description=None, + default=( + field.default + if type(field.default) is _resolver.unwrap_origin(option) + else _fields.MISSING_NONPROP + ), + ) + + subparser = ParserSpecification.from_callable( option, - description=None, + description=subcommand_config.description, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, - default_instance=field.default - if type(field.default) == _resolver.unwrap_origin(option) # type: ignore - else _fields.MISSING_NONPROP, + default_instance=subcommand_config.default, prefix=prefix, avoid_subparsers=avoid_subparsers, ) + subparser = dataclasses.replace( + subparser, + helptext_from_nested_class_field_name={ + _strings.make_field_name([field.name, k]): v + for k, v in subparser.helptext_from_nested_class_field_name.items() + }, + ) + parser_from_name[name] = subparser # Optional if: type hint is Optional[], or a default instance is provided. required = True @@ -286,9 +307,14 @@ def from_field( "Default values for generic subparsers are not supported." ) - default_parser = parser_from_name[ - _strings.subparser_name_from_type(prefix, type(field.default)) - ] + default_name = _strings.subparser_name_from_type( + prefix, type(field.default) + ) + assert default_name in parser_from_name, ( + "Default with type {type(field.default)} was passed in, but no matching" + " subparser." + ) + default_parser = parser_from_name[default_name] if any(map(lambda arg: arg.lowered.required, default_parser.args)): required = True if any( diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index f6598cf92..88e3fe032 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -2,18 +2,30 @@ import copy import dataclasses -from typing import Any, Callable, Dict, List, Tuple, Type, TypeVar, Union, cast +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Tuple, + Type, + TypeVar, + Union, + cast, +) from typing_extensions import get_args, get_origin, get_type_hints TypeOrCallable = TypeVar("TypeOrCallable", Type, Callable) -def unwrap_origin(tp: TypeOrCallable) -> TypeOrCallable: - """Returns the origin of tp if it exists. Otherwise, returns tp.""" - origin = get_origin(tp) +def unwrap_origin(typ: TypeOrCallable) -> TypeOrCallable: + """Returns the origin of typ if it exists. Otherwise, returns typ.""" + typ, _ = unwrap_annotated(typ) + origin = get_origin(typ) if origin is None: - return tp + return typ else: return origin @@ -57,7 +69,7 @@ def resolved_fields(cls: Type) -> List[dataclasses.Field]: assert dataclasses.is_dataclass(cls) fields = [] - annotations = get_type_hints(cls) + annotations = get_type_hints(cls, include_extras=True) for field in dataclasses.fields(cls): # Avoid mutating original field. field = copy.copy(field) @@ -108,3 +120,37 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: except TypeError: pass return typ + + +MetadataType = TypeVar("MetadataType") + + +def unwrap_annotated( + typ: TypeOrCallable, search_type: Optional[Type[MetadataType]] = None +) -> Tuple[TypeOrCallable, Optional[MetadataType]]: + """Helper for parsing typing.Annotated types. + + Examples: + - int, int => (int, ()) + - Annotated[int, 1], int => (int, 1) + - Annotated[int, "1"], int => (int, None) + """ + if not hasattr(typ, "__metadata__"): + return typ, None + + args = get_args(typ) + assert len(args) >= 2 + + # Don't search for a specific metadata type if `None` is passed in. + if search_type is None: + return args[0], None + + # Look through metadata for desired metadata type. + targets = tuple(x for x in args[1:] if isinstance(x, search_type)) + if len(targets) == 0: + return args[0], None + else: + assert ( + len(targets) == 1 + ), f"Found two instances of {search_type} in metadata, but only expected one." + return args[0], targets[0] diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 7ac05c46d..1cc781eef 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -54,7 +54,18 @@ def hyphen_separated_from_camel_case(name: str) -> str: def _subparser_name_from_type(cls: Type) -> str: + from .metadata import _subcommands # Prevent circular imports + cls, type_from_typevar = _resolver.resolve_generic_types(cls) + cls, subcommand_config = _resolver.unwrap_annotated( + cls, _subcommands._SubcommandConfiguration + ) + + # Subparser name from `dcargs.metadata.subcommand()`. + if subcommand_config is not None: + return subcommand_config.name + + # Subparser name from class name. if len(type_from_typevar) == 0: assert hasattr(cls, "__name__") return hyphen_separated_from_camel_case(cls.__name__) # type: ignore diff --git a/dcargs/extras/__init__.py b/dcargs/extras/__init__.py index 71ab76515..dd38ce865 100644 --- a/dcargs/extras/__init__.py +++ b/dcargs/extras/__init__.py @@ -1,3 +1,7 @@ +"""The :mod:`dcargs.extras` submodule contains helpers that complement :func:`dcargs.cli()`, but +aren't considered part of the core interface.""" + +from ._base_configs import union_type_from_mapping from ._serialization import from_yaml, to_yaml -__all__ = ["to_yaml", "from_yaml"] +__all__ = ["union_type_from_mapping", "to_yaml", "from_yaml"] diff --git a/dcargs/extras/_base_configs.py b/dcargs/extras/_base_configs.py new file mode 100644 index 000000000..cd2bb5ee2 --- /dev/null +++ b/dcargs/extras/_base_configs.py @@ -0,0 +1,75 @@ +from typing import Mapping, Type, TypeVar, Union + +from typing_extensions import Annotated + +from ..metadata import subcommand + +T = TypeVar("T") + + +def union_type_from_mapping(base_mapping: Mapping[str, T]) -> Type[T]: + """Returns a Union type for defining subcommands that choose between nested types. + + For example, when `base_mapping` is set to: + + ```python + { + "small": Config(...), + "big": Config(...), + } + ``` + + We return: + + ```python + Union[ + Annotated[ + Config, + dcargs.metadata.subcommand("small", default=Config(...)) + ], + Annotated[ + Config, + dcargs.metadata.subcommand("big", default=Config(...)) + ] + ] + ``` + + This can be used directly in dcargs.cli: + + ```python + config = dcargs.cli(union_from_base_mapping(base_mapping)) + reveal_type(config) # Should be correct! + ``` + + Or to generate annotations for functions: + + ```python + SelectableConfig = union_from_base_mapping(base_mapping) + + def train( + config: SelectableConfig, + checkpoint_path: Optional[pathlib.Path] = None, + ) -> None: + ... + + dcargs.cli(train) + ``` + + Note that Pyright understands the latter case, but mypy does not. If mypy support is + necessary we can work around this with an `if TYPE_CHECKING` guard: + + ```python + if TYPE_CHECKING: + SelectableConfig = ExperimentConfig + else: + SelectableConfig = union_from_base_mapping(base_mapping) + ``` + """ + return Union.__getitem__( # type: ignore + tuple( + Annotated.__class_getitem__( # type: ignore + (type(v), subcommand(k, default=v)) + ) + for k, v in base_mapping.items() + ) + ) diff --git a/dcargs/metadata/__init__.py b/dcargs/metadata/__init__.py new file mode 100644 index 000000000..6f2156b91 --- /dev/null +++ b/dcargs/metadata/__init__.py @@ -0,0 +1,16 @@ +"""The :mod:`dcargs.metadata` submodule contains helpers for attaching parsing-specific +metadata to types. For the forseeable future, this is limited to subcommand configuration. + +Features here are supported, but generally contradict the core design ethos of +:func:`dcargs.cli()`. + +As such: +1. Usage of existing functionality should be avoided unless absolutely necessary. +2. Introduction of new functionality should be avoided unless it (a) cannot be + reproduced with standard type annotations and (b) meaningfully improves the + usefulness of the library. +""" + +from ._subcommands import subcommand + +__all__ = ["subcommand"] diff --git a/dcargs/metadata/_subcommands.py b/dcargs/metadata/_subcommands.py new file mode 100644 index 000000000..eee8f2cb5 --- /dev/null +++ b/dcargs/metadata/_subcommands.py @@ -0,0 +1,55 @@ +import dataclasses +from typing import Any, Optional + +from .._fields import MISSING_NONPROP + + +@dataclasses.dataclass(frozen=True) +class _SubcommandConfiguration: + # Things we could potentially add: + # - `description` + # - `avoid_subparsers` + name: str + description: Optional[str] + default: Any + + +def subcommand( + name: str, + *, + description: Optional[str] = None, + default: Any = MISSING_NONPROP, +) -> Any: + """Returns a metadata object for configuring subcommands with `typing.Annotated`. + Use of this function is supported but discouraged unless absolutely necessary. + + --- + + Consider the standard approach for creating subcommands: + + ```python + dcargs.cli( + Union[NestedTypeA, NestedTypeB] + ) + ``` + + This will create two subcommands: nested-type-a and nested-type-b. + + + Annotating each type with `dcargs.metadata.subcommand()` allows us to override for + each subcommand the (a) name and (b) defaults. + + ```python + dcargs.cli( + Union[ + Annotated[ + NestedTypeA, subcommand("a", default=NestedTypeA(...)) + ], + Annotated[ + NestedTypeA, subcommand("b", default=NestedTypeA(...)) + ], + ] + ) + ``` + """ + return _SubcommandConfiguration(name, description, default) diff --git a/docs/source/examples/07_literals_and_unions.rst b/docs/source/examples/06_literals_and_unions.rst similarity index 92% rename from docs/source/examples/07_literals_and_unions.rst rename to docs/source/examples/06_literals_and_unions.rst index 6f43fa83b..f5bbf58c9 100644 --- a/docs/source/examples/07_literals_and_unions.rst +++ b/docs/source/examples/06_literals_and_unions.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -7. Literals And Unions +6. Literals And Unions ========================================== @@ -60,6 +60,6 @@ .. raw:: html - python 07_literals_and_unions.py --help + python 06_literals_and_unions.py --help -.. program-output:: python ../../examples/07_literals_and_unions.py --help +.. program-output:: python ../../examples/06_literals_and_unions.py --help diff --git a/docs/source/examples/08_positional_args.rst b/docs/source/examples/07_positional_args.rst similarity index 88% rename from docs/source/examples/08_positional_args.rst rename to docs/source/examples/07_positional_args.rst index 6dd19374a..270c9b9df 100644 --- a/docs/source/examples/08_positional_args.rst +++ b/docs/source/examples/07_positional_args.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -8. Positional Args +7. Positional Args ========================================== @@ -69,14 +69,14 @@ Positional-only arguments in functions are converted to positional CLI arguments .. raw:: html - python 08_positional_args.py --help + python 07_positional_args.py --help -.. program-output:: python ../../examples/08_positional_args.py --help +.. program-output:: python ../../examples/07_positional_args.py --help ------------ .. raw:: html - python 08_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 + python 07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 -.. program-output:: python ../../examples/08_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 +.. program-output:: python ../../examples/07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 diff --git a/docs/source/examples/09_subparsers.rst b/docs/source/examples/08_subparsers.rst similarity index 68% rename from docs/source/examples/09_subparsers.rst rename to docs/source/examples/08_subparsers.rst index f41c03a43..bc1c20f6c 100644 --- a/docs/source/examples/09_subparsers.rst +++ b/docs/source/examples/08_subparsers.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -9. Subparsers +8. Subparsers ========================================== @@ -49,38 +49,38 @@ Unions over nested types (classes or dataclasses) are populated using subparsers .. raw:: html - python 09_subparsers.py --help + python 08_subparsers.py --help -.. program-output:: python ../../examples/09_subparsers.py --help +.. program-output:: python ../../examples/08_subparsers.py --help ------------ .. raw:: html - python 09_subparsers.py cmd:commit --help + python 08_subparsers.py cmd:commit --help -.. program-output:: python ../../examples/09_subparsers.py cmd:commit --help +.. program-output:: python ../../examples/08_subparsers.py cmd:commit --help ------------ .. raw:: html - python 09_subparsers.py cmd:commit --cmd.message hello --cmd.all + python 08_subparsers.py cmd:commit --cmd.message hello --cmd.all -.. program-output:: python ../../examples/09_subparsers.py cmd:commit --cmd.message hello --cmd.all +.. program-output:: python ../../examples/08_subparsers.py cmd:commit --cmd.message hello --cmd.all ------------ .. raw:: html - python 09_subparsers.py cmd:checkout --help + python 08_subparsers.py cmd:checkout --help -.. program-output:: python ../../examples/09_subparsers.py cmd:checkout --help +.. program-output:: python ../../examples/08_subparsers.py cmd:checkout --help ------------ .. raw:: html - python 09_subparsers.py cmd:checkout --cmd.branch main + python 08_subparsers.py cmd:checkout --cmd.branch main -.. program-output:: python ../../examples/09_subparsers.py cmd:checkout --cmd.branch main +.. program-output:: python ../../examples/08_subparsers.py cmd:checkout --cmd.branch main diff --git a/docs/source/examples/10_multiple_subparsers.rst b/docs/source/examples/09_multiple_subparsers.rst similarity index 79% rename from docs/source/examples/10_multiple_subparsers.rst rename to docs/source/examples/09_multiple_subparsers.rst index ee2fd7330..0af826b7e 100644 --- a/docs/source/examples/10_multiple_subparsers.rst +++ b/docs/source/examples/09_multiple_subparsers.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -10. Multiple Subparsers +9. Multiple Subparsers ========================================== @@ -75,30 +75,30 @@ Multiple unions over nested types are populated using a series of subparsers. .. raw:: html - python 10_multiple_subparsers.py + python 09_multiple_subparsers.py -.. program-output:: python ../../examples/10_multiple_subparsers.py +.. program-output:: python ../../examples/09_multiple_subparsers.py ------------ .. raw:: html - python 10_multiple_subparsers.py --help + python 09_multiple_subparsers.py --help -.. program-output:: python ../../examples/10_multiple_subparsers.py --help +.. program-output:: python ../../examples/09_multiple_subparsers.py --help ------------ .. raw:: html - python 10_multiple_subparsers.py dataset:mnist --help + python 09_multiple_subparsers.py dataset:mnist --help -.. program-output:: python ../../examples/10_multiple_subparsers.py dataset:mnist --help +.. program-output:: python ../../examples/09_multiple_subparsers.py dataset:mnist --help ------------ .. raw:: html - python 10_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 + python 09_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 -.. program-output:: python ../../examples/10_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 +.. program-output:: python ../../examples/09_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 diff --git a/docs/source/examples/06_base_configs.rst b/docs/source/examples/10_base_configs.rst similarity index 65% rename from docs/source/examples/06_base_configs.rst rename to docs/source/examples/10_base_configs.rst index f28c00416..a78b030c3 100644 --- a/docs/source/examples/06_base_configs.rst +++ b/docs/source/examples/10_base_configs.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -6. Base Configs +10. Base Configs ========================================== @@ -20,11 +20,11 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. .. code-block:: python :linenos: - import sys from dataclasses import dataclass - from typing import Callable, Dict, Literal, Tuple, TypeVar, Union + from typing import Callable, Literal, Mapping, Tuple, Type, TypeVar, Union from torch import nn + from typing_extensions import Annotated, reveal_type import dcargs @@ -95,71 +95,52 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. } - T = TypeVar("T") - - - def cli_from_base_configs(base_library: Dict[str, T]) -> T: - """Populate an instance of `cls`, where the first positional argument is used to - select from a library of named base configs.""" - # Get base configuration name from the first positional argument. - if len(sys.argv) < 2 or sys.argv[1] not in base_library: - valid_usages = map(lambda k: f"{sys.argv[0]} {k} --help", base_library.keys()) - raise SystemExit("usage:\n " + "\n ".join(valid_usages)) - - # Get base configuration from our library, and use it for default CLI parameters. - default_instance = base_library[sys.argv[1]] - return dcargs.cli( - type(default_instance), - prog=" ".join(sys.argv[:2]), - args=sys.argv[2:], - default_instance=default_instance, + if __name__ == "__main__": + config = dcargs.cli( + dcargs.extras.union_type_from_mapping(base_configs), # `avoid_subparsers` will avoid making a subparser for unions when a default is - # provided; in this case, it simplifies our CLI but makes it less expressive - # (cannot switch away from the base optimizer types). + # provided; it simplifies our CLI but makes it less expressive. avoid_subparsers=True, ) - - - if __name__ == "__main__": - config = cli_from_base_configs(base_configs) + reveal_type(config) # Should ExperimentConfig, both staticaly and dynamically. print(config) ------------ .. raw:: html - python 06_base_configs.py + python 10_base_configs.py -.. program-output:: python ../../examples/06_base_configs.py +.. program-output:: python ../../examples/10_base_configs.py ------------ .. raw:: html - python 06_base_configs.py small --help + python 10_base_configs.py small --help -.. program-output:: python ../../examples/06_base_configs.py small --help +.. program-output:: python ../../examples/10_base_configs.py small --help ------------ .. raw:: html - python 06_base_configs.py small --seed 94720 + python 10_base_configs.py small --seed 94720 -.. program-output:: python ../../examples/06_base_configs.py small --seed 94720 +.. program-output:: python ../../examples/10_base_configs.py small --seed 94720 ------------ .. raw:: html - python 06_base_configs.py big --help + python 10_base_configs.py big --help -.. program-output:: python ../../examples/06_base_configs.py big --help +.. program-output:: python ../../examples/10_base_configs.py big --help ------------ .. raw:: html - python 06_base_configs.py big --seed 94720 + python 10_base_configs.py big --seed 94720 -.. program-output:: python ../../examples/06_base_configs.py big --seed 94720 +.. program-output:: python ../../examples/10_base_configs.py big --seed 94720 diff --git a/examples/07_literals_and_unions.py b/examples/06_literals_and_unions.py similarity index 96% rename from examples/07_literals_and_unions.py rename to examples/06_literals_and_unions.py index b8231ae11..a46fff65c 100644 --- a/examples/07_literals_and_unions.py +++ b/examples/06_literals_and_unions.py @@ -2,7 +2,7 @@ `typing.Union[]` can be used to restrict inputs to a fixed set of types. Usage: -`python ./07_literals_and_unions.py --help` +`python ./06_literals_and_unions.py --help` """ import dataclasses diff --git a/examples/08_positional_args.py b/examples/07_positional_args.py similarity index 93% rename from examples/08_positional_args.py rename to examples/07_positional_args.py index 713d2aab0..317eccaa1 100644 --- a/examples/08_positional_args.py +++ b/examples/07_positional_args.py @@ -1,8 +1,8 @@ """Positional-only arguments in functions are converted to positional CLI arguments. Usage: -`python ./08_positional_args.py --help` -`python ./08_positional_args.py ./a ./b --optimizer.learning-rate 1e-5` +`python ./07_positional_args.py --help` +`python ./07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5` """ from __future__ import annotations diff --git a/examples/09_subparsers.py b/examples/08_subparsers.py similarity index 73% rename from examples/09_subparsers.py rename to examples/08_subparsers.py index aa0ec8838..d0156da2d 100644 --- a/examples/09_subparsers.py +++ b/examples/08_subparsers.py @@ -1,11 +1,11 @@ """Unions over nested types (classes or dataclasses) are populated using subparsers. Usage: -`python ./09_subparsers.py --help` -`python ./09_subparsers.py cmd:commit --help` -`python ./09_subparsers.py cmd:commit --cmd.message hello --cmd.all` -`python ./09_subparsers.py cmd:checkout --help` -`python ./09_subparsers.py cmd:checkout --cmd.branch main` +`python ./08_subparsers.py --help` +`python ./08_subparsers.py cmd:commit --help` +`python ./08_subparsers.py cmd:commit --cmd.message hello --cmd.all` +`python ./08_subparsers.py cmd:checkout --help` +`python ./08_subparsers.py cmd:checkout --cmd.branch main` """ from __future__ import annotations diff --git a/examples/10_multiple_subparsers.py b/examples/09_multiple_subparsers.py similarity index 84% rename from examples/10_multiple_subparsers.py rename to examples/09_multiple_subparsers.py index fd474660a..512d61df0 100644 --- a/examples/10_multiple_subparsers.py +++ b/examples/09_multiple_subparsers.py @@ -1,10 +1,10 @@ """Multiple unions over nested types are populated using a series of subparsers. Usage: -`python ./10_multiple_subparsers.py` -`python ./10_multiple_subparsers.py --help` -`python ./10_multiple_subparsers.py dataset:mnist --help` -`python ./10_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4` +`python ./09_multiple_subparsers.py` +`python ./09_multiple_subparsers.py --help` +`python ./09_multiple_subparsers.py dataset:mnist --help` +`python ./09_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4` """ from __future__ import annotations diff --git a/examples/06_base_configs.py b/examples/10_base_configs.py similarity index 66% rename from examples/06_base_configs.py rename to examples/10_base_configs.py index 0e721b5f0..98a0cd370 100644 --- a/examples/06_base_configs.py +++ b/examples/10_base_configs.py @@ -9,18 +9,18 @@ avoid fussing with `sys.argv` by using a `BASE_CONFIG` environment variable. Usage: -`python ./06_base_configs.py` -`python ./06_base_configs.py small --help` -`python ./06_base_configs.py small --seed 94720` -`python ./06_base_configs.py big --help` -`python ./06_base_configs.py big --seed 94720` +`python ./10_base_configs.py` +`python ./10_base_configs.py small --help` +`python ./10_base_configs.py small --seed 94720` +`python ./10_base_configs.py big --help` +`python ./10_base_configs.py big --seed 94720` """ -import sys from dataclasses import dataclass -from typing import Callable, Dict, Literal, Tuple, TypeVar, Union +from typing import Callable, Literal, Mapping, Tuple, Type, TypeVar, Union from torch import nn +from typing_extensions import Annotated, reveal_type import dcargs @@ -91,31 +91,12 @@ class ExperimentConfig: } -T = TypeVar("T") - - -def cli_from_base_configs(base_library: Dict[str, T]) -> T: - """Populate an instance of `cls`, where the first positional argument is used to - select from a library of named base configs.""" - # Get base configuration name from the first positional argument. - if len(sys.argv) < 2 or sys.argv[1] not in base_library: - valid_usages = map(lambda k: f"{sys.argv[0]} {k} --help", base_library.keys()) - raise SystemExit("usage:\n " + "\n ".join(valid_usages)) - - # Get base configuration from our library, and use it for default CLI parameters. - default_instance = base_library[sys.argv[1]] - return dcargs.cli( - type(default_instance), - prog=" ".join(sys.argv[:2]), - args=sys.argv[2:], - default_instance=default_instance, +if __name__ == "__main__": + config = dcargs.cli( + dcargs.extras.union_type_from_mapping(base_configs), # `avoid_subparsers` will avoid making a subparser for unions when a default is - # provided; in this case, it simplifies our CLI but makes it less expressive - # (cannot switch away from the base optimizer types). + # provided; it simplifies our CLI but makes it less expressive. avoid_subparsers=True, ) - - -if __name__ == "__main__": - config = cli_from_base_configs(base_configs) + reveal_type(config) # Should ExperimentConfig, both staticaly and dynamically. print(config) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 2160c40b9..eaee76bf5 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -8,7 +8,7 @@ import pytest import torch.nn as nn -from typing_extensions import Literal +from typing_extensions import Annotated, Literal import dcargs import dcargs._argparse_formatter @@ -39,7 +39,7 @@ class Helptext: x: int # Documentation 1 # Documentation 2 - y: int + y: Annotated[int, "ignored"] z: int = 3 """Documentation 3""" @@ -525,9 +525,9 @@ def main(x: Any = Struct()): helptext = _get_helptext(main) assert "--x {fixed}" in helptext - def main(x: Callable = nn.ReLU): + def main2(x: Callable = nn.ReLU): pass - helptext = _get_helptext(main) + helptext = _get_helptext(main2) assert "--x {fixed}" in helptext assert "(fixed to: Date: Tue, 30 Aug 2022 07:22:49 -0700 Subject: [PATCH 110/491] Tests, improvements for generics --- dcargs/_arguments.py | 4 +- dcargs/_calling.py | 4 +- dcargs/_parsers.py | 11 ++- dcargs/_resolver.py | 29 +++++- dcargs/metadata/_subcommands.py | 3 + examples/10_base_configs.py | 6 +- tests/test_nested.py | 162 +++++++++++++++++++++++++++++++ tests/test_union_from_mapping.py | 48 +++++++++ 8 files changed, 250 insertions(+), 17 deletions(-) create mode 100644 tests/test_union_from_mapping.py diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index d10df3c6a..dc9d64bec 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -23,7 +23,7 @@ import termcolor -from . import _fields, _instantiators, _strings +from . import _fields, _instantiators, _resolver, _strings try: # Python >=3.8. @@ -132,7 +132,7 @@ def _rule_handle_boolean_flags( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: - if arg.type_from_typevar.get(arg.field.typ, arg.field.typ) is not bool: # type: ignore + if _resolver.apply_type_from_typevar(arg.field.typ, arg.type_from_typevar) is not bool: # type: ignore return lowered if lowered.default is False and not arg.field.positional: diff --git a/dcargs/_calling.py b/dcargs/_calling.py index cfab0e48e..be534d588 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -56,7 +56,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: prefixed_field_name = _strings.make_field_name([field_name_prefix, field.name]) # Resolve field type. - field_type = type_from_typevar.get(field.typ, field.typ) # type: ignore + field_type = _resolver.apply_type_from_typevar(field.typ, type_from_typevar) # type: ignore if prefixed_field_name in arg_from_prefixed_field_name: assert prefixed_field_name not in consumed_keywords @@ -144,7 +144,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: value = None else: options = map( - lambda x: type_from_typevar.get(x, x), + lambda x: _resolver.apply_type_from_typevar(x, type_from_typevar), get_args(field_type), ) chosen_f = None diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 5277d4a38..0a2642b9c 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -80,9 +80,9 @@ def from_callable( field = dataclasses.replace( field, typ=_resolver.type_from_typevar_constraints( - type_from_typevar.get( # type: ignore - field.typ, + _resolver.apply_type_from_typevar( field.typ, + type_from_typevar, ) ), ) @@ -243,7 +243,10 @@ def from_field( return None # We don't use sets here to retain order of subcommands. - options = [type_from_typevar.get(typ, typ) for typ in get_args(field.typ)] + options = [ + _resolver.apply_type_from_typevar(typ, type_from_typevar) + for typ in get_args(field.typ) + ] options_no_none = [o for o in options if o != type(None)] # noqa if not all( [ @@ -283,7 +286,7 @@ def from_field( subparser = dataclasses.replace( subparser, helptext_from_nested_class_field_name={ - _strings.make_field_name([field.name, k]): v + _strings.make_field_name([prefix, k]): v for k, v in subparser.helptext_from_nested_class_field_name.items() }, ) diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 88e3fe032..cbe9d8ccb 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -1,5 +1,6 @@ """Utilities for resolving types and forward references.""" +import collections.abc import copy import dataclasses from typing import ( @@ -15,7 +16,7 @@ cast, ) -from typing_extensions import get_args, get_origin, get_type_hints +from typing_extensions import Annotated, get_args, get_origin, get_type_hints TypeOrCallable = TypeVar("TypeOrCallable", Type, Callable) @@ -43,13 +44,13 @@ def resolve_generic_types( origin_cls = get_origin(cls) - type_from_typevars = {} + type_from_typevar = {} if origin_cls is not None and hasattr(origin_cls, "__parameters__"): typevars = origin_cls.__parameters__ typevar_values = get_args(cls) assert len(typevars) == len(typevar_values) cls = origin_cls - type_from_typevars.update(dict(zip(typevars, typevar_values))) + type_from_typevar.update(dict(zip(typevars, typevar_values))) if hasattr(cls, "__orig_bases__"): bases = getattr(cls, "__orig_bases__") @@ -59,9 +60,9 @@ def resolve_generic_types( continue typevars = origin_base.__parameters__ typevar_values = get_args(base) - type_from_typevars.update(dict(zip(typevars, typevar_values))) + type_from_typevar.update(dict(zip(typevars, typevar_values))) - return cls, type_from_typevars + return cls, type_from_typevar def resolved_fields(cls: Type) -> List[dataclasses.Field]: @@ -154,3 +155,21 @@ def unwrap_annotated( len(targets) == 1 ), f"Found two instances of {search_type} in metadata, but only expected one." return args[0], targets[0] + + +def apply_type_from_typevar( + typ: TypeOrCallable, type_from_typevar: Dict[TypeVar, Type] +) -> TypeOrCallable: + if typ in type_from_typevar: + return type_from_typevar[typ] # type: ignore + + if len(get_args(typ)) > 0: + args = get_args(typ) + if get_origin(typ) is Annotated: + args = args[:1] + if get_origin(typ) is collections.abc.Callable: + assert isinstance(args[0], list) + args = tuple(args[0]) + args[1:] + return typ.copy_with(tuple(apply_type_from_typevar(x, type_from_typevar) for x in args)) # type: ignore + + return typ diff --git a/dcargs/metadata/_subcommands.py b/dcargs/metadata/_subcommands.py index eee8f2cb5..c36b7f699 100644 --- a/dcargs/metadata/_subcommands.py +++ b/dcargs/metadata/_subcommands.py @@ -13,6 +13,9 @@ class _SubcommandConfiguration: description: Optional[str] default: Any + def __hash__(self) -> int: + return object.__hash__(self) + def subcommand( name: str, diff --git a/examples/10_base_configs.py b/examples/10_base_configs.py index 98a0cd370..d8f67c75a 100644 --- a/examples/10_base_configs.py +++ b/examples/10_base_configs.py @@ -9,7 +9,7 @@ avoid fussing with `sys.argv` by using a `BASE_CONFIG` environment variable. Usage: -`python ./10_base_configs.py` +`python ./10_base_configs.py --help` `python ./10_base_configs.py small --help` `python ./10_base_configs.py small --seed 94720` `python ./10_base_configs.py big --help` @@ -17,10 +17,9 @@ """ from dataclasses import dataclass -from typing import Callable, Literal, Mapping, Tuple, Type, TypeVar, Union +from typing import Callable, Literal, Tuple, Union from torch import nn -from typing_extensions import Annotated, reveal_type import dcargs @@ -98,5 +97,4 @@ class ExperimentConfig: # provided; it simplifies our CLI but makes it less expressive. avoid_subparsers=True, ) - reveal_type(config) # Should ExperimentConfig, both staticaly and dynamically. print(config) diff --git a/tests/test_nested.py b/tests/test_nested.py index ffd366ef4..22cbff7e8 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -761,3 +761,165 @@ def main( assert hash(dcargs.cli(main, args="--x.num-epochs 10".split(" "))) == hash( frozendict({"num_epochs": 10, "batch_size": 64}) ) + + +def test_subparser_in_nested_with_metadata(): + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + @dataclasses.dataclass + class Nested2: + subcommand: Union[ + Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + ] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2 + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_subparser_in_nested_with_metadata_generic(): + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + T = TypeVar("T") + + @dataclasses.dataclass + class Nested2(Generic[T]): + subcommand: T + + @dataclasses.dataclass + class Nested1: + nested2: Nested2[ + Union[ + Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + ] + ] + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_subparser_in_nested_with_metadata_generic_alt(): + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + T = TypeVar("T") + + @dataclasses.dataclass + class Nested2(Generic[T]): + subcommand: Union[ + Annotated[T, dcargs.metadata.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + ] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2[A] + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) diff --git a/tests/test_union_from_mapping.py b/tests/test_union_from_mapping.py new file mode 100644 index 000000000..b82fd3f92 --- /dev/null +++ b/tests/test_union_from_mapping.py @@ -0,0 +1,48 @@ +import dataclasses +from typing import Optional + +import dcargs + + +@dataclasses.dataclass +class A: + x: int + + +def test_union_from_mapping(): + base_configs = { + "one": A(1), + "two": A(2), + "three": A(3), + } + ConfigUnion = dcargs.extras.union_type_from_mapping(base_configs) + + assert dcargs.cli(ConfigUnion, args="one".split(" ")) == A(1) + assert dcargs.cli(ConfigUnion, args="two".split(" ")) == A(2) + assert dcargs.cli(ConfigUnion, args="two --x 4".split(" ")) == A(4) + assert dcargs.cli(ConfigUnion, args="three".split(" ")) == A(3) + + +def test_union_from_mapping_in_function(): + base_configs = { + "one": A(1), + "two": A(2), + "three": A(3), + } + + # Hack for mypy. Not needed for pyright. + ConfigUnion = A + ConfigUnion = dcargs.extras.union_type_from_mapping(base_configs) # type: ignore + + def main(config: ConfigUnion, flag: bool = False) -> Optional[A]: + if flag: + return config + return None + + assert dcargs.cli(main, args="--flag config:one".split(" ")) == A(1) + assert dcargs.cli(main, args="--flag config:one --config.x 3".split(" ")) == A(3) + assert dcargs.cli(main, args="config:one --config.x 1".split(" ")) is None + + assert dcargs.cli(main, args="--flag config:two".split(" ")) == A(2) + assert dcargs.cli(main, args="--flag config:two --config.x 3".split(" ")) == A(3) + assert dcargs.cli(main, args="config:two --config.x 1".split(" ")) is None From f7dd1bdb721628c1c51174f1f6a018993cf531ad Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 31 Aug 2022 17:59:41 -0700 Subject: [PATCH 111/491] Minor stuff for shtab --- dcargs/_arguments.py | 5 ++++- dcargs/_parsers.py | 2 ++ dcargs/extras/_base_configs.py | 13 ++++++++++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index dc9d64bec..a015784ba 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -175,7 +175,10 @@ def _rule_recursive_instantiator_from_type( ) except _instantiators.UnsupportedTypeAnnotationError as e: if arg.field.default in _fields.MISSING_SINGLETONS: - raise e + raise _instantiators.UnsupportedTypeAnnotationError( + "Unsupported type annotation for the field" + f" {_strings.make_field_name([arg.prefix, arg.field.name])}. To suppress this error, assign the field a default value." + ) from e else: # For fields with a default, we'll get by even if there's no instantiator # available. diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 0a2642b9c..e76c715cd 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -397,6 +397,7 @@ def apply( subparser = argparse_subparsers.add_parser( name=_strings.subparser_name_from_type(self.prefix, None), formatter_class=_argparse_formatter.make_formatter_class(0), + help="", ) subparser_tree_nodes.append(subparser) @@ -406,6 +407,7 @@ def apply( formatter_class=_argparse_formatter.make_formatter_class( len(subparser_def.args) ), + help=subparser_def.description, ) subparser_def.apply(subparser) diff --git a/dcargs/extras/_base_configs.py b/dcargs/extras/_base_configs.py index cd2bb5ee2..7c58b84c9 100644 --- a/dcargs/extras/_base_configs.py +++ b/dcargs/extras/_base_configs.py @@ -1,4 +1,4 @@ -from typing import Mapping, Type, TypeVar, Union +from typing import Mapping, Tuple, Type, TypeVar, Union from typing_extensions import Annotated @@ -7,7 +7,9 @@ T = TypeVar("T") -def union_type_from_mapping(base_mapping: Mapping[str, T]) -> Type[T]: +def union_type_from_mapping( + base_mapping: Mapping[Union[str, Tuple[str, str]], T] +) -> Type[T]: """Returns a Union type for defining subcommands that choose between nested types. For example, when `base_mapping` is set to: @@ -68,7 +70,12 @@ def train( return Union.__getitem__( # type: ignore tuple( Annotated.__class_getitem__( # type: ignore - (type(v), subcommand(k, default=v)) + ( + type(v), + subcommand(k, default=v) + if isinstance(k, str) + else subcommand(k[0], default=v, description=k[1]), + ) ) for k, v in base_mapping.items() ) From 6356df9bb5f5d41766d167f6303f788e340a60be Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 31 Aug 2022 18:37:25 -0700 Subject: [PATCH 112/491] Fix bug for defaults set to dataclass types --- dcargs/_fields.py | 6 ++++-- tests/test_dcargs.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 412e84b37..4f9d0cf92 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -530,7 +530,7 @@ def _ensure_dataclass_instance_used_as_default_is_frozen( frozen.""" assert dataclasses.is_dataclass(default_instance) cls = type(default_instance) - if not cls.__dataclass_params__.frozen: + if not cls.__dataclass_params__.frozen: # type: ignore warnings.warn( f"Mutable type {cls} is used as a default value for `{field.name}`. This is" " dangerous! Consider using `dataclasses.field(default_factory=...)` or" @@ -566,7 +566,9 @@ def _get_dataclass_field_default( # Try grabbing default from dataclass field. if field.default not in MISSING_SINGLETONS: default = field.default - if dataclasses.is_dataclass(default): + # Note that dataclasses.is_dataclass() will also return true for dataclass + # _types_, not just instances. + if type(default) is not type and dataclasses.is_dataclass(default): _ensure_dataclass_instance_used_as_default_is_frozen(field, default) return default diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index e01c86b64..6c4740646 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -468,6 +468,19 @@ def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]: dcargs.cli(main, args=["--x", "something"]) +def test_fixed_dataclass_type(): + @dataclasses.dataclass + class Dummy: + pass + + def main(x: Callable = Dummy) -> Callable: + return x + + assert dcargs.cli(main, args=[]) is Dummy + with pytest.raises(SystemExit): + dcargs.cli(main, args=["--x", "something"]) + + def test_missing_singleton(): assert dcargs.MISSING is copy.deepcopy(dcargs.MISSING) From f445c895f420231d1805b3deeecb17d835e54375 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 31 Aug 2022 18:37:25 -0700 Subject: [PATCH 113/491] Fix bug for defaults set to dataclass types --- dcargs/_fields.py | 6 ++++-- tests/test_dcargs.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dcargs/_fields.py b/dcargs/_fields.py index a61de72d0..390192d76 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -539,7 +539,7 @@ def _ensure_dataclass_instance_used_as_default_is_frozen( frozen.""" assert dataclasses.is_dataclass(default_instance) cls = type(default_instance) - if not cls.__dataclass_params__.frozen: + if not cls.__dataclass_params__.frozen: # type: ignore warnings.warn( f"Mutable type {cls} is used as a default value for `{field.name}`. This is" " dangerous! Consider using `dataclasses.field(default_factory=...)` or" @@ -575,7 +575,9 @@ def _get_dataclass_field_default( # Try grabbing default from dataclass field. if field.default not in MISSING_SINGLETONS: default = field.default - if dataclasses.is_dataclass(default): + # Note that dataclasses.is_dataclass() will also return true for dataclass + # _types_, not just instances. + if type(default) is not type and dataclasses.is_dataclass(default): _ensure_dataclass_instance_used_as_default_is_frozen(field, default) return default diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index e01c86b64..6c4740646 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -468,6 +468,19 @@ def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]: dcargs.cli(main, args=["--x", "something"]) +def test_fixed_dataclass_type(): + @dataclasses.dataclass + class Dummy: + pass + + def main(x: Callable = Dummy) -> Callable: + return x + + assert dcargs.cli(main, args=[]) is Dummy + with pytest.raises(SystemExit): + dcargs.cli(main, args=["--x", "something"]) + + def test_missing_singleton(): assert dcargs.MISSING is copy.deepcopy(dcargs.MISSING) From 1e04ba39294ec8bce936f490edb8551fe60482f3 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Sep 2022 00:54:04 -0700 Subject: [PATCH 114/491] Marker implementation: Fixed[], FlagsOff[], SubcommandsOff[], etc --- dcargs/_arguments.py | 37 +++- dcargs/_calling.py | 12 +- dcargs/_cli.py | 15 -- dcargs/_docstrings.py | 3 +- dcargs/_fields.py | 46 ++--- dcargs/_parsers.py | 83 +++++--- dcargs/_resolver.py | 29 ++- dcargs/_singleton.py | 13 ++ dcargs/_strings.py | 8 +- dcargs/extras/__init__.py | 4 +- dcargs/extras/_base_configs.py | 17 +- dcargs/metadata/__init__.py | 19 +- dcargs/metadata/_markers.py | 33 ++++ dcargs/metadata/_subcommands.py | 10 +- examples/10_base_configs.py | 88 ++++++--- tests/test_helptext.py | 4 - tests/test_metadata.py | 330 +++++++++++++++++++++++++++++++ tests/test_nested.py | 215 -------------------- tests/test_union_from_mapping.py | 4 +- 19 files changed, 581 insertions(+), 389 deletions(-) create mode 100644 dcargs/_singleton.py create mode 100644 dcargs/metadata/_markers.py create mode 100644 tests/test_metadata.py diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index a015784ba..a1aca8f89 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -24,6 +24,7 @@ import termcolor from . import _fields, _instantiators, _resolver, _strings +from .metadata import _markers try: # Python >=3.8. @@ -95,9 +96,9 @@ class LoweredArgumentDefinition: def is_fixed(self) -> bool: """If the instantiator is set to `None`, even after all argument - transformations, it means that we weren't able to determine a valid instantiator - for an argument. We then mark the argument as 'fixed', with a value always equal - to the field default.""" + transformations, it means that we don't have a valid instantiator for an + argument. We then mark the argument as 'fixed', with a value always equal to the + field default.""" return self.instantiator is None # From here on out, all fields correspond 1:1 to inputs to argparse's @@ -135,23 +136,32 @@ def _rule_handle_boolean_flags( if _resolver.apply_type_from_typevar(arg.field.typ, arg.type_from_typevar) is not bool: # type: ignore return lowered - if lowered.default is False and not arg.field.positional: + if ( + arg.field.default in _fields.MISSING_SINGLETONS + or arg.field.positional + or _markers.FLAGS_OFF in arg.field.markers + ): + # Treat bools as a normal parameter. + return lowered + elif arg.field.default is False: # Default `False` => --flag passed in flips to `True`. return dataclasses.replace( lowered, action="store_true", instantiator=lambda x: x, # argparse will directly give us a bool! ) - elif lowered.default is True and not arg.field.positional: + elif arg.field.default is True: # Default `True` => --no-flag passed in flips to `False`. return dataclasses.replace( lowered, action="store_false", instantiator=lambda x: x, # argparse will directly give us a bool! ) - else: - # Treat bools as a normal parameter. - return lowered + + assert False, ( + "Expected a boolean as a default for {arg.field.name}, but got" + " {lowered.default}." + ) def _rule_recursive_instantiator_from_type( @@ -166,6 +176,14 @@ def _rule_recursive_instantiator_from_type( Conversions from strings to our desired types happen in the instantiator; this is a bit more flexible, and lets us handle more complex types like enums and multi-type tuples.""" + if _markers.FIXED in arg.field.markers: + return dataclasses.replace( + lowered, + instantiator=None, + metavar=termcolor.colored("{fixed}", color="red"), + required=False, + default=_fields.MISSING_PROP, + ) if lowered.instantiator is not None: return lowered try: @@ -177,7 +195,8 @@ def _rule_recursive_instantiator_from_type( if arg.field.default in _fields.MISSING_SINGLETONS: raise _instantiators.UnsupportedTypeAnnotationError( "Unsupported type annotation for the field" - f" {_strings.make_field_name([arg.prefix, arg.field.name])}. To suppress this error, assign the field a default value." + f" {_strings.make_field_name([arg.prefix, arg.field.name])}. To" + " suppress this error, assign the field a default value." ) from e else: # For fields with a default, we'll get by even if there's no instantiator diff --git a/dcargs/_calling.py b/dcargs/_calling.py index be534d588..d20656cad 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, List, Sequence, Set, Tuple, TypeVar, Union -from typing_extensions import get_args, get_origin +from typing_extensions import get_args from . import _arguments, _fields, _parsers, _resolver, _strings @@ -24,7 +24,6 @@ def call_from_args( default_instance: Union[T, _fields.NonpropagatingMissingType], value_from_prefixed_field_name: Dict[str, Any], field_name_prefix: str, - avoid_subparsers: bool, ) -> Tuple[T, Set[str]]: """Call `f` with arguments specified by a dictionary of values from argparse. @@ -96,8 +95,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: in parser_definition.helptext_from_nested_class_field_name ): # Nested callable. - if get_origin(field_type) is Union: - assert avoid_subparsers + if _resolver.unwrap_origin_strip_extras(field_type) is Union: field_type = type(field.default) value, consumed_keywords_child = call_from_args( field_type, @@ -105,7 +103,6 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: field.default, value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, - avoid_subparsers=avoid_subparsers, ) consumed_keywords |= consumed_keywords_child else: @@ -145,7 +142,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: else: options = map( lambda x: _resolver.apply_type_from_typevar(x, type_from_typevar), - get_args(field_type), + get_args(_resolver.unwrap_annotated(field_type)[0]), ) chosen_f = None for option in options: @@ -162,7 +159,6 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: field.default if type(field.default) is chosen_f else None, value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, - avoid_subparsers=avoid_subparsers, ) consumed_keywords |= consumed_keywords_child @@ -174,7 +170,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: field.name if field.name_override is None else field.name_override ] = value - unwrapped_f = _resolver.unwrap_origin(f) + unwrapped_f = _resolver.unwrap_origin_strip_extras(f) unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore unwrapped_f = _resolver.narrow_type(unwrapped_f, default_instance) if unwrapped_f in (tuple, list, set): diff --git a/dcargs/_cli.py b/dcargs/_cli.py index ee40c5308..1376bce86 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -27,7 +27,6 @@ def cli( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> OutT: ... @@ -40,7 +39,6 @@ def cli( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> OutT: ... @@ -52,7 +50,6 @@ def cli( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> OutT: """Call `f(...)`, with arguments populated from an automatically generated CLI interface. @@ -101,8 +98,6 @@ def cli( if `T` is a dataclass, TypedDict, or NamedTuple. Helpful for merging CLI arguments with values loaded from elsewhere. (for example, a config object loaded from a yaml file) - avoid_subparsers: Avoid creating a subparser when defaults are provided for - unions over nested types. Generates cleaner but less expressive CLIs. Returns: The output of `f(...)`. @@ -114,7 +109,6 @@ def cli( description=description, args=args, default_instance=default_instance, - avoid_subparsers=avoid_subparsers, ) return out @@ -127,7 +121,6 @@ def generate_parser( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> argparse.ArgumentParser: ... @@ -140,7 +133,6 @@ def generate_parser( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> argparse.ArgumentParser: ... @@ -152,7 +144,6 @@ def generate_parser( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> argparse.ArgumentParser: """Returns the argparse parser that would be used under-the-hood if `dcargs.cli()` was called with the same arguments. @@ -166,7 +157,6 @@ def generate_parser( description=description, args=args, default_instance=default_instance, - avoid_subparsers=avoid_subparsers, ) assert isinstance(out, argparse.ArgumentParser) return out @@ -181,7 +171,6 @@ def _cli_impl( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> argparse.ArgumentParser: ... @@ -195,7 +184,6 @@ def _cli_impl( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> OutT: ... @@ -208,7 +196,6 @@ def _cli_impl( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default_instance: Optional[OutT] = None, - avoid_subparsers: bool = False, ) -> Union[OutT, argparse.ArgumentParser]: default_instance_internal: Union[_fields.NonpropagatingMissingType, OutT] = ( _fields.MISSING_NONPROP if default_instance is None else default_instance @@ -239,7 +226,6 @@ def _cli_impl( parent_type_from_typevar=None, # Used for recursive calls. default_instance=default_instance_internal, # Overrides for default values. prefix="", # Used for recursive calls. - avoid_subparsers=avoid_subparsers, ) # Generate parser! @@ -269,7 +255,6 @@ def _cli_impl( default_instance_internal, value_from_prefixed_field_name, field_name_prefix="", - avoid_subparsers=avoid_subparsers, ) except _calling.InstantiationError as e: # Emulate argparse's error behavior when invalid arguments are passed in. diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index dcca98ade..492620a63 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -254,7 +254,8 @@ def get_callable_description(f: Callable) -> str: these docstrings.""" f, _unused = _resolver.resolve_generic_types(f) - if _resolver.unwrap_origin(f) in _callable_description_blocklist: + f = _resolver.unwrap_origin_strip_extras(f) + if f in _callable_description_blocklist: return "" # Note inspect.getdoc() causes some corner cases with TypedDicts. diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 390192d76..6c5169fad 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -17,7 +17,8 @@ import typing_extensions from typing_extensions import get_args, get_type_hints, is_typeddict -from . import _docstrings, _instantiators, _resolver, _strings +from . import _docstrings, _instantiators, _resolver, _singleton, _strings +from .metadata import _markers @dataclasses.dataclass(frozen=True) @@ -27,37 +28,32 @@ class FieldDefinition: default: Any helptext: Optional[str] positional: bool + markers: List[_markers.Marker] = dataclasses.field(default_factory=list) # Override the name in our kwargs. Currently only used for dictionary types when # the key values aren't strings, but in the future could be used whenever the # user-facing argument name doesn't match the keyword expected by our callable. name_override: Optional[Any] = None - -class _Singleton: - # Singleton pattern. - # https://www.python.org/download/releases/2.2/descrintro/#__new__ - def __new__(cls, *args, **kwds): - it = cls.__dict__.get("__it__") - if it is not None: - return it - cls.__it__ = it = object.__new__(cls) - it.init(*args, **kwds) - return it - - def init(self, *args, **kwds): - pass + def __post_init__(self): # + # Auto-populate markers if unset; this is meant to not run when we do + # dataclasses.replace, etc. TODO: the whole markers design, handling of + # Annotated[], etc, should be revisited... + if len(self.markers) == 0: + self.markers.extend( + _resolver.unwrap_annotated(self.typ, _markers.Marker)[1] + ) -class PropagatingMissingType(_Singleton): +class PropagatingMissingType(_singleton.Singleton): pass -class NonpropagatingMissingType(_Singleton): +class NonpropagatingMissingType(_singleton.Singleton): pass -class ExcludeFromCallType(_Singleton): +class ExcludeFromCallType(_singleton.Singleton): pass @@ -147,11 +143,11 @@ def _try_field_list_from_callable( ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: from . import metadata as _metadata - f, subcommand_config = _resolver.unwrap_annotated( + f, found_subcommand_configs = _resolver.unwrap_annotated( f, _metadata._subcommands._SubcommandConfiguration ) - if subcommand_config is not None: - default_instance = subcommand_config.default + if len(found_subcommand_configs) > 0: + default_instance = found_subcommand_configs[0].default # Unwrap generics. f, type_from_typevar = _resolver.resolve_generic_types(f) @@ -165,7 +161,7 @@ def _try_field_list_from_callable( cls = f f = cls.__init__ # type: ignore f_origin: Callable = cls # type: ignore - f_origin = _resolver.unwrap_origin(f) + f_origin = _resolver.unwrap_origin_strip_extras(f) # Try special cases. if cls is not None and is_typeddict(cls): @@ -224,9 +220,9 @@ def _try_field_list_from_callable( return container_fields # General cases. - if (cls is not None and cls in _known_parsable_types) or _resolver.unwrap_origin( - f - ) in _known_parsable_types: + if ( + cls is not None and cls in _known_parsable_types + ) or _resolver.unwrap_origin_strip_extras(f) in _known_parsable_types: return UnsupportedNestedTypeMessage(f"{f} should be parsed directly!") else: return _try_field_list_from_general_callable(f, cls, default_instance) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index e76c715cd..9cca1d718 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -43,11 +43,11 @@ def from_callable( T, _fields.PropagatingMissingType, _fields.NonpropagatingMissingType ], prefix: str, - avoid_subparsers: bool, ) -> ParserSpecification: """Create a parser definition from a callable.""" # Resolve generic types. + markers = _resolver.unwrap_annotated(f, _metadata._markers.Marker)[1] f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) if parent_type_from_typevar is not None: @@ -86,36 +86,56 @@ def from_callable( ) ), ) + field.markers.extend(markers) # TODO: would be nice to avoid this mutation! + if isinstance(field.typ, TypeVar): raise _instantiators.UnsupportedTypeAnnotationError( f"Field {field.name} has an unbound TypeVar: {field.typ}." ) - # (1) Handle Unions over callables; these result in subparsers. + # (1) Handle fields marked as fixed. + if _metadata._markers.FIXED in field.markers: + args.append( + _arguments.ArgumentDefinition( + prefix=prefix, + field=field, + type_from_typevar=type_from_typevar, + ) + ) + continue + + # (2) Handle Unions over callables; these result in subparsers. subparsers_attempt = SubparsersSpecification.from_field( field, type_from_typevar=type_from_typevar, parent_classes=parent_classes, prefix=_strings.make_field_name([prefix, field.name]), - avoid_subparsers=avoid_subparsers, ) if subparsers_attempt is not None: if ( - not avoid_subparsers - # Required subparsers => must create a subparser. - or subparsers_attempt.required + not subparsers_attempt.required + and _metadata._markers.SUBCOMMANDS_OFF in field.markers ): + # Don't make a subparser. + field = dataclasses.replace(field, typ=type(field.default)) + else: subparsers_from_name[ _strings.make_field_name([prefix, subparsers_attempt.name]) ] = subparsers_attempt + + for subparser_def in subparsers_attempt.parser_from_name.values(): + for arg in subparser_def.args: + arg.field.markers.extend(field.markers) continue - else: - field = dataclasses.replace(field, typ=type(field.default)) - # (2) Handle nested callables. + # (3) Handle nested callables. if _fields.is_nested_type(field.typ, field.default): field = dataclasses.replace( - field, typ=_resolver.narrow_type(field.typ, field.default) + field, + typ=_resolver.narrow_type( + field.typ, + field.default, + ), ) nested_parser = ParserSpecification.from_callable( field.typ, @@ -124,10 +144,12 @@ def from_callable( parent_type_from_typevar=type_from_typevar, default_instance=field.default, prefix=_strings.make_field_name([prefix, field.name]), - avoid_subparsers=avoid_subparsers, ) args.extend(nested_parser.args) + for arg in nested_parser.args: + arg.field.markers.extend(field.markers) + # Include nested subparsers. subparsers_from_name.update(nested_parser.subparsers_from_name) @@ -145,10 +167,12 @@ def from_callable( ] = _docstrings.get_callable_description(field.typ) continue - # (3) Handle primitive types. These produce a single argument! + # (4) Handle primitive types. These produce a single argument! args.append( _arguments.ArgumentDefinition( - prefix=prefix, field=field, type_from_typevar=type_from_typevar + prefix=prefix, + field=field, + type_from_typevar=type_from_typevar, ) ) @@ -236,16 +260,16 @@ def from_field( type_from_typevar: Dict[TypeVar, Type], parent_classes: Set[Type], prefix: str, - avoid_subparsers: bool, ) -> Optional[SubparsersSpecification]: # Union of classes should create subparsers. - if get_origin(field.typ) is not Union: + typ = _resolver.unwrap_annotated(field.typ)[0] + if get_origin(typ) is not Union: return None # We don't use sets here to retain order of subcommands. options = [ _resolver.apply_type_from_typevar(typ, type_from_typevar) - for typ in get_args(field.typ) + for typ in get_args(typ) ] options_no_none = [o for o in options if o != type(None)] # noqa if not all( @@ -260,28 +284,31 @@ def from_field( parser_from_name: Dict[str, ParserSpecification] = {} for option in options_no_none: name = _strings.subparser_name_from_type(prefix, option) - option, subcommand_config = _resolver.unwrap_annotated( + option, found_subcommand_configs = _resolver.unwrap_annotated( option, _metadata._subcommands._SubcommandConfiguration ) - if subcommand_config is None: - subcommand_config = _metadata._subcommands._SubcommandConfiguration( - "unused", - description=None, - default=( - field.default - if type(field.default) is _resolver.unwrap_origin(option) - else _fields.MISSING_NONPROP + if len(found_subcommand_configs) == 0: + # Make a dummy subcommand config. + found_subcommand_configs = ( + _metadata._subcommands._SubcommandConfiguration( + "unused", + description=None, + default=( + field.default + if type(field.default) + is _resolver.unwrap_origin_strip_extras(option) + else _fields.MISSING_NONPROP + ), ), ) subparser = ParserSpecification.from_callable( option, - description=subcommand_config.description, + description=found_subcommand_configs[0].description, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, - default_instance=subcommand_config.default, + default_instance=found_subcommand_configs[0].default, prefix=prefix, - avoid_subparsers=avoid_subparsers, ) subparser = dataclasses.replace( subparser, diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index cbe9d8ccb..464896729 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -21,8 +21,9 @@ TypeOrCallable = TypeVar("TypeOrCallable", Type, Callable) -def unwrap_origin(typ: TypeOrCallable) -> TypeOrCallable: - """Returns the origin of typ if it exists. Otherwise, returns typ.""" +def unwrap_origin_strip_extras(typ: TypeOrCallable) -> TypeOrCallable: + """Returns the origin, ignoring typing.Annotated, of typ if it exists. Otherwise, returns typ.""" + # TODO: Annotated[] handling should be revisited... typ, _ = unwrap_annotated(typ) origin = get_origin(typ) if origin is None: @@ -33,7 +34,7 @@ def unwrap_origin(typ: TypeOrCallable) -> TypeOrCallable: def is_dataclass(cls: Union[Type, Callable]) -> bool: """Same as `dataclasses.is_dataclass`, but also handles generic aliases.""" - return dataclasses.is_dataclass(unwrap_origin(cls)) + return dataclasses.is_dataclass(unwrap_origin_strip_extras(cls)) def resolve_generic_types( @@ -55,7 +56,7 @@ def resolve_generic_types( if hasattr(cls, "__orig_bases__"): bases = getattr(cls, "__orig_bases__") for base in bases: - origin_base = unwrap_origin(base) + origin_base = unwrap_origin_strip_extras(base) if origin_base is base or not hasattr(origin_base, "__parameters__"): continue typevars = origin_base.__parameters__ @@ -115,8 +116,12 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: should parse as Cat.""" try: potential_subclass = type(default_instance) - superclass = typ + superclass = unwrap_annotated(typ)[0] if superclass is Any or issubclass(potential_subclass, superclass): # type: ignore + if get_origin(typ) is Annotated: + return Annotated.__class_getitem__( # type: ignore + (potential_subclass,) + get_args(typ)[1:] + ) return cast(TypeT, potential_subclass) except TypeError: pass @@ -128,7 +133,7 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: def unwrap_annotated( typ: TypeOrCallable, search_type: Optional[Type[MetadataType]] = None -) -> Tuple[TypeOrCallable, Optional[MetadataType]]: +) -> Tuple[TypeOrCallable, Tuple[MetadataType, ...]]: """Helper for parsing typing.Annotated types. Examples: @@ -137,24 +142,18 @@ def unwrap_annotated( - Annotated[int, "1"], int => (int, None) """ if not hasattr(typ, "__metadata__"): - return typ, None + return typ, () args = get_args(typ) assert len(args) >= 2 # Don't search for a specific metadata type if `None` is passed in. if search_type is None: - return args[0], None + return args[0], () # Look through metadata for desired metadata type. targets = tuple(x for x in args[1:] if isinstance(x, search_type)) - if len(targets) == 0: - return args[0], None - else: - assert ( - len(targets) == 1 - ), f"Found two instances of {search_type} in metadata, but only expected one." - return args[0], targets[0] + return args[0], targets def apply_type_from_typevar( diff --git a/dcargs/_singleton.py b/dcargs/_singleton.py new file mode 100644 index 000000000..f52efefd1 --- /dev/null +++ b/dcargs/_singleton.py @@ -0,0 +1,13 @@ +class Singleton: + # Singleton pattern. + # https://www.python.org/download/releases/2.2/descrintro/#__new__ + def __new__(cls, *args, **kwds): + it = cls.__dict__.get("__it__") + if it is not None: + return it + cls.__it__ = it = object.__new__(cls) + it.init(*args, **kwds) + return it + + def init(self, *args, **kwds): + pass diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 1cc781eef..7b2ddf149 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -57,13 +57,13 @@ def _subparser_name_from_type(cls: Type) -> str: from .metadata import _subcommands # Prevent circular imports cls, type_from_typevar = _resolver.resolve_generic_types(cls) - cls, subcommand_config = _resolver.unwrap_annotated( + cls, found_subcommand_configs = _resolver.unwrap_annotated( cls, _subcommands._SubcommandConfiguration ) # Subparser name from `dcargs.metadata.subcommand()`. - if subcommand_config is not None: - return subcommand_config.name + if len(found_subcommand_configs) > 0: + return found_subcommand_configs[0].name # Subparser name from class name. if len(type_from_typevar) == 0: @@ -82,7 +82,7 @@ def subparser_name_from_type(prefix: str, cls: Union[Type, None]) -> str: suffix = _subparser_name_from_type(cls) if cls is not None else "None" if len(prefix) == 0: return suffix - return f"{prefix}:{suffix}" + return f"{prefix}:{suffix}".replace("_", "-") @functools.lru_cache(maxsize=None) diff --git a/dcargs/extras/__init__.py b/dcargs/extras/__init__.py index dd38ce865..9671a9571 100644 --- a/dcargs/extras/__init__.py +++ b/dcargs/extras/__init__.py @@ -1,7 +1,7 @@ """The :mod:`dcargs.extras` submodule contains helpers that complement :func:`dcargs.cli()`, but aren't considered part of the core interface.""" -from ._base_configs import union_type_from_mapping +from ._base_configs import subcommand_union_from_mapping from ._serialization import from_yaml, to_yaml -__all__ = ["union_type_from_mapping", "to_yaml", "from_yaml"] +__all__ = ["subcommand_union_from_mapping", "to_yaml", "from_yaml"] diff --git a/dcargs/extras/_base_configs.py b/dcargs/extras/_base_configs.py index 7c58b84c9..3a452888e 100644 --- a/dcargs/extras/_base_configs.py +++ b/dcargs/extras/_base_configs.py @@ -1,4 +1,4 @@ -from typing import Mapping, Tuple, Type, TypeVar, Union +from typing import Mapping, Type, TypeVar, Union from typing_extensions import Annotated @@ -7,8 +7,8 @@ T = TypeVar("T") -def union_type_from_mapping( - base_mapping: Mapping[Union[str, Tuple[str, str]], T] +def subcommand_union_from_mapping( + defaults: Mapping[str, T], descriptions: Mapping[str, str] = {} ) -> Type[T]: """Returns a Union type for defining subcommands that choose between nested types. @@ -43,7 +43,7 @@ def union_type_from_mapping( reveal_type(config) # Should be correct! ``` - Or to generate annotations for functions: + Or to generate annotations for classes and functions: ```python SelectableConfig = union_from_base_mapping(base_mapping) @@ -70,13 +70,8 @@ def train( return Union.__getitem__( # type: ignore tuple( Annotated.__class_getitem__( # type: ignore - ( - type(v), - subcommand(k, default=v) - if isinstance(k, str) - else subcommand(k[0], default=v, description=k[1]), - ) + (type(v), subcommand(k, default=v, description=descriptions.get(k))) ) - for k, v in base_mapping.items() + for k, v in defaults.items() ) ) diff --git a/dcargs/metadata/__init__.py b/dcargs/metadata/__init__.py index 6f2156b91..9324a8086 100644 --- a/dcargs/metadata/__init__.py +++ b/dcargs/metadata/__init__.py @@ -1,16 +1,15 @@ """The :mod:`dcargs.metadata` submodule contains helpers for attaching parsing-specific -metadata to types. For the forseeable future, this is limited to subcommand configuration. +metadata to types. -Features here are supported, but generally contradict the core design ethos of -:func:`dcargs.cli()`. - -As such: -1. Usage of existing functionality should be avoided unless absolutely necessary. -2. Introduction of new functionality should be avoided unless it (a) cannot be - reproduced with standard type annotations and (b) meaningfully improves the - usefulness of the library. +Features here are supported, but generally should be unnecessary. """ +from ._markers import Fixed, FlagsOff, SubcommandsOff from ._subcommands import subcommand -__all__ = ["subcommand"] +__all__ = [ + "Fixed", + "FlagsOff", + "SubcommandsOff", + "subcommand", +] diff --git a/dcargs/metadata/_markers.py b/dcargs/metadata/_markers.py new file mode 100644 index 000000000..0835bad38 --- /dev/null +++ b/dcargs/metadata/_markers.py @@ -0,0 +1,33 @@ +from typing import Type, TypeVar + +from typing_extensions import Annotated + +from .. import _singleton + + +class Marker(_singleton.Singleton): + pass + + +def _make_marker(description: str) -> Marker: + class _InnerMarker(Marker): + def __repr__(self): + return description + + return _InnerMarker() + + +T = TypeVar("T", bound=Type) + +FIXED = _make_marker("Fixed") +Fixed = Annotated[T, FIXED] +"""A type T can be annotated as Fixed[T] if we don't want dcargs to parse it. A default +value should be set instead.""" + +FLAGS_OFF = _make_marker("FlagsOff") +FlagsOff = Annotated[T, FLAGS_OFF] +"""Turn off flag conversion, which.""" + +SUBCOMMANDS_OFF = _make_marker("SubcommandsOff") +SubcommandsOff = Annotated[T, SUBCOMMANDS_OFF] +"""A boolean type can be annotated as NoFlag[bool] to turn off automatic flag generation.""" diff --git a/dcargs/metadata/_subcommands.py b/dcargs/metadata/_subcommands.py index c36b7f699..104e5524d 100644 --- a/dcargs/metadata/_subcommands.py +++ b/dcargs/metadata/_subcommands.py @@ -6,9 +6,6 @@ @dataclasses.dataclass(frozen=True) class _SubcommandConfiguration: - # Things we could potentially add: - # - `description` - # - `avoid_subparsers` name: str description: Optional[str] default: Any @@ -24,9 +21,7 @@ def subcommand( default: Any = MISSING_NONPROP, ) -> Any: """Returns a metadata object for configuring subcommands with `typing.Annotated`. - Use of this function is supported but discouraged unless absolutely necessary. - - --- + This is useful but can make code harder to read, so usage is discouraged. Consider the standard approach for creating subcommands: @@ -36,8 +31,7 @@ def subcommand( ) ``` - This will create two subcommands: nested-type-a and nested-type-b. - + This will create two subcommands: `nested-type-a` and `nested-type-b`. Annotating each type with `dcargs.metadata.subcommand()` allows us to override for each subcommand the (a) name and (b) defaults. diff --git a/examples/10_base_configs.py b/examples/10_base_configs.py index d8f67c75a..c85b72e8d 100644 --- a/examples/10_base_configs.py +++ b/examples/10_base_configs.py @@ -1,9 +1,9 @@ """We can integrate `dcargs.cli()` into common configuration patterns: here, we select -one of multiple possible base configurations, and then use the CLI to either override -(existing) or fill in (missing) values. +one of multiple possible base configurations, create a subcommand for each one, and then +use the CLI to either override (existing) or fill in (missing) values. -Note that our interfaces don't prescribe any of the mechanics used for storing or -choosing between base configurations. A Hydra-style YAML approach could just as easily +Note that our interfaces don't prescribe any of the mechanics used for storing +base configurations. A Hydra-style YAML approach could just as easily be used for the config libary (although we generally prefer to avoid YAMLs; staying in Python is convenient for autocompletion and type checking). For selection, we could also avoid fussing with `sys.argv` by using a `BASE_CONFIG` environment variable. @@ -64,37 +64,61 @@ class ExperimentConfig: # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. -base_configs = { - "small": ExperimentConfig( - dataset="mnist", - optimizer=SgdOptimizer(), - batch_size=2048, - num_layers=4, - units=64, - train_steps=30_000, - # The dcargs.MISSING sentinel allows us to specify that the seed should have no - # default, and needs to be populated from the CLI. - seed=dcargs.MISSING, - activation=nn.ReLU, - ), - "big": ExperimentConfig( - dataset="imagenet-50", - optimizer=AdamOptimizer(), - batch_size=32, - num_layers=8, - units=256, - train_steps=100_000, - seed=dcargs.MISSING, - activation=nn.GELU, - ), -} +descriptions = {} +base_configs = {} + +descriptions["small"] = "Train a smaller model." +base_configs["small"] = ExperimentConfig( + dataset="mnist", + optimizer=SgdOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + # The dcargs.MISSING sentinel allows us to specify that the seed should have no + # default, and needs to be populated from the CLI. + seed=dcargs.MISSING, + activation=nn.ReLU, +) + + +descriptions["big"] = "Train a bigger model." +base_configs["big"] = ExperimentConfig( + dataset="imagenet-50", + optimizer=AdamOptimizer(), + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=dcargs.MISSING, + activation=nn.GELU, +) if __name__ == "__main__": config = dcargs.cli( - dcargs.extras.union_type_from_mapping(base_configs), - # `avoid_subparsers` will avoid making a subparser for unions when a default is - # provided; it simplifies our CLI but makes it less expressive. - avoid_subparsers=True, + dcargs.extras.subcommand_union_from_mapping(base_configs, descriptions), ) + # Note that this is equivalent to: + # + # config = dcargs.cli( + # Union[ + # Annotated[ + # ExperimentConfig, + # dcargs.metadata.subcommand( + # "small", + # default=base_configs["small"], + # description=descriptions["small"], + # ), + # ], + # Annotated[ + # ExperimentConfig, + # dcargs.metadata.subcommand( + # "big", + # default=base_configs["big"], + # description=descriptions["big"], + # ), + # ], + # ] + # ) print(config) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index eaee76bf5..485b70a17 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -104,8 +104,6 @@ def some_method(self) -> None: # noqa class ChildClass(ParentClass): """This docstring should be printed as a description.""" - pass - def main(x: ParentClass = ChildClass(x=5, y=5)) -> Any: return x @@ -127,7 +125,6 @@ def __init__(self, a: int): Args: a (int): Hello world! """ - pass def main_with_docstring(a: Inner) -> None: """main_with_docstring. @@ -138,7 +135,6 @@ def main_with_docstring(a: Inner) -> None: def main_no_docstring(a: Inner) -> None: """main_no_docstring.""" - pass helptext = _get_helptext(main_with_docstring) assert "Documented in function" in helptext and str(Inner.__doc__) not in helptext diff --git a/tests/test_metadata.py b/tests/test_metadata.py new file mode 100644 index 000000000..ca653cb74 --- /dev/null +++ b/tests/test_metadata.py @@ -0,0 +1,330 @@ +import dataclasses +from typing import Generic, TypeVar, Union + +import pytest +from typing_extensions import Annotated + +import dcargs + + +def test_avoid_subparser_with_default_instance(): + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + bc: dcargs.metadata.SubcommandsOff[ + Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + ] + + assert ( + dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], + ) + == dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "--bc.y", "5"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=3) + ), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) + ) + assert ( + dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], + ) + == dcargs.cli( + DefaultInstanceSubparser, + args=["--bc.y", "8"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=7) + ), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + ) + + +def test_avoid_subparser_with_default_instance_recursive(): + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + + assert ( + dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], + ) + == dcargs.cli( + dcargs.metadata.SubcommandsOff[DefaultInstanceSubparser], + args=["--x", "1", "--bc.y", "5"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=3) + ), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) + ) + assert dcargs.cli( + DefaultInstanceSubparser, + args=["bc:default-instance-smtp-server", "--bc.z", "3"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=5) + ), + ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) + assert ( + dcargs.cli( + dcargs.metadata.SubcommandsOff[DefaultInstanceSubparser], + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], + ) + == dcargs.cli( + dcargs.metadata.SubcommandsOff[DefaultInstanceSubparser], + args=["--bc.y", "8"], + default_instance=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=7) + ), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + ) + + +def test_subparser_in_nested_with_metadata(): + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + @dataclasses.dataclass + class Nested2: + subcommand: Union[ + Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + ] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2 + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_subparser_in_nested_with_metadata_generic(): + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + T = TypeVar("T") + + @dataclasses.dataclass + class Nested2(Generic[T]): + subcommand: T + + @dataclasses.dataclass + class Nested1: + nested2: Nested2[ + Union[ + Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + ] + ] + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_subparser_in_nested_with_metadata_generic_alt(): + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + T = TypeVar("T") + + @dataclasses.dataclass + class Nested2(Generic[T]): + subcommand: Union[ + Annotated[T, dcargs.metadata.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + ] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2[A] + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert dcargs.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert dcargs.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_flag(): + """When boolean flags have no default value, they must be explicitly specified.""" + + @dataclasses.dataclass + class A: + x: bool + + assert dcargs.cli( + A, + args=["--x"], + default_instance=A(False), + ) == A(True) + + assert dcargs.cli( + dcargs.metadata.FlagsOff[A], + args=["--x", "True"], + default_instance=A(False), + ) == A(True) + + +def test_fixed(): + """When boolean flags have no default value, they must be explicitly specified.""" + + @dataclasses.dataclass + class A: + x: dcargs.metadata.Fixed[bool] + + assert dcargs.cli( + A, + args=[], + default_instance=A(True), + ) == A(True) + + with pytest.raises(SystemExit): + assert dcargs.cli( + dcargs.metadata.FlagsOff[A], + args=["--x", "True"], + default_instance=A(False), + ) == A(True) + + +def test_fixed_recursive(): + """When boolean flags have no default value, they must be explicitly specified.""" + + @dataclasses.dataclass + class A: + x: bool + + assert dcargs.cli( + A, + args=["--x"], + default_instance=A(False), + ) == A(True) + + with pytest.raises(SystemExit): + assert dcargs.cli( + dcargs.metadata.Fixed[ + dcargs.metadata.FlagsOff[A], + ], + args=["--x", "True"], + default_instance=A(False), + ) == A(True) diff --git a/tests/test_nested.py b/tests/test_nested.py index 22cbff7e8..fc41f76a1 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -306,59 +306,6 @@ class DefaultInstanceSubparser: dcargs.cli(DefaultInstanceSubparser, args=["--x", "1", "c", "--bc.y", "3"]) -def test_avoid_subparser_with_default_instance(): - @dataclasses.dataclass - class DefaultInstanceHTTPServer: - y: int = 0 - - @dataclasses.dataclass - class DefaultInstanceSMTPServer: - z: int = 0 - - @dataclasses.dataclass - class DefaultInstanceSubparser: - x: int - bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] - - assert ( - dcargs.cli( - DefaultInstanceSubparser, - args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], - ) - == dcargs.cli( - DefaultInstanceSubparser, - args=["--x", "1", "--bc.y", "5"], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=3) - ), - avoid_subparsers=True, - ) - == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) - ) - assert dcargs.cli( - DefaultInstanceSubparser, - args=["bc:default-instance-smtp-server", "--bc.z", "3"], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=5) - ), - ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) - assert ( - dcargs.cli( - DefaultInstanceSubparser, - args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], - ) - == dcargs.cli( - DefaultInstanceSubparser, - args=["--bc.y", "8"], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=7) - ), - avoid_subparsers=True, - ) - == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) - ) - - def test_optional_subparser(): @dataclasses.dataclass class OptionalHTTPServer: @@ -761,165 +708,3 @@ def main( assert hash(dcargs.cli(main, args="--x.num-epochs 10".split(" "))) == hash( frozendict({"num_epochs": 10, "batch_size": 64}) ) - - -def test_subparser_in_nested_with_metadata(): - @dataclasses.dataclass - class A: - a: int - - @dataclasses.dataclass - class B: - b: int - a: A = A(5) - - @dataclasses.dataclass - class Nested2: - subcommand: Union[ - Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], - ] - - @dataclasses.dataclass - class Nested1: - nested2: Nested2 - - @dataclasses.dataclass - class Parent: - nested1: Nested1 - - assert dcargs.cli( - Parent, - args="nested1.nested2.subcommand:command-a".split(" "), - ) == Parent(Nested1(Nested2(A(7)))) - assert dcargs.cli( - Parent, - args=( - "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( - " " - ) - ), - ) == Parent(Nested1(Nested2(A(3)))) - - assert dcargs.cli( - Parent, - args="nested1.nested2.subcommand:command-b".split(" "), - ) == Parent(Nested1(Nested2(B(9)))) - assert dcargs.cli( - Parent, - args=( - "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( - " " - ) - ), - ) == Parent(Nested1(Nested2(B(7)))) - - -def test_subparser_in_nested_with_metadata_generic(): - @dataclasses.dataclass - class A: - a: int - - @dataclasses.dataclass - class B: - b: int - a: A = A(5) - - T = TypeVar("T") - - @dataclasses.dataclass - class Nested2(Generic[T]): - subcommand: T - - @dataclasses.dataclass - class Nested1: - nested2: Nested2[ - Union[ - Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], - ] - ] - - @dataclasses.dataclass - class Parent: - nested1: Nested1 - - assert dcargs.cli( - Parent, - args="nested1.nested2.subcommand:command-a".split(" "), - ) == Parent(Nested1(Nested2(A(7)))) - assert dcargs.cli( - Parent, - args=( - "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( - " " - ) - ), - ) == Parent(Nested1(Nested2(A(3)))) - - assert dcargs.cli( - Parent, - args="nested1.nested2.subcommand:command-b".split(" "), - ) == Parent(Nested1(Nested2(B(9)))) - assert dcargs.cli( - Parent, - args=( - "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( - " " - ) - ), - ) == Parent(Nested1(Nested2(B(7)))) - - -def test_subparser_in_nested_with_metadata_generic_alt(): - @dataclasses.dataclass - class A: - a: int - - @dataclasses.dataclass - class B: - b: int - a: A = A(5) - - T = TypeVar("T") - - @dataclasses.dataclass - class Nested2(Generic[T]): - subcommand: Union[ - Annotated[T, dcargs.metadata.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], - ] - - @dataclasses.dataclass - class Nested1: - nested2: Nested2[A] - - @dataclasses.dataclass - class Parent: - nested1: Nested1 - - assert dcargs.cli( - Parent, - args="nested1.nested2.subcommand:command-a".split(" "), - ) == Parent(Nested1(Nested2(A(7)))) - assert dcargs.cli( - Parent, - args=( - "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( - " " - ) - ), - ) == Parent(Nested1(Nested2(A(3)))) - - assert dcargs.cli( - Parent, - args="nested1.nested2.subcommand:command-b".split(" "), - ) == Parent(Nested1(Nested2(B(9)))) - assert dcargs.cli( - Parent, - args=( - "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( - " " - ) - ), - ) == Parent(Nested1(Nested2(B(7)))) diff --git a/tests/test_union_from_mapping.py b/tests/test_union_from_mapping.py index b82fd3f92..14142f4b3 100644 --- a/tests/test_union_from_mapping.py +++ b/tests/test_union_from_mapping.py @@ -15,7 +15,7 @@ def test_union_from_mapping(): "two": A(2), "three": A(3), } - ConfigUnion = dcargs.extras.union_type_from_mapping(base_configs) + ConfigUnion = dcargs.extras.subcommand_union_from_mapping(base_configs) assert dcargs.cli(ConfigUnion, args="one".split(" ")) == A(1) assert dcargs.cli(ConfigUnion, args="two".split(" ")) == A(2) @@ -32,7 +32,7 @@ def test_union_from_mapping_in_function(): # Hack for mypy. Not needed for pyright. ConfigUnion = A - ConfigUnion = dcargs.extras.union_type_from_mapping(base_configs) # type: ignore + ConfigUnion = dcargs.extras.subcommand_union_from_mapping(base_configs) # type: ignore def main(config: ConfigUnion, flag: bool = False) -> Optional[A]: if flag: From bd48a7cc8628325c0e8a71490513bf72a25fa00c Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Sep 2022 01:41:49 -0700 Subject: [PATCH 115/491] `dcargs.metadata` => `dcargs.conf` --- dcargs/__init__.py | 4 +-- dcargs/_arguments.py | 4 +-- dcargs/_fields.py | 7 ++-- dcargs/_parsers.py | 12 +++---- dcargs/_strings.py | 2 +- dcargs/conf/__init__.py | 15 ++++++++ dcargs/conf/_markers.py | 44 +++++++++++++++++++++++ dcargs/{metadata => conf}/_subcommands.py | 0 dcargs/extras/_base_configs.py | 6 ++-- dcargs/metadata/__init__.py | 15 -------- dcargs/metadata/_markers.py | 33 ----------------- docs/source/goals_and_alternatives.md | 2 +- examples/10_base_configs.py | 4 +-- tests/test_metadata.py | 30 ++++++++-------- 14 files changed, 94 insertions(+), 84 deletions(-) create mode 100644 dcargs/conf/__init__.py create mode 100644 dcargs/conf/_markers.py rename dcargs/{metadata => conf}/_subcommands.py (100%) delete mode 100644 dcargs/metadata/__init__.py delete mode 100644 dcargs/metadata/_markers.py diff --git a/dcargs/__init__.py b/dcargs/__init__.py index 0b317d4f1..466b49457 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,11 +1,11 @@ -from . import extras, metadata +from . import conf, extras from ._cli import cli, generate_parser from ._fields import MISSING_PUBLIC as MISSING from ._instantiators import UnsupportedTypeAnnotationError __all__ = [ + "conf", "extras", - "metadata", "cli", "generate_parser", "MISSING", diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index a1aca8f89..4df17a394 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -24,7 +24,7 @@ import termcolor from . import _fields, _instantiators, _resolver, _strings -from .metadata import _markers +from .conf import _markers try: # Python >=3.8. @@ -139,7 +139,7 @@ def _rule_handle_boolean_flags( if ( arg.field.default in _fields.MISSING_SINGLETONS or arg.field.positional - or _markers.FLAGS_OFF in arg.field.markers + or _markers.FLAG_CONVERSION_OFF in arg.field.markers ): # Treat bools as a normal parameter. return lowered diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 6c5169fad..87319bf2a 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -17,8 +17,9 @@ import typing_extensions from typing_extensions import get_args, get_type_hints, is_typeddict +from . import conf # Avoid circular import. from . import _docstrings, _instantiators, _resolver, _singleton, _strings -from .metadata import _markers +from .conf import _markers @dataclasses.dataclass(frozen=True) @@ -141,10 +142,8 @@ def _try_field_list_from_callable( f: Union[Callable, Type], default_instance: _DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - from . import metadata as _metadata - f, found_subcommand_configs = _resolver.unwrap_annotated( - f, _metadata._subcommands._SubcommandConfiguration + f, conf._subcommands._SubcommandConfiguration ) if len(found_subcommand_configs) > 0: default_instance = found_subcommand_configs[0].default diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 9cca1d718..730cd228a 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -17,8 +17,8 @@ _instantiators, _resolver, _strings, + conf, ) -from . import metadata as _metadata T = TypeVar("T") @@ -47,7 +47,7 @@ def from_callable( """Create a parser definition from a callable.""" # Resolve generic types. - markers = _resolver.unwrap_annotated(f, _metadata._markers.Marker)[1] + markers = _resolver.unwrap_annotated(f, conf._markers.Marker)[1] f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) if parent_type_from_typevar is not None: @@ -94,7 +94,7 @@ def from_callable( ) # (1) Handle fields marked as fixed. - if _metadata._markers.FIXED in field.markers: + if conf._markers.FIXED in field.markers: args.append( _arguments.ArgumentDefinition( prefix=prefix, @@ -114,7 +114,7 @@ def from_callable( if subparsers_attempt is not None: if ( not subparsers_attempt.required - and _metadata._markers.SUBCOMMANDS_OFF in field.markers + and conf._markers.AVOID_SUBCOMMANDS in field.markers ): # Don't make a subparser. field = dataclasses.replace(field, typ=type(field.default)) @@ -285,12 +285,12 @@ def from_field( for option in options_no_none: name = _strings.subparser_name_from_type(prefix, option) option, found_subcommand_configs = _resolver.unwrap_annotated( - option, _metadata._subcommands._SubcommandConfiguration + option, conf._subcommands._SubcommandConfiguration ) if len(found_subcommand_configs) == 0: # Make a dummy subcommand config. found_subcommand_configs = ( - _metadata._subcommands._SubcommandConfiguration( + conf._subcommands._SubcommandConfiguration( "unused", description=None, default=( diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 7b2ddf149..4c30c1a26 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -54,7 +54,7 @@ def hyphen_separated_from_camel_case(name: str) -> str: def _subparser_name_from_type(cls: Type) -> str: - from .metadata import _subcommands # Prevent circular imports + from .conf import _subcommands # Prevent circular imports cls, type_from_typevar = _resolver.resolve_generic_types(cls) cls, found_subcommand_configs = _resolver.unwrap_annotated( diff --git a/dcargs/conf/__init__.py b/dcargs/conf/__init__.py new file mode 100644 index 000000000..7415c82e8 --- /dev/null +++ b/dcargs/conf/__init__.py @@ -0,0 +1,15 @@ +"""The :mod:`dcargs.conf` submodule contains helpers for attaching parsing-specific +configuration metadata to types via PEP 593 runtime annotations. + +Features here are supported, but are generally unnecessary and should be used sparingly. +""" + +from ._markers import AvoidSubcommands, Fixed, FlagConversionOff +from ._subcommands import subcommand + +__all__ = [ + "AvoidSubcommands", + "Fixed", + "FlagConversionOff", + "subcommand", +] diff --git a/dcargs/conf/_markers.py b/dcargs/conf/_markers.py new file mode 100644 index 000000000..91fa24d49 --- /dev/null +++ b/dcargs/conf/_markers.py @@ -0,0 +1,44 @@ +from typing import Type, TypeVar + +from typing_extensions import Annotated + +from .. import _singleton + + +class Marker(_singleton.Singleton): + pass + + +def _make_marker(description: str) -> Marker: + class _InnerMarker(Marker): + def __repr__(self): + return description + + return _InnerMarker() + + +# Current design issue: markers are applied recursively to nested structures, but can't +# be unapplied. + +T = TypeVar("T", bound=Type) + +FIXED = _make_marker("Fixed") +Fixed = Annotated[T, FIXED] +"""A type `T` can be annotated as `Fixed[T]` if we don't want dcargs to parse it. A +default value should be set instead.""" + +FLAG_CONVERSION_OFF = _make_marker("FlagConversionOff") +FlagConversionOff = Annotated[T, FLAG_CONVERSION_OFF] +"""Turn off flag conversion for booleans with default values. Instead, types annotated +with `bool` will expect an explicit True or False. + +Can be used directly on boolean annotations, `FlagConversionOff[bool]`, or recursively +applied to nested types.""" + +AVOID_SUBCOMMANDS = _make_marker("AvoidSubcommands") +AvoidSubcommands = Annotated[T, AVOID_SUBCOMMANDS] +"""Avoid creating subcommands when a default is provided for unions over nested types. +This simplifies CLI interfaces, but makes them less expressive. + +Can be used directly on union types, `AvoidSubcommands[Union[...]]`, or recursively +applied to nested types.""" diff --git a/dcargs/metadata/_subcommands.py b/dcargs/conf/_subcommands.py similarity index 100% rename from dcargs/metadata/_subcommands.py rename to dcargs/conf/_subcommands.py diff --git a/dcargs/extras/_base_configs.py b/dcargs/extras/_base_configs.py index 3a452888e..e8576090e 100644 --- a/dcargs/extras/_base_configs.py +++ b/dcargs/extras/_base_configs.py @@ -2,7 +2,7 @@ from typing_extensions import Annotated -from ..metadata import subcommand +from ..conf import subcommand T = TypeVar("T") @@ -27,11 +27,11 @@ def subcommand_union_from_mapping( Union[ Annotated[ Config, - dcargs.metadata.subcommand("small", default=Config(...)) + dcargs.conf.subcommand("small", default=Config(...)) ], Annotated[ Config, - dcargs.metadata.subcommand("big", default=Config(...)) + dcargs.conf.subcommand("big", default=Config(...)) ] ] ``` diff --git a/dcargs/metadata/__init__.py b/dcargs/metadata/__init__.py deleted file mode 100644 index 9324a8086..000000000 --- a/dcargs/metadata/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""The :mod:`dcargs.metadata` submodule contains helpers for attaching parsing-specific -metadata to types. - -Features here are supported, but generally should be unnecessary. -""" - -from ._markers import Fixed, FlagsOff, SubcommandsOff -from ._subcommands import subcommand - -__all__ = [ - "Fixed", - "FlagsOff", - "SubcommandsOff", - "subcommand", -] diff --git a/dcargs/metadata/_markers.py b/dcargs/metadata/_markers.py deleted file mode 100644 index 0835bad38..000000000 --- a/dcargs/metadata/_markers.py +++ /dev/null @@ -1,33 +0,0 @@ -from typing import Type, TypeVar - -from typing_extensions import Annotated - -from .. import _singleton - - -class Marker(_singleton.Singleton): - pass - - -def _make_marker(description: str) -> Marker: - class _InnerMarker(Marker): - def __repr__(self): - return description - - return _InnerMarker() - - -T = TypeVar("T", bound=Type) - -FIXED = _make_marker("Fixed") -Fixed = Annotated[T, FIXED] -"""A type T can be annotated as Fixed[T] if we don't want dcargs to parse it. A default -value should be set instead.""" - -FLAGS_OFF = _make_marker("FlagsOff") -FlagsOff = Annotated[T, FLAGS_OFF] -"""Turn off flag conversion, which.""" - -SUBCOMMANDS_OFF = _make_marker("SubcommandsOff") -SubcommandsOff = Annotated[T, SUBCOMMANDS_OFF] -"""A boolean type can be annotated as NoFlag[bool] to turn off automatic flag generation.""" diff --git a/docs/source/goals_and_alternatives.md b/docs/source/goals_and_alternatives.md index 445903b95..e689b7717 100644 --- a/docs/source/goals_and_alternatives.md +++ b/docs/source/goals_and_alternatives.md @@ -12,7 +12,7 @@ Usage distinctions are the result of two API goals: using the standard `typing.Literal` type, subparsers with `typing.Union` of nested types, and positional arguments with `/`. - In contrast, similar libraries have more expansive APIs (sometimes spanning - dozens of specialized class and functions), and often require + dozens of specialized class and functions), and require more library-specific structures, decorators, or metadata formats for configuring parsing behavior. - **Strict typing.** Any type that can be annotated and unambiguously parsed diff --git a/examples/10_base_configs.py b/examples/10_base_configs.py index c85b72e8d..e87bf945c 100644 --- a/examples/10_base_configs.py +++ b/examples/10_base_configs.py @@ -105,7 +105,7 @@ class ExperimentConfig: # Union[ # Annotated[ # ExperimentConfig, - # dcargs.metadata.subcommand( + # dcargs.conf.subcommand( # "small", # default=base_configs["small"], # description=descriptions["small"], @@ -113,7 +113,7 @@ class ExperimentConfig: # ], # Annotated[ # ExperimentConfig, - # dcargs.metadata.subcommand( + # dcargs.conf.subcommand( # "big", # default=base_configs["big"], # description=descriptions["big"], diff --git a/tests/test_metadata.py b/tests/test_metadata.py index ca653cb74..c00e56ee1 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -19,7 +19,7 @@ class DefaultInstanceSMTPServer: @dataclasses.dataclass class DefaultInstanceSubparser: x: int - bc: dcargs.metadata.SubcommandsOff[ + bc: dcargs.conf.AvoidSubcommands[ Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] ] @@ -73,7 +73,7 @@ class DefaultInstanceSubparser: args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], ) == dcargs.cli( - dcargs.metadata.SubcommandsOff[DefaultInstanceSubparser], + dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "--bc.y", "5"], default_instance=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=3) @@ -90,11 +90,11 @@ class DefaultInstanceSubparser: ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) assert ( dcargs.cli( - dcargs.metadata.SubcommandsOff[DefaultInstanceSubparser], + dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], ) == dcargs.cli( - dcargs.metadata.SubcommandsOff[DefaultInstanceSubparser], + dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--bc.y", "8"], default_instance=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=7) @@ -117,8 +117,8 @@ class B: @dataclasses.dataclass class Nested2: subcommand: Union[ - Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + Annotated[A, dcargs.conf.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.conf.subcommand("command-b", default=B(9))], ] @dataclasses.dataclass @@ -176,8 +176,8 @@ class Nested2(Generic[T]): class Nested1: nested2: Nested2[ Union[ - Annotated[A, dcargs.metadata.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + Annotated[A, dcargs.conf.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.conf.subcommand("command-b", default=B(9))], ] ] @@ -227,8 +227,8 @@ class B: @dataclasses.dataclass class Nested2(Generic[T]): subcommand: Union[ - Annotated[T, dcargs.metadata.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.metadata.subcommand("command-b", default=B(9))], + Annotated[T, dcargs.conf.subcommand("command-a", default=A(7))], + Annotated[B, dcargs.conf.subcommand("command-b", default=B(9))], ] @dataclasses.dataclass @@ -280,7 +280,7 @@ class A: ) == A(True) assert dcargs.cli( - dcargs.metadata.FlagsOff[A], + dcargs.conf.FlagConversionOff[A], args=["--x", "True"], default_instance=A(False), ) == A(True) @@ -291,7 +291,7 @@ def test_fixed(): @dataclasses.dataclass class A: - x: dcargs.metadata.Fixed[bool] + x: dcargs.conf.Fixed[bool] assert dcargs.cli( A, @@ -301,7 +301,7 @@ class A: with pytest.raises(SystemExit): assert dcargs.cli( - dcargs.metadata.FlagsOff[A], + dcargs.conf.FlagConversionOff[A], args=["--x", "True"], default_instance=A(False), ) == A(True) @@ -322,8 +322,8 @@ class A: with pytest.raises(SystemExit): assert dcargs.cli( - dcargs.metadata.Fixed[ - dcargs.metadata.FlagsOff[A], + dcargs.conf.Fixed[ + dcargs.conf.FlagConversionOff[A], ], args=["--x", "True"], default_instance=A(False), From e22247eabb4cfc7ff3bf3c3e8387668fca0ba5cc Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Sep 2022 02:09:52 -0700 Subject: [PATCH 116/491] `default_instance` => `default`, `generate_parser` => `get_parser` --- dcargs/__init__.py | 4 +-- dcargs/_cli.py | 67 ++++++++++++++++++++++++----------- dcargs/_deprecated.py | 1 + dcargs/_parsers.py | 28 ++++++++++----- dcargs/_resolver.py | 4 ++- dcargs/conf/_subcommands.py | 10 +++--- tests/test_dcargs.py | 2 +- tests/test_dict_namedtuple.py | 10 +++--- tests/test_helptext.py | 2 +- tests/test_metadata.py | 26 +++++++------- tests/test_missing.py | 12 +++---- tests/test_nested.py | 42 +++++++++------------- 12 files changed, 119 insertions(+), 89 deletions(-) diff --git a/dcargs/__init__.py b/dcargs/__init__.py index 466b49457..3073b8b20 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,5 +1,5 @@ from . import conf, extras -from ._cli import cli, generate_parser +from ._cli import cli, get_parser from ._fields import MISSING_PUBLIC as MISSING from ._instantiators import UnsupportedTypeAnnotationError @@ -7,7 +7,7 @@ "conf", "extras", "cli", - "generate_parser", + "get_parser", "MISSING", "UnsupportedTypeAnnotationError", ] diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 1376bce86..71da45770 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -1,12 +1,12 @@ """Core public API.""" - import argparse import dataclasses +import warnings from typing import Callable, Optional, Sequence, Type, TypeVar, Union, cast, overload from typing_extensions import Literal, assert_never -from . import _argparse_formatter, _calling, _fields, _parsers, _strings +from . import _argparse_formatter, _calling, _fields, _parsers, _strings, conf OutT = TypeVar("OutT") @@ -26,7 +26,7 @@ def cli( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, ) -> OutT: ... @@ -38,7 +38,7 @@ def cli( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, ) -> OutT: ... @@ -49,7 +49,8 @@ def cli( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, + **deprecated_kwargs, ) -> OutT: """Call `f(...)`, with arguments populated from an automatically generated CLI interface. @@ -94,7 +95,7 @@ def cli( `argparse.ArgumentParser()`. args: If set, parse arguments from a sequence of strings instead of the commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. - default_instance: An instance of `T` to use for default values; only supported + default: An instance of `T` to use for default values; only supported if `T` is a dataclass, TypedDict, or NamedTuple. Helpful for merging CLI arguments with values loaded from elsewhere. (for example, a config object loaded from a yaml file) @@ -108,42 +109,44 @@ def cli( prog=prog, description=description, args=args, - default_instance=default_instance, + default=default, + **deprecated_kwargs, ) return out @overload -def generate_parser( +def get_parser( f: Type[OutT], *, prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, ) -> argparse.ArgumentParser: ... @overload -def generate_parser( +def get_parser( f: Callable[..., OutT], *, prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, ) -> argparse.ArgumentParser: ... -def generate_parser( +def get_parser( f: Union[Type[OutT], Callable[..., OutT]], *, prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, + **deprecated_kwargs, ) -> argparse.ArgumentParser: """Returns the argparse parser that would be used under-the-hood if `dcargs.cli()` was called with the same arguments. @@ -156,7 +159,8 @@ def generate_parser( prog=prog, description=description, args=args, - default_instance=default_instance, + default=default, + **deprecated_kwargs, ) assert isinstance(out, argparse.ArgumentParser) return out @@ -170,7 +174,8 @@ def _cli_impl( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, + **deprecated_kwargs, ) -> argparse.ArgumentParser: ... @@ -183,7 +188,8 @@ def _cli_impl( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, + **deprecated_kwargs, ) -> OutT: ... @@ -195,19 +201,37 @@ def _cli_impl( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default_instance: Optional[OutT] = None, + default: Optional[OutT] = None, + **deprecated_kwargs, ) -> Union[OutT, argparse.ArgumentParser]: + + if "default_instance" in deprecated_kwargs: + warnings.warn( + "`default_instance=` is deprecated! use `default=` instead.", stacklevel=2 + ) + default = deprecated_kwargs["default_instance"] + if deprecated_kwargs.get("avoid_subparsers", False): + f = conf.AvoidSubcommands[f] + warnings.warn( + "`avoid_subparsers=` is deprecated! use `dcargs.conf.AvoidSubparsers[]` instead.", + stacklevel=2, + ) + + # Internally, we distinguish between two concepts: + # - "default", which is used for individual arguments. + # - "default_instance", which is used for _fields_ (which may be broken down into + # one or many arguments, depending on various factors). + # + # This could be revisited. default_instance_internal: Union[_fields.NonpropagatingMissingType, OutT] = ( - _fields.MISSING_NONPROP if default_instance is None else default_instance + _fields.MISSING_NONPROP if default is None else default ) if not _fields.is_nested_type(cast(Type, f), default_instance_internal): dummy_field = cast( dataclasses.Field, dataclasses.field( - default=default_instance - if default_instance is not None - else dataclasses.MISSING + default=default if default is not None else dataclasses.MISSING ), ) f = dataclasses.make_dataclass( @@ -224,6 +248,7 @@ def _cli_impl( description=description, parent_classes=set(), # Used for recursive calls. parent_type_from_typevar=None, # Used for recursive calls. + parent_markers=(), # Used for recursive calls. default_instance=default_instance_internal, # Overrides for default values. prefix="", # Used for recursive calls. ) diff --git a/dcargs/_deprecated.py b/dcargs/_deprecated.py index c879522a8..05e32b46b 100644 --- a/dcargs/_deprecated.py +++ b/dcargs/_deprecated.py @@ -1,2 +1,3 @@ from ._cli import cli as parse # noqa +from ._cli import get_parser as generate_parser # noqa from .extras._serialization import from_yaml, to_yaml # noqa diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 730cd228a..128c53437 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -4,7 +4,19 @@ import argparse import dataclasses import itertools -from typing import Any, Callable, Dict, List, Optional, Set, Type, TypeVar, Union, cast +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, + cast, +) import termcolor from typing_extensions import get_args, get_origin @@ -39,6 +51,7 @@ def from_callable( description: Optional[str], parent_classes: Set[Type], parent_type_from_typevar: Optional[Dict[TypeVar, Type]], + parent_markers: Tuple[conf._markers.Marker, ...], default_instance: Union[ T, _fields.PropagatingMissingType, _fields.NonpropagatingMissingType ], @@ -47,7 +60,9 @@ def from_callable( """Create a parser definition from a callable.""" # Resolve generic types. - markers = _resolver.unwrap_annotated(f, conf._markers.Marker)[1] + markers = ( + parent_markers + _resolver.unwrap_annotated(f, conf._markers.Marker)[1] + ) f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) if parent_type_from_typevar is not None: @@ -122,10 +137,6 @@ def from_callable( subparsers_from_name[ _strings.make_field_name([prefix, subparsers_attempt.name]) ] = subparsers_attempt - - for subparser_def in subparsers_attempt.parser_from_name.values(): - for arg in subparser_def.args: - arg.field.markers.extend(field.markers) continue # (3) Handle nested callables. @@ -142,14 +153,12 @@ def from_callable( description=None, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, + parent_markers=markers, default_instance=field.default, prefix=_strings.make_field_name([prefix, field.name]), ) args.extend(nested_parser.args) - for arg in nested_parser.args: - arg.field.markers.extend(field.markers) - # Include nested subparsers. subparsers_from_name.update(nested_parser.subparsers_from_name) @@ -307,6 +316,7 @@ def from_field( description=found_subcommand_configs[0].description, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, + parent_markers=tuple(field.markers), default_instance=found_subcommand_configs[0].default, prefix=prefix, ) diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 464896729..8848eade0 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -113,7 +113,9 @@ def type_from_typevar_constraints(typ: Union[Type, TypeVar]) -> Union[Type, Type def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: """Type narrowing: if we annotate as Animal but specify a default instance of Cat, we - should parse as Cat.""" + should parse as Cat. + + Note that Union types are intentionally excluded here.""" try: potential_subclass = type(default_instance) superclass = unwrap_annotated(typ)[0] diff --git a/dcargs/conf/_subcommands.py b/dcargs/conf/_subcommands.py index 104e5524d..05f8ebc5d 100644 --- a/dcargs/conf/_subcommands.py +++ b/dcargs/conf/_subcommands.py @@ -7,8 +7,8 @@ @dataclasses.dataclass(frozen=True) class _SubcommandConfiguration: name: str - description: Optional[str] default: Any + description: Optional[str] def __hash__(self) -> int: return object.__hash__(self) @@ -17,8 +17,8 @@ def __hash__(self) -> int: def subcommand( name: str, *, - description: Optional[str] = None, default: Any = MISSING_NONPROP, + description: Optional[str] = None, ) -> Any: """Returns a metadata object for configuring subcommands with `typing.Annotated`. This is useful but can make code harder to read, so usage is discouraged. @@ -34,16 +34,16 @@ def subcommand( This will create two subcommands: `nested-type-a` and `nested-type-b`. Annotating each type with `dcargs.metadata.subcommand()` allows us to override for - each subcommand the (a) name and (b) defaults. + each subcommand the (a) name, (b) defaults, and (c) helptext. ```python dcargs.cli( Union[ Annotated[ - NestedTypeA, subcommand("a", default=NestedTypeA(...)) + NestedTypeA, subcommand("a", default=NestedTypeA(...), description="...") ], Annotated[ - NestedTypeA, subcommand("b", default=NestedTypeA(...)) + NestedTypeA, subcommand("b", default=NestedTypeA(...), description="...") ], ] ) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 6c4740646..656f1aa80 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -40,7 +40,7 @@ class ManyTypes: f: float p: pathlib.Path - assert isinstance(dcargs.generate_parser(ManyTypes), argparse.ArgumentParser) + assert isinstance(dcargs.get_parser(ManyTypes), argparse.ArgumentParser) # We can directly pass a dataclass to `dcargs.cli()`: assert dcargs.cli( diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index d1898e183..85889a26d 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -165,7 +165,7 @@ class NestedTypedDict(TypedDict): dcargs.cli(NestedTypedDict, args=["--x", "1"]) -def test_helptext_and_default_instance_typeddict(): +def test_helptext_and_default_typeddict(): class HelptextTypedDict(TypedDict): """This docstring should be printed as a description.""" @@ -180,7 +180,7 @@ class HelptextTypedDict(TypedDict): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.cli(HelptextTypedDict, default_instance={"z": 3}, args=["--help"]) + dcargs.cli(HelptextTypedDict, default={"z": 3}, args=["--help"]) helptext = dcargs._strings.strip_ansi_sequences(f.getvalue()) assert cast(str, HelptextTypedDict.__doc__) in helptext assert "--x INT" in helptext @@ -254,7 +254,7 @@ class HelptextNamedTupleDefault(NamedTuple): assert "Documentation 3 (default: 3)\n" in helptext -def test_helptext_and_default_instance_namedtuple(): +def test_helptext_and_default_namedtuple(): class HelptextNamedTuple(NamedTuple): """This docstring should be printed as a description.""" @@ -269,7 +269,7 @@ class HelptextNamedTuple(NamedTuple): with pytest.raises(SystemExit): dcargs.cli( HelptextNamedTuple, - default_instance=dcargs.MISSING, + default=dcargs.MISSING, args=[], ) @@ -278,7 +278,7 @@ class HelptextNamedTuple(NamedTuple): with contextlib.redirect_stdout(f): dcargs.cli( HelptextNamedTuple, - default_instance=HelptextNamedTuple( + default=HelptextNamedTuple( x=dcargs.MISSING, y=dcargs.MISSING, z=3, diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 485b70a17..1eae5ea1b 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -25,7 +25,7 @@ def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: target2 = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): with dcargs._argparse_formatter.ansi_context(): - dcargs.generate_parser(f).parse_args(args) + dcargs.get_parser(f).parse_args(args) assert target.getvalue() == target2.getvalue() return dcargs._strings.strip_ansi_sequences(target.getvalue()) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index c00e56ee1..0b5dfcff6 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -7,7 +7,7 @@ import dcargs -def test_avoid_subparser_with_default_instance(): +def test_avoid_subparser_with_default(): @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 @@ -31,7 +31,7 @@ class DefaultInstanceSubparser: == dcargs.cli( DefaultInstanceSubparser, args=["--x", "1", "--bc.y", "5"], - default_instance=DefaultInstanceSubparser( + default=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=3) ), ) @@ -45,7 +45,7 @@ class DefaultInstanceSubparser: == dcargs.cli( DefaultInstanceSubparser, args=["--bc.y", "8"], - default_instance=DefaultInstanceSubparser( + default=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=7) ), ) @@ -53,7 +53,7 @@ class DefaultInstanceSubparser: ) -def test_avoid_subparser_with_default_instance_recursive(): +def test_avoid_subparser_with_default_recursive(): @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 @@ -75,7 +75,7 @@ class DefaultInstanceSubparser: == dcargs.cli( dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "--bc.y", "5"], - default_instance=DefaultInstanceSubparser( + default=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=3) ), ) @@ -84,7 +84,7 @@ class DefaultInstanceSubparser: assert dcargs.cli( DefaultInstanceSubparser, args=["bc:default-instance-smtp-server", "--bc.z", "3"], - default_instance=DefaultInstanceSubparser( + default=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=5) ), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) @@ -96,7 +96,7 @@ class DefaultInstanceSubparser: == dcargs.cli( dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--bc.y", "8"], - default_instance=DefaultInstanceSubparser( + default=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=7) ), ) @@ -276,13 +276,13 @@ class A: assert dcargs.cli( A, args=["--x"], - default_instance=A(False), + default=A(False), ) == A(True) assert dcargs.cli( dcargs.conf.FlagConversionOff[A], args=["--x", "True"], - default_instance=A(False), + default=A(False), ) == A(True) @@ -296,14 +296,14 @@ class A: assert dcargs.cli( A, args=[], - default_instance=A(True), + default=A(True), ) == A(True) with pytest.raises(SystemExit): assert dcargs.cli( dcargs.conf.FlagConversionOff[A], args=["--x", "True"], - default_instance=A(False), + default=A(False), ) == A(True) @@ -317,7 +317,7 @@ class A: assert dcargs.cli( A, args=["--x"], - default_instance=A(False), + default=A(False), ) == A(True) with pytest.raises(SystemExit): @@ -326,5 +326,5 @@ class A: dcargs.conf.FlagConversionOff[A], ], args=["--x", "True"], - default_instance=A(False), + default=A(False), ) == A(True) diff --git a/tests/test_missing.py b/tests/test_missing.py index 74b0eb27c..16cee1687 100644 --- a/tests/test_missing.py +++ b/tests/test_missing.py @@ -29,7 +29,7 @@ class Args2: assert dcargs.cli(Args2, args=["--b", "7"]) == Args2(5, 7, 3) -def test_missing_default_instance(): +def test_missing_default(): @dataclasses.dataclass class Args2: a: int @@ -40,16 +40,16 @@ class Args2: dcargs.cli( Args2, args=[], - default_instance=Args2(5, dcargs.MISSING, 3), + default=Args2(5, dcargs.MISSING, 3), ) assert dcargs.cli( Args2, args=["--b", "7"], - default_instance=Args2(5, dcargs.MISSING, 3), + default=Args2(5, dcargs.MISSING, 3), ) == Args2(5, 7, 3) -def test_missing_nested_default_instance(): +def test_missing_nested_default(): @dataclasses.dataclass class Child: a: int = 5 @@ -64,10 +64,10 @@ class Parent: dcargs.cli( Parent, args=[], - default_instance=Parent(child=dcargs.MISSING), + default=Parent(child=dcargs.MISSING), ) assert dcargs.cli( Parent, args=["--child.a", "5", "--child.b", "7", "--child.c", "3"], - default_instance=Parent(child=dcargs.MISSING), + default=Parent(child=dcargs.MISSING), ) == Parent(Child(5, 7, 3)) diff --git a/tests/test_nested.py b/tests/test_nested.py index fc41f76a1..e69332bc0 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -38,7 +38,7 @@ class Nested: dcargs.cli(Nested, args=["--x", "1"]) -def test_nested_default_instance(): +def test_nested_default(): @dataclasses.dataclass class B: y: int = 1 @@ -48,9 +48,9 @@ class Nested: x: int = 2 b: B = B() - assert dcargs.cli( - Nested, args=[], default_instance=Nested(x=1, b=B(y=2)) - ) == Nested(x=1, b=B(y=2)) + assert dcargs.cli(Nested, args=[], default=Nested(x=1, b=B(y=2))) == Nested( + x=1, b=B(y=2) + ) def test_nested_default(): @@ -66,7 +66,7 @@ class Nested: assert ( Nested(x=1, b=B(y=3)) == dcargs.cli(Nested, args=["--x", "1", "--b.y", "3"]) - == dcargs.cli(Nested, args=[], default_instance=Nested(x=1, b=B(y=3))) + == dcargs.cli(Nested, args=[], default=Nested(x=1, b=B(y=3))) ) assert dcargs.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=3)) @@ -232,7 +232,7 @@ class DefaultSubparser: == dcargs.cli( DefaultSubparser, args=[], - default_instance=DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)), + default=DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)), ) == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)) ) @@ -243,7 +243,7 @@ class DefaultSubparser: dcargs.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) -def test_subparser_with_default_instance(): +def test_subparser_with_default(): @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 @@ -265,25 +265,19 @@ class DefaultInstanceSubparser: == dcargs.cli( DefaultInstanceSubparser, args=[], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=5) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), ) == dcargs.cli( DefaultInstanceSubparser, args=["bc:default-instance-http-server"], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=5) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) ) assert dcargs.cli( DefaultInstanceSubparser, args=["bc:default-instance-smtp-server", "--bc.z", "3"], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=5) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) assert ( dcargs.cli( @@ -293,9 +287,7 @@ class DefaultInstanceSubparser: == dcargs.cli( DefaultInstanceSubparser, args=[], - default_instance=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=8) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) ) @@ -459,7 +451,7 @@ class MultipleSubparsers: dcargs.cli( MultipleSubparsers, args=[], - default_instance=MultipleSubparsers( + default=MultipleSubparsers( Subcommand1(), Subcommand2(), Subcommand3(dcargs.MISSING), @@ -471,7 +463,7 @@ class MultipleSubparsers: args=[ "a:subcommand1", ], - default_instance=MultipleSubparsers( + default=MultipleSubparsers( Subcommand1(), Subcommand2(), Subcommand3(dcargs.MISSING), @@ -481,7 +473,7 @@ class MultipleSubparsers: dcargs.cli( MultipleSubparsers, args=["a:subcommand1", "b:subcommand2"], - default_instance=MultipleSubparsers( + default=MultipleSubparsers( Subcommand1(), Subcommand2(), Subcommand3(dcargs.MISSING), @@ -491,7 +483,7 @@ class MultipleSubparsers: dcargs.cli( MultipleSubparsers, args=["a:subcommand1", "b:subcommand2", "c:subcommand3"], - default_instance=MultipleSubparsers( + default=MultipleSubparsers( Subcommand1(), Subcommand2(), Subcommand3(dcargs.MISSING), @@ -500,7 +492,7 @@ class MultipleSubparsers: assert dcargs.cli( MultipleSubparsers, args=["a:subcommand1", "b:subcommand2", "c:subcommand3", "--c.z", "3"], - default_instance=MultipleSubparsers( + default=MultipleSubparsers( Subcommand1(), Subcommand2(), Subcommand3(dcargs.MISSING), @@ -509,7 +501,7 @@ class MultipleSubparsers: assert dcargs.cli( MultipleSubparsers, args=["a:subcommand1", "b:subcommand2", "c:subcommand2"], - default_instance=MultipleSubparsers( + default=MultipleSubparsers( Subcommand1(), Subcommand2(), Subcommand3(dcargs.MISSING), From 880e90d9d0258c3e3fdbf6d3ceeddbe6384c2dd1 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Sep 2022 02:23:59 -0700 Subject: [PATCH 117/491] Cleanup --- dcargs/_cli.py | 6 +- dcargs/_parsers.py | 123 +++++++++--------- dcargs/conf/_subcommands.py | 2 +- dcargs/extras/_base_configs.py | 12 +- .../{08_subparsers.rst => 08_subcommands.rst} | 24 ++-- ...arsers.rst => 09_multiple_subcommands.rst} | 20 +-- docs/source/examples/10_base_configs.rst | 96 ++++++++------ docs/source/goals_and_alternatives.md | 19 ++- .../{08_subparsers.py => 08_subcommands.py} | 12 +- ...bparsers.py => 09_multiple_subcommands.py} | 10 +- tests/test_dict_namedtuple.py | 2 +- tests/test_metadata.py | 20 +-- tests/test_nested.py | 4 +- 13 files changed, 178 insertions(+), 172 deletions(-) rename docs/source/examples/{08_subparsers.rst => 08_subcommands.rst} (61%) rename docs/source/examples/{09_multiple_subparsers.rst => 09_multiple_subcommands.rst} (72%) rename examples/{08_subparsers.py => 08_subcommands.py} (70%) rename examples/{09_multiple_subparsers.py => 09_multiple_subcommands.py} (80%) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 71da45770..02ce64658 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -204,16 +204,16 @@ def _cli_impl( default: Optional[OutT] = None, **deprecated_kwargs, ) -> Union[OutT, argparse.ArgumentParser]: - if "default_instance" in deprecated_kwargs: warnings.warn( "`default_instance=` is deprecated! use `default=` instead.", stacklevel=2 ) default = deprecated_kwargs["default_instance"] if deprecated_kwargs.get("avoid_subparsers", False): - f = conf.AvoidSubcommands[f] + f = conf.AvoidSubcommands[f] # type: ignore warnings.warn( - "`avoid_subparsers=` is deprecated! use `dcargs.conf.AvoidSubparsers[]` instead.", + "`avoid_subparsers=` is deprecated! use `dcargs.conf.AvoidSubparsers[]`" + " instead.", stacklevel=2, ) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 128c53437..197dc4fbc 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -108,75 +108,70 @@ def from_callable( f"Field {field.name} has an unbound TypeVar: {field.typ}." ) - # (1) Handle fields marked as fixed. - if conf._markers.FIXED in field.markers: - args.append( - _arguments.ArgumentDefinition( - prefix=prefix, - field=field, - type_from_typevar=type_from_typevar, - ) - ) - continue - - # (2) Handle Unions over callables; these result in subparsers. - subparsers_attempt = SubparsersSpecification.from_field( - field, - type_from_typevar=type_from_typevar, - parent_classes=parent_classes, - prefix=_strings.make_field_name([prefix, field.name]), - ) - if subparsers_attempt is not None: - if ( - not subparsers_attempt.required - and conf._markers.AVOID_SUBCOMMANDS in field.markers - ): - # Don't make a subparser. - field = dataclasses.replace(field, typ=type(field.default)) - else: - subparsers_from_name[ - _strings.make_field_name([prefix, subparsers_attempt.name]) - ] = subparsers_attempt - continue - - # (3) Handle nested callables. - if _fields.is_nested_type(field.typ, field.default): - field = dataclasses.replace( + if conf._markers.FIXED not in field.markers: + # (1) Handle Unions over callables; these result in subparsers. + subparsers_attempt = SubparsersSpecification.from_field( field, - typ=_resolver.narrow_type( - field.typ, - field.default, - ), - ) - nested_parser = ParserSpecification.from_callable( - field.typ, - description=None, + type_from_typevar=type_from_typevar, parent_classes=parent_classes, - parent_type_from_typevar=type_from_typevar, - parent_markers=markers, - default_instance=field.default, prefix=_strings.make_field_name([prefix, field.name]), ) - args.extend(nested_parser.args) - - # Include nested subparsers. - subparsers_from_name.update(nested_parser.subparsers_from_name) - - # Include nested strings. - for k, v in nested_parser.helptext_from_nested_class_field_name.items(): - helptext_from_nested_class_field_name[ - _strings.make_field_name([field.name, k]) - ] = v - - if field.helptext is not None: - helptext_from_nested_class_field_name[field.name] = field.helptext - else: - helptext_from_nested_class_field_name[ - field.name - ] = _docstrings.get_callable_description(field.typ) - continue + if subparsers_attempt is not None: + if ( + not subparsers_attempt.required + and conf._markers.AVOID_SUBCOMMANDS in field.markers + ): + # Don't make a subparser. + field = dataclasses.replace(field, typ=type(field.default)) + else: + subparsers_from_name[ + _strings.make_field_name([prefix, subparsers_attempt.name]) + ] = subparsers_attempt + continue + + # (2) Handle nested callables. + if _fields.is_nested_type(field.typ, field.default): + field = dataclasses.replace( + field, + typ=_resolver.narrow_type( + field.typ, + field.default, + ), + ) + nested_parser = ParserSpecification.from_callable( + field.typ, + description=None, + parent_classes=parent_classes, + parent_type_from_typevar=type_from_typevar, + parent_markers=markers, + default_instance=field.default, + prefix=_strings.make_field_name([prefix, field.name]), + ) + args.extend(nested_parser.args) + + # Include nested subparsers. + subparsers_from_name.update(nested_parser.subparsers_from_name) + + # Include nested strings. + for ( + k, + v, + ) in nested_parser.helptext_from_nested_class_field_name.items(): + helptext_from_nested_class_field_name[ + _strings.make_field_name([field.name, k]) + ] = v + + if field.helptext is not None: + helptext_from_nested_class_field_name[ + field.name + ] = field.helptext + else: + helptext_from_nested_class_field_name[ + field.name + ] = _docstrings.get_callable_description(field.typ) + continue - # (4) Handle primitive types. These produce a single argument! + # (3) Handle primitive or fixed types. These produce a single argument! args.append( _arguments.ArgumentDefinition( prefix=prefix, diff --git a/dcargs/conf/_subcommands.py b/dcargs/conf/_subcommands.py index 05f8ebc5d..e5a31957b 100644 --- a/dcargs/conf/_subcommands.py +++ b/dcargs/conf/_subcommands.py @@ -49,4 +49,4 @@ def subcommand( ) ``` """ - return _SubcommandConfiguration(name, description, default) + return _SubcommandConfiguration(name, default, description) diff --git a/dcargs/extras/_base_configs.py b/dcargs/extras/_base_configs.py index e8576090e..1159df5d9 100644 --- a/dcargs/extras/_base_configs.py +++ b/dcargs/extras/_base_configs.py @@ -8,11 +8,11 @@ def subcommand_union_from_mapping( - defaults: Mapping[str, T], descriptions: Mapping[str, str] = {} + default_from_name: Mapping[str, T], descriptions: Mapping[str, str] = {} ) -> Type[T]: """Returns a Union type for defining subcommands that choose between nested types. - For example, when `base_mapping` is set to: + For example, when `default` is set to: ```python { @@ -39,14 +39,14 @@ def subcommand_union_from_mapping( This can be used directly in dcargs.cli: ```python - config = dcargs.cli(union_from_base_mapping(base_mapping)) + config = dcargs.cli(subcommand_union_from_mapping(default_from_name)) reveal_type(config) # Should be correct! ``` Or to generate annotations for classes and functions: ```python - SelectableConfig = union_from_base_mapping(base_mapping) + SelectableConfig = subcommand_union_from_mapping(default_from_name) def train( config: SelectableConfig, @@ -64,7 +64,7 @@ def train( if TYPE_CHECKING: SelectableConfig = ExperimentConfig else: - SelectableConfig = union_from_base_mapping(base_mapping) + SelectableConfig = subcommand_union_from_mapping(base_mapping) ``` """ return Union.__getitem__( # type: ignore @@ -72,6 +72,6 @@ def train( Annotated.__class_getitem__( # type: ignore (type(v), subcommand(k, default=v, description=descriptions.get(k))) ) - for k, v in defaults.items() + for k, v in default_from_name.items() ) ) diff --git a/docs/source/examples/08_subparsers.rst b/docs/source/examples/08_subcommands.rst similarity index 61% rename from docs/source/examples/08_subparsers.rst rename to docs/source/examples/08_subcommands.rst index bc1c20f6c..841b408f7 100644 --- a/docs/source/examples/08_subparsers.rst +++ b/docs/source/examples/08_subcommands.rst @@ -1,11 +1,11 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -8. Subparsers +8. Subcommands ========================================== -Unions over nested types (classes or dataclasses) are populated using subparsers. +Unions over nested types (classes or dataclasses) are populated using subcommands. @@ -49,38 +49,38 @@ Unions over nested types (classes or dataclasses) are populated using subparsers .. raw:: html - python 08_subparsers.py --help + python 08_subcommands.py --help -.. program-output:: python ../../examples/08_subparsers.py --help +.. program-output:: python ../../examples/08_subcommands.py --help ------------ .. raw:: html - python 08_subparsers.py cmd:commit --help + python 08_subcommands.py cmd:commit --help -.. program-output:: python ../../examples/08_subparsers.py cmd:commit --help +.. program-output:: python ../../examples/08_subcommands.py cmd:commit --help ------------ .. raw:: html - python 08_subparsers.py cmd:commit --cmd.message hello --cmd.all + python 08_subcommands.py cmd:commit --cmd.message hello --cmd.all -.. program-output:: python ../../examples/08_subparsers.py cmd:commit --cmd.message hello --cmd.all +.. program-output:: python ../../examples/08_subcommands.py cmd:commit --cmd.message hello --cmd.all ------------ .. raw:: html - python 08_subparsers.py cmd:checkout --help + python 08_subcommands.py cmd:checkout --help -.. program-output:: python ../../examples/08_subparsers.py cmd:checkout --help +.. program-output:: python ../../examples/08_subcommands.py cmd:checkout --help ------------ .. raw:: html - python 08_subparsers.py cmd:checkout --cmd.branch main + python 08_subcommands.py cmd:checkout --cmd.branch main -.. program-output:: python ../../examples/08_subparsers.py cmd:checkout --cmd.branch main +.. program-output:: python ../../examples/08_subcommands.py cmd:checkout --cmd.branch main diff --git a/docs/source/examples/09_multiple_subparsers.rst b/docs/source/examples/09_multiple_subcommands.rst similarity index 72% rename from docs/source/examples/09_multiple_subparsers.rst rename to docs/source/examples/09_multiple_subcommands.rst index 0af826b7e..bf44d9356 100644 --- a/docs/source/examples/09_multiple_subparsers.rst +++ b/docs/source/examples/09_multiple_subcommands.rst @@ -1,11 +1,11 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -9. Multiple Subparsers +9. Multiple Subcommands ========================================== -Multiple unions over nested types are populated using a series of subparsers. +Multiple unions over nested types are populated using a series of subcommands. @@ -75,30 +75,30 @@ Multiple unions over nested types are populated using a series of subparsers. .. raw:: html - python 09_multiple_subparsers.py + python 09_multiple_subcommands.py -.. program-output:: python ../../examples/09_multiple_subparsers.py +.. program-output:: python ../../examples/09_multiple_subcommands.py ------------ .. raw:: html - python 09_multiple_subparsers.py --help + python 09_multiple_subcommands.py --help -.. program-output:: python ../../examples/09_multiple_subparsers.py --help +.. program-output:: python ../../examples/09_multiple_subcommands.py --help ------------ .. raw:: html - python 09_multiple_subparsers.py dataset:mnist --help + python 09_multiple_subcommands.py dataset:mnist --help -.. program-output:: python ../../examples/09_multiple_subparsers.py dataset:mnist --help +.. program-output:: python ../../examples/09_multiple_subcommands.py dataset:mnist --help ------------ .. raw:: html - python 09_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 + python 09_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 -.. program-output:: python ../../examples/09_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 +.. program-output:: python ../../examples/09_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 diff --git a/docs/source/examples/10_base_configs.rst b/docs/source/examples/10_base_configs.rst index a78b030c3..5e1a3b700 100644 --- a/docs/source/examples/10_base_configs.rst +++ b/docs/source/examples/10_base_configs.rst @@ -6,11 +6,11 @@ We can integrate ``dcargs.cli()`` into common configuration patterns: here, we select -one of multiple possible base configurations, and then use the CLI to either override -(existing) or fill in (missing) values. +one of multiple possible base configurations, create a subcommand for each one, and then +use the CLI to either override (existing) or fill in (missing) values. -Note that our interfaces don't prescribe any of the mechanics used for storing or -choosing between base configurations. A Hydra-style YAML approach could just as easily +Note that our interfaces don't prescribe any of the mechanics used for storing +base configurations. A Hydra-style YAML approach could just as easily be used for the config libary (although we generally prefer to avoid YAMLs; staying in Python is convenient for autocompletion and type checking). For selection, we could also avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. @@ -21,10 +21,9 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. :linenos: from dataclasses import dataclass - from typing import Callable, Literal, Mapping, Tuple, Type, TypeVar, Union + from typing import Callable, Literal, Tuple, Union from torch import nn - from typing_extensions import Annotated, reveal_type import dcargs @@ -69,49 +68,72 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. - base_configs = { - "small": ExperimentConfig( - dataset="mnist", - optimizer=SgdOptimizer(), - batch_size=2048, - num_layers=4, - units=64, - train_steps=30_000, - # The dcargs.MISSING sentinel allows us to specify that the seed should have no - # default, and needs to be populated from the CLI. - seed=dcargs.MISSING, - activation=nn.ReLU, - ), - "big": ExperimentConfig( - dataset="imagenet-50", - optimizer=AdamOptimizer(), - batch_size=32, - num_layers=8, - units=256, - train_steps=100_000, - seed=dcargs.MISSING, - activation=nn.GELU, - ), - } + descriptions = {} + base_configs = {} + + descriptions["small"] = "Train a smaller model." + base_configs["small"] = ExperimentConfig( + dataset="mnist", + optimizer=SgdOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + # The dcargs.MISSING sentinel allows us to specify that the seed should have no + # default, and needs to be populated from the CLI. + seed=dcargs.MISSING, + activation=nn.ReLU, + ) + + + descriptions["big"] = "Train a bigger model." + base_configs["big"] = ExperimentConfig( + dataset="imagenet-50", + optimizer=AdamOptimizer(), + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=dcargs.MISSING, + activation=nn.GELU, + ) if __name__ == "__main__": config = dcargs.cli( - dcargs.extras.union_type_from_mapping(base_configs), - # `avoid_subparsers` will avoid making a subparser for unions when a default is - # provided; it simplifies our CLI but makes it less expressive. - avoid_subparsers=True, + dcargs.extras.subcommand_union_from_mapping(base_configs, descriptions), ) - reveal_type(config) # Should ExperimentConfig, both staticaly and dynamically. + # Note that this is equivalent to: + # + # config = dcargs.cli( + # Union[ + # Annotated[ + # ExperimentConfig, + # dcargs.conf.subcommand( + # "small", + # default=base_configs["small"], + # description=descriptions["small"], + # ), + # ], + # Annotated[ + # ExperimentConfig, + # dcargs.conf.subcommand( + # "big", + # default=base_configs["big"], + # description=descriptions["big"], + # ), + # ], + # ] + # ) print(config) ------------ .. raw:: html - python 10_base_configs.py + python 10_base_configs.py --help -.. program-output:: python ../../examples/10_base_configs.py +.. program-output:: python ../../examples/10_base_configs.py --help ------------ diff --git a/docs/source/goals_and_alternatives.md b/docs/source/goals_and_alternatives.md index e689b7717..e1667ac85 100644 --- a/docs/source/goals_and_alternatives.md +++ b/docs/source/goals_and_alternatives.md @@ -6,18 +6,17 @@ The core functionality of `dcargs` — generating argument parsers from type annotations — overlaps significantly with features offered by other libraries. Usage distinctions are the result of two API goals: -- **One uninvasive function.** Whenever possible, learning to use `dcargs` - should reduce to learning to write (type-annotated) Python. For example, types - are specified using standard annotations, helptext using docstrings, choices - using the standard `typing.Literal` type, subparsers with `typing.Union` of - nested types, and positional arguments with `/`. - - In contrast, similar libraries have more expansive APIs (sometimes spanning - dozens of specialized class and functions), and require more +- **One uninvasive function.** For all core functionality, learning to use + `dcargs` should reduce to learning to write (type-annotated) Python. For + example, types are specified using standard annotations, helptext using + docstrings, choices using the standard `typing.Literal` type, subcommands with + `typing.Union` of nested types, and positional arguments with `/`. + - In contrast, similar libraries have more expansive APIs , and require more library-specific structures, decorators, or metadata formats for configuring parsing behavior. - **Strict typing.** Any type that can be annotated and unambiguously parsed with an `argparse`-style CLI interface should work out-of-the-box; any public - API that isn't statically analyzable shouldn't be implemented. + API that isn't statically analyzable should be avoided. - In contrast, many similar libraries implement features that depend on dynamic argparse-style namespaces, or string-based accessors that can't be statically checked. @@ -56,10 +55,10 @@ More concretely, we can also compare specific features. A noncomprehensive set: [omegaconf]: https://omegaconf.readthedocs.io/en/2.1_branch/structured_config.html [^datargs_unions_nested]: One allowed per class. -[^tap_unions_nested]: Not supported, but API exists for creating subparsers that accomplish a similar goal. +[^tap_unions_nested]: Not supported, but API exists for creating subcommands that accomplish a similar goal. [^simp_unions_nested]: One allowed per class. [^yahp_unions_nested]: Not supported, but similar functionality available via ["registries"](https://docs.mosaicml.com/projects/yahp/en/stable/examples/registry.html). -[^typer_unions_nested]: Not supported, but API exists for creating subparsers that accomplish a similar goal. +[^typer_unions_nested]: Not supported, but API exists for creating subcommands that accomplish a similar goal. [^simp_literals]: Not supported for mixed (eg `Literal[5, "five"]`) or in container (eg `List[Literal[1, 2]]`) types. [^datargs_literals]: Not supported for mixed types (eg `Literal[5, "five"]`). [^typer_containers]: `typer` uses positional arguments for all required fields, which means that only one variable-length argument (such as `List[int]`) without a default is supported per argument parser. diff --git a/examples/08_subparsers.py b/examples/08_subcommands.py similarity index 70% rename from examples/08_subparsers.py rename to examples/08_subcommands.py index d0156da2d..dba66b898 100644 --- a/examples/08_subparsers.py +++ b/examples/08_subcommands.py @@ -1,11 +1,11 @@ -"""Unions over nested types (classes or dataclasses) are populated using subparsers. +"""Unions over nested types (classes or dataclasses) are populated using subcommands. Usage: -`python ./08_subparsers.py --help` -`python ./08_subparsers.py cmd:commit --help` -`python ./08_subparsers.py cmd:commit --cmd.message hello --cmd.all` -`python ./08_subparsers.py cmd:checkout --help` -`python ./08_subparsers.py cmd:checkout --cmd.branch main` +`python ./08_subcommands.py --help` +`python ./08_subcommands.py cmd:commit --help` +`python ./08_subcommands.py cmd:commit --cmd.message hello --cmd.all` +`python ./08_subcommands.py cmd:checkout --help` +`python ./08_subcommands.py cmd:checkout --cmd.branch main` """ from __future__ import annotations diff --git a/examples/09_multiple_subparsers.py b/examples/09_multiple_subcommands.py similarity index 80% rename from examples/09_multiple_subparsers.py rename to examples/09_multiple_subcommands.py index 512d61df0..cede20a18 100644 --- a/examples/09_multiple_subparsers.py +++ b/examples/09_multiple_subcommands.py @@ -1,10 +1,10 @@ -"""Multiple unions over nested types are populated using a series of subparsers. +"""Multiple unions over nested types are populated using a series of subcommands. Usage: -`python ./09_multiple_subparsers.py` -`python ./09_multiple_subparsers.py --help` -`python ./09_multiple_subparsers.py dataset:mnist --help` -`python ./09_multiple_subparsers.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4` +`python ./09_multiple_subcommands.py` +`python ./09_multiple_subcommands.py --help` +`python ./09_multiple_subcommands.py dataset:mnist --help` +`python ./09_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4` """ from __future__ import annotations diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index 85889a26d..c129a4241 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -254,7 +254,7 @@ class HelptextNamedTupleDefault(NamedTuple): assert "Documentation 3 (default: 3)\n" in helptext -def test_helptext_and_default_namedtuple(): +def test_helptext_and_default_namedtuple_alternate(): class HelptextNamedTuple(NamedTuple): """This docstring should be printed as a description.""" diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 0b5dfcff6..b7c8b3941 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -31,9 +31,7 @@ class DefaultInstanceSubparser: == dcargs.cli( DefaultInstanceSubparser, args=["--x", "1", "--bc.y", "5"], - default=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=3) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) ) @@ -45,9 +43,7 @@ class DefaultInstanceSubparser: == dcargs.cli( DefaultInstanceSubparser, args=["--bc.y", "8"], - default=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=7) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) ) @@ -75,18 +71,14 @@ class DefaultInstanceSubparser: == dcargs.cli( dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "--bc.y", "5"], - default=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=3) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) ) assert dcargs.cli( DefaultInstanceSubparser, args=["bc:default-instance-smtp-server", "--bc.z", "3"], - default=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=5) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) assert ( dcargs.cli( @@ -96,9 +88,7 @@ class DefaultInstanceSubparser: == dcargs.cli( dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--bc.y", "8"], - default=DefaultInstanceSubparser( - x=1, bc=DefaultInstanceHTTPServer(y=7) - ), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) ) diff --git a/tests/test_nested.py b/tests/test_nested.py index e69332bc0..0bdbf4436 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -53,7 +53,7 @@ class Nested: ) -def test_nested_default(): +def test_nested_default_alternate(): @dataclasses.dataclass class B: y: int = 3 @@ -243,7 +243,7 @@ class DefaultSubparser: dcargs.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) -def test_subparser_with_default(): +def test_subparser_with_default_alternate(): @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 From af4cf7afbf3ace0f4e6ed922c98c2019e5a2dcf2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Sep 2022 04:18:15 -0700 Subject: [PATCH 118/491] Strip colors from get_parser(), wrapping tweaks --- dcargs/_argparse_formatter.py | 44 +++++++++++++++++++++++++++++++++++ dcargs/_cli.py | 23 ++++++++++-------- dcargs/_parsers.py | 4 ++-- tests/test_helptext.py | 4 ++-- 4 files changed, 61 insertions(+), 14 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index bfc302926..0322cc928 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -5,6 +5,8 @@ import shutil from typing import Any, ContextManager, Generator +import termcolor + from . import _strings @@ -15,6 +17,22 @@ def monkeypatch_len(obj: Any) -> int: return len(obj) +def dummy_termcolor_context() -> ContextManager[None]: + """Context for turning termcolor off.""" + + def dummy_colored(*args, **kwargs) -> str: + return args[0] + + @contextlib.contextmanager + def inner() -> Generator[None, None, None]: + orig_colored = termcolor.colored + termcolor.colored = dummy_colored + yield + termcolor.colored = orig_colored + + return inner() + + def ansi_context() -> ContextManager[None]: """Context for working with ANSI codes + argparse: - Applies a temporary monkey patch for making argparse ignore ANSI codes when @@ -25,6 +43,7 @@ def ansi_context() -> ContextManager[None]: @contextlib.contextmanager def inner() -> Generator[None, None, None]: if not hasattr(argparse, "len"): + # Sketchy, but seems to work. argparse.len = monkeypatch_len # type: ignore try: # Use Colorama to support coloring in Windows shells. @@ -136,6 +155,7 @@ def _format_action(self, action): ) ) # + parts.append("%*s%s\n" % (indent_first, "", help_lines[0])) # type: ignore for line in help_lines[1:]: parts.append("%*s%s\n" % (help_position, "", line)) @@ -150,3 +170,27 @@ def _format_action(self, action): # return a single string return self._join_parts(parts) + + def _split_lines(self, text, width): + text = self._whitespace_matcher.sub(" ", text).strip() + # The textwrap module is used only for formatting help. + # Delay its import for speeding up the common usage of argparse. + import textwrap as textwrap + + # Sketchy, but seems to work. + textwrap.len = monkeypatch_len # type: ignore + out = textwrap.wrap(text, width) + del textwrap.len # type: ignore + return out + + def _fill_text(self, text, width, indent): + text = self._whitespace_matcher.sub(" ", text).strip() + import textwrap as textwrap + + # Sketchy, but seems to work. + textwrap.len = monkeypatch_len # type: ignore + out = textwrap.fill( + text, width, initial_indent=indent, subsequent_indent=indent + ) + del textwrap.len # type: ignore + return out diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 02ce64658..8849d413a 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -152,16 +152,19 @@ def get_parser( was called with the same arguments. This can be useful for libraries like argcomplete, pyzshcomplete, or shtab, which - enable autocompletion for argparse parsers.""" - out = _cli_impl( - "parser", - f, - prog=prog, - description=description, - args=args, - default=default, - **deprecated_kwargs, - ) + enable autocompletion for argparse parsers. + + To reduce compatibility issues, strips out color formatting.""" + with _argparse_formatter.dummy_termcolor_context(): + out = _cli_impl( + "parser", + f, + prog=prog, + description=description, + args=args, + default=default, + **deprecated_kwargs, + ) assert isinstance(out, argparse.ArgumentParser) return out diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 197dc4fbc..b09a7b112 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -346,8 +346,8 @@ def from_field( prefix, type(field.default) ) assert default_name in parser_from_name, ( - "Default with type {type(field.default)} was passed in, but no matching" - " subparser." + f"Default with type {type(field.default)} was passed in, but no" + " matching subparser." ) default_parser = parser_from_name[default_name] if any(map(lambda arg: arg.lowered.required, default_parser.args)): diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 1eae5ea1b..56ad2963f 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -26,9 +26,9 @@ def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): with dcargs._argparse_formatter.ansi_context(): dcargs.get_parser(f).parse_args(args) - assert target.getvalue() == target2.getvalue() + assert dcargs._strings.strip_ansi_sequences(target.getvalue()) == target2.getvalue() - return dcargs._strings.strip_ansi_sequences(target.getvalue()) + return target2.getvalue() def test_helptext(): From 40e01c743b76801894deae2f56860cf7b7ed2065 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Sep 2022 15:14:17 -0700 Subject: [PATCH 119/491] Fix #7, bump version --- dcargs/extras/_serialization.py | 11 +++++-- pyproject.toml | 2 +- tests/test_generics_and_serialization.py | 41 ++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/dcargs/extras/_serialization.py b/dcargs/extras/_serialization.py index c4ba73a46..c5c433a4e 100644 --- a/dcargs/extras/_serialization.py +++ b/dcargs/extras/_serialization.py @@ -2,10 +2,11 @@ import dataclasses import enum +import functools from typing import IO, Any, Optional, Set, Type, TypeVar, Union import yaml -from typing_extensions import get_origin +from typing_extensions import get_args, get_origin from .. import _fields, _resolver @@ -48,15 +49,21 @@ def _get_contained_special_types_from_type( contained_dataclasses = {cls} def handle_type(typ) -> Set[Type]: + print(typ) + # Handle dataclasses. if _resolver.is_dataclass(typ) and typ not in parent_contained_dataclasses: return _get_contained_special_types_from_type( typ, _parent_contained_dataclasses=contained_dataclasses | parent_contained_dataclasses, ) + + # Handle enums. elif type(typ) is enum.EnumMeta: return {typ} - return set() + + # Handle Union, Annotated, List, etc. No-op when there are no args. + return functools.reduce(set.union, map(handle_type, get_args(typ)), set()) # Handle generics. for typ in type_from_typevar.values(): diff --git a/pyproject.toml b/pyproject.toml index 7dd66399e..cda21fa86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.2.6" +version = "0.2.7" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 0ac23f7cf..c8d4e65ff 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -5,6 +5,7 @@ from typing import Generic, List, Tuple, Type, TypeVar, Union import pytest +from typing_extensions import Annotated import dcargs @@ -346,3 +347,43 @@ def main(x: ActualParentClass[int] = ChildClass(5, 5)) -> ActualParentClass: return x assert dcargs.cli(main, args="--x.x 3".split(" ")) == ChildClass(3, 5, 3) + + +def test_pculbertson(): + # https://github.com/brentyi/dcargs/issues/7 + from typing import Union + + @dataclasses.dataclass + class TypeA: + data: int + + @dataclasses.dataclass + class TypeB: + data: int + + @dataclasses.dataclass + class Wrapper: + subclass: Union[TypeA, TypeB] = TypeA(1) + + wrapper1 = Wrapper() # Create Wrapper object. + wrapper2 = dcargs.extras.from_yaml( + Wrapper, dcargs.extras.to_yaml(wrapper1) + ) # Errors, no constructor for TypeA + + +def test_annotated(): + # https://github.com/brentyi/dcargs/issues/7 + from typing import Union + + @dataclasses.dataclass + class TypeA: + data: int + + @dataclasses.dataclass + class Wrapper: + subclass: Annotated[int, TypeA] = TypeA(1) + + wrapper1 = Wrapper() # Create Wrapper object. + wrapper2 = dcargs.extras.from_yaml( + Wrapper, dcargs.extras.to_yaml(wrapper1) + ) # Errors, no constructor for TypeA From 7924ff9f174c5960952b4dc1de5bac25a18b694b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Sep 2022 15:22:40 -0700 Subject: [PATCH 120/491] Fix broken test --- tests/test_generics_and_serialization.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index c8d4e65ff..43a4d8950 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -366,14 +366,13 @@ class Wrapper: subclass: Union[TypeA, TypeB] = TypeA(1) wrapper1 = Wrapper() # Create Wrapper object. - wrapper2 = dcargs.extras.from_yaml( + dcargs.extras.from_yaml( Wrapper, dcargs.extras.to_yaml(wrapper1) ) # Errors, no constructor for TypeA def test_annotated(): # https://github.com/brentyi/dcargs/issues/7 - from typing import Union @dataclasses.dataclass class TypeA: @@ -381,9 +380,9 @@ class TypeA: @dataclasses.dataclass class Wrapper: - subclass: Annotated[int, TypeA] = TypeA(1) + subclass: Annotated[TypeA, int] = TypeA(1) wrapper1 = Wrapper() # Create Wrapper object. - wrapper2 = dcargs.extras.from_yaml( + dcargs.extras.from_yaml( Wrapper, dcargs.extras.to_yaml(wrapper1) ) # Errors, no constructor for TypeA From 767e4dfbc1635d0928ae177c569c842c2a0db72f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 2 Sep 2022 14:54:53 -0700 Subject: [PATCH 121/491] Backport subparser fix from #5; add test from #9 --- dcargs/_parsers.py | 14 ++++++++++++-- pyproject.toml | 2 +- tests/test_nested.py | 27 +++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 46d6965c1..815bd187e 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -255,8 +255,8 @@ def from_field( # Add subparser for each option. parser_from_name: Dict[str, ParserSpecification] = {} for option in options_no_none: - subparser_name = _strings.subparser_name_from_type(prefix, option) - parser_from_name[subparser_name] = ParserSpecification.from_callable( + name = _strings.subparser_name_from_type(prefix, option) + subparser = ParserSpecification.from_callable( option, description=None, parent_classes=parent_classes, @@ -268,6 +268,16 @@ def from_field( avoid_subparsers=avoid_subparsers, ) + # Apply prefix to helptext in nested classes in subparsers. + subparser = dataclasses.replace( + subparser, + helptext_from_nested_class_field_name={ + _strings.make_field_name([prefix, k]): v + for k, v in subparser.helptext_from_nested_class_field_name.items() + }, + ) + parser_from_name[name] = subparser + # Optional if: type hint is Optional[], or a default instance is provided. required = True if field.default not in _fields.MISSING_SINGLETONS: diff --git a/pyproject.toml b/pyproject.toml index cda21fa86..bd329a319 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.2.7" +version = "0.2.8" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] diff --git a/tests/test_nested.py b/tests/test_nested.py index eae4840da..b52f25be5 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -745,3 +745,30 @@ def main( assert hash(dcargs.cli(main, args="--x.num-epochs 10".split(" "))) == hash( frozendict({"num_epochs": 10, "batch_size": 64}) ) + + +def test_nested_in_subparser(): + # https://github.com/brentyi/dcargs/issues/9 + @dataclasses.dataclass(frozen=True) + class Subtype: + data: int = 1 + + @dataclasses.dataclass(frozen=True) + class TypeA: + subtype: Subtype = Subtype(1) + + @dataclasses.dataclass(frozen=True) + class TypeB: + subtype: Subtype = Subtype(2) + + @dataclasses.dataclass(frozen=True) + class Wrapper: + supertype: Union[TypeA, TypeB] = TypeA() + + assert dcargs.cli(Wrapper, args=[]) == Wrapper() + assert ( + dcargs.cli( + Wrapper, args="supertype:type-a --supertype.subtype.data 1".split(" ") + ) + == Wrapper() + ) From 79eb7a515a74b7bfc529b516b122f68fa80237ba Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 3 Sep 2022 06:17:29 -0700 Subject: [PATCH 122/491] Test fix, prefix name toggle --- dcargs/_parsers.py | 27 +++++++++++++++++++-------- dcargs/_strings.py | 29 +++++++++++++++++------------ dcargs/conf/_subcommands.py | 11 +++++++---- dcargs/extras/_serialization.py | 2 +- 4 files changed, 44 insertions(+), 25 deletions(-) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 1ce645e5e..cf5462ecf 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -39,6 +39,7 @@ class ParserSpecification: """Each parser contains a list of arguments and optionally some subparsers.""" + f: Callable description: str args: List[_arguments.ArgumentDefinition] helptext_from_nested_class_field_name: Dict[str, Optional[str]] @@ -190,6 +191,7 @@ def from_callable( ) return ParserSpecification( + f=f, description=description if description is not None else _docstrings.get_callable_description(f), @@ -303,6 +305,7 @@ def from_field( is _resolver.unwrap_origin_strip_extras(option) else _fields.MISSING_NONPROP ), + prefix_name=True, ), ) @@ -333,6 +336,7 @@ def from_field( # If there are any required arguments in the default subparser, we should mark # the subparser group as a whole as required. + default_name = None if ( field.default is not None and field.default not in _fields.MISSING_SINGLETONS @@ -347,10 +351,20 @@ def from_field( default_name = _strings.subparser_name_from_type( prefix, type(field.default) ) - assert default_name in parser_from_name, ( - f"Default with type {type(field.default)} was passed in, but no" - " matching subparser." - ) + if default_name not in parser_from_name: + # If we can't find the subparser by name, search by type. This is needed + # when the user renames their subcommands. (eg via dcargs.conf.subcommand) + default_name = None + for name, parser in parser_from_name.items(): + if type(field.default) is _resolver.unwrap_origin_strip_extras( + parser.f + ): + default_name = name + break + assert default_name is not None, ( + f"Default with type {type(field.default)} was passed in, but no" + " matching subparser." + ) default_parser = parser_from_name[default_name] if any(map(lambda arg: arg.lowered.required, default_parser.args)): required = True @@ -367,10 +381,7 @@ def from_field( if field.helptext is not None: description_parts.append(field.helptext) if not required and field.default not in _fields.MISSING_SINGLETONS: - default = field.default - if default is not None: - default = _strings.subparser_name_from_type(prefix, type(default)) - description_parts.append(f" (default: {default})") + description_parts.append(f" (default: {default_name})") description = ( # We use `None` instead of an empty string to prevent a line break from # being created where the description would be. diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 4c30c1a26..15e5bfcd9 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -3,7 +3,7 @@ import functools import re import textwrap -from typing import Iterable, List, Sequence, Type, Union +from typing import Iterable, List, Sequence, Tuple, Type, Union import termcolor @@ -53,7 +53,7 @@ def hyphen_separated_from_camel_case(name: str) -> str: return _camel_separator_pattern().sub(r"-\1", name).lower() -def _subparser_name_from_type(cls: Type) -> str: +def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: from .conf import _subcommands # Prevent circular imports cls, type_from_typevar = _resolver.resolve_generic_types(cls) @@ -63,24 +63,29 @@ def _subparser_name_from_type(cls: Type) -> str: # Subparser name from `dcargs.metadata.subcommand()`. if len(found_subcommand_configs) > 0: - return found_subcommand_configs[0].name + return found_subcommand_configs[0].name, found_subcommand_configs[0].prefix_name # Subparser name from class name. if len(type_from_typevar) == 0: assert hasattr(cls, "__name__") - return hyphen_separated_from_camel_case(cls.__name__) # type: ignore - - return "-".join( - map( - _subparser_name_from_type, - [cls] + list(type_from_typevar.values()), - ) + return hyphen_separated_from_camel_case(cls.__name__), True # type: ignore + + return ( + "-".join( + map( + lambda x: _subparser_name_from_type(x)[0], + [cls] + list(type_from_typevar.values()), + ) + ), + True, ) def subparser_name_from_type(prefix: str, cls: Union[Type, None]) -> str: - suffix = _subparser_name_from_type(cls) if cls is not None else "None" - if len(prefix) == 0: + suffix, use_prefix = ( + _subparser_name_from_type(cls) if cls is not None else ("None", True) + ) + if len(prefix) == 0 or not use_prefix: return suffix return f"{prefix}:{suffix}".replace("_", "-") diff --git a/dcargs/conf/_subcommands.py b/dcargs/conf/_subcommands.py index e5a31957b..03d99661a 100644 --- a/dcargs/conf/_subcommands.py +++ b/dcargs/conf/_subcommands.py @@ -9,6 +9,7 @@ class _SubcommandConfiguration: name: str default: Any description: Optional[str] + prefix_name: bool def __hash__(self) -> int: return object.__hash__(self) @@ -19,6 +20,7 @@ def subcommand( *, default: Any = MISSING_NONPROP, description: Optional[str] = None, + prefix_name: bool = True, ) -> Any: """Returns a metadata object for configuring subcommands with `typing.Annotated`. This is useful but can make code harder to read, so usage is discouraged. @@ -34,19 +36,20 @@ def subcommand( This will create two subcommands: `nested-type-a` and `nested-type-b`. Annotating each type with `dcargs.metadata.subcommand()` allows us to override for - each subcommand the (a) name, (b) defaults, and (c) helptext. + each subcommand the (a) name, (b) defaults, (c) helptext, and (d) whether to prefix + the name or not. ```python dcargs.cli( Union[ Annotated[ - NestedTypeA, subcommand("a", default=NestedTypeA(...), description="...") + NestedTypeA, subcommand("a", ...) ], Annotated[ - NestedTypeA, subcommand("b", default=NestedTypeA(...), description="...") + NestedTypeA, subcommand("b", ...) ], ] ) ``` """ - return _SubcommandConfiguration(name, default, description) + return _SubcommandConfiguration(name, default, description, prefix_name) diff --git a/dcargs/extras/_serialization.py b/dcargs/extras/_serialization.py index c5c433a4e..6dc9931e4 100644 --- a/dcargs/extras/_serialization.py +++ b/dcargs/extras/_serialization.py @@ -44,12 +44,12 @@ def _get_contained_special_types_from_type( else _parent_contained_dataclasses ) + cls, _ = _resolver.unwrap_annotated(cls) cls, type_from_typevar = _resolver.resolve_generic_types(cls) contained_dataclasses = {cls} def handle_type(typ) -> Set[Type]: - print(typ) # Handle dataclasses. if _resolver.is_dataclass(typ) and typ not in parent_contained_dataclasses: return _get_contained_special_types_from_type( From cfbf61869297bb1611e26fbb140f658d1a335175 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 4 Sep 2022 10:40:07 -0700 Subject: [PATCH 123/491] Generalize positional logic --- dcargs/_arguments.py | 10 ++-- dcargs/_calling.py | 2 +- dcargs/_cli.py | 1 - dcargs/_fields.py | 103 +++++++++++++++++++++++++++------------- dcargs/_parsers.py | 36 ++++---------- dcargs/conf/__init__.py | 8 ++-- dcargs/conf/_markers.py | 8 +++- 7 files changed, 97 insertions(+), 71 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 4df17a394..dfb8d08d6 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -138,7 +138,7 @@ def _rule_handle_boolean_flags( if ( arg.field.default in _fields.MISSING_SINGLETONS - or arg.field.positional + or arg.field.is_positional() or _markers.FLAG_CONVERSION_OFF in arg.field.markers ): # Treat bools as a normal parameter. @@ -260,9 +260,9 @@ def _rule_generate_helptext( # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string docstring_help = docstring_help.replace("%", "%%") help_parts.append(docstring_help) - elif arg.field.positional and arg.field.name != _strings.dummy_field_name: + elif arg.field.is_positional() and arg.field.name != _strings.dummy_field_name: # Place the type in the helptext. Note that we skip this for dummy fields, which - # will sitll have the type in the metavar. + # will still have the type in the metavar. help_parts.append(str(lowered.metavar)) default = lowered.default @@ -309,7 +309,7 @@ def _rule_set_name_or_flag( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: - if arg.field.positional: + if arg.field.is_positional(): name_or_flag = _strings.make_field_name([arg.prefix, arg.field.name]) elif lowered.action == "store_false": name_or_flag = "--" + _strings.make_field_name( @@ -331,7 +331,7 @@ def _rule_positional_special_handling( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: - if not arg.field.positional: + if not arg.field.is_positional(): return lowered metavar = _strings.make_field_name([arg.prefix, arg.field.name]).upper() diff --git a/dcargs/_calling.py b/dcargs/_calling.py index d20656cad..1c979ecae 100644 --- a/dcargs/_calling.py +++ b/dcargs/_calling.py @@ -163,7 +163,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: consumed_keywords |= consumed_keywords_child if value is not _fields.EXCLUDE_FROM_CALL: - if field.positional: + if field.is_positional(): args.append(value) else: kwargs[ diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 8849d413a..64dc45e27 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -251,7 +251,6 @@ def _cli_impl( description=description, parent_classes=set(), # Used for recursive calls. parent_type_from_typevar=None, # Used for recursive calls. - parent_markers=(), # Used for recursive calls. default_instance=default_instance_internal, # Overrides for default values. prefix="", # Used for recursive calls. ) diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 87319bf2a..95048efc0 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -11,11 +11,23 @@ import itertools import typing import warnings -from typing import Any, Callable, Hashable, Iterable, List, Optional, Type, Union, cast +from typing import ( + Any, + Callable, + FrozenSet, + Hashable, + Iterable, + List, + Optional, + Tuple, + Type, + Union, + cast, +) import docstring_parser import typing_extensions -from typing_extensions import get_args, get_type_hints, is_typeddict +from typing_extensions import Annotated, get_args, get_type_hints, is_typeddict from . import conf # Avoid circular import. from . import _docstrings, _instantiators, _resolver, _singleton, _strings @@ -28,22 +40,49 @@ class FieldDefinition: typ: Type default: Any helptext: Optional[str] - positional: bool - markers: List[_markers.Marker] = dataclasses.field(default_factory=list) + markers: FrozenSet[_markers.Marker] # Override the name in our kwargs. Currently only used for dictionary types when # the key values aren't strings, but in the future could be used whenever the # user-facing argument name doesn't match the keyword expected by our callable. - name_override: Optional[Any] = None - - def __post_init__(self): # - # Auto-populate markers if unset; this is meant to not run when we do - # dataclasses.replace, etc. TODO: the whole markers design, handling of - # Annotated[], etc, should be revisited... - if len(self.markers) == 0: - self.markers.extend( - _resolver.unwrap_annotated(self.typ, _markers.Marker)[1] - ) + name_override: Optional[Any] + + @staticmethod + def make( + name: str, + typ: Type, + default: Any, + helptext: Optional[str], + *, + markers: Tuple[_markers.Marker, ...] = (), + name_override: Optional[Any] = None, + ): + _, inferred_markers = _resolver.unwrap_annotated(typ, _markers.Marker) + return FieldDefinition( + name, + typ, + default, + helptext, + frozenset(inferred_markers).union(markers), + name_override, + ) + + def add_markers(self, markers: Tuple[_markers.Marker, ...]) -> FieldDefinition: + if len(markers) == 0: + return self + return dataclasses.replace( + self, + typ=Annotated.__class_getitem__((self.typ,) + markers), # type: ignore + markers=self.markers.union(markers), + ) + + def is_positional(self) -> bool: + return ( + # Explicit positionals. + _markers.POSITIONAL in self.markers + # Dummy dataclasses should have a single positional field. + or self.name == _strings.dummy_field_name + ) class PropagatingMissingType(_singleton.Singleton): @@ -115,6 +154,10 @@ def field_list_from_callable( if isinstance(out, UnsupportedNestedTypeMessage): raise _instantiators.UnsupportedTypeAnnotationError(out.message) + + # Recursively apply markers. + _, parent_markers = _resolver.unwrap_annotated(f, _markers.Marker) + out = list(map(lambda field: field.add_markers(parent_markers), out)) return out @@ -249,12 +292,11 @@ def _try_field_list_from_typeddict( default = MISSING_PROP field_list.append( - FieldDefinition( + FieldDefinition.make( name=name, typ=typ, default=default, helptext=_docstrings.get_field_docstring(cls, name), - positional=False, ) ) return field_list @@ -281,12 +323,11 @@ def _try_field_list_from_namedtuple( default = MISSING_PROP field_list.append( - FieldDefinition( + FieldDefinition.make( name=name, typ=typ, default=default, helptext=_docstrings.get_field_docstring(cls, name), - positional=False, ) ) return field_list @@ -300,14 +341,11 @@ def _try_field_list_from_dataclass( for dc_field in filter(lambda field: field.init, _resolver.resolved_fields(cls)): default = _get_dataclass_field_default(dc_field, default_instance) field_list.append( - FieldDefinition( + FieldDefinition.make( name=dc_field.name, typ=dc_field.type, default=default, helptext=_docstrings.get_field_docstring(cls, dc_field.name), - # Only mark positional if using a dummy field, for taking single types - # directly as input. - positional=dc_field.name == _strings.dummy_field_name, ) ) return field_list @@ -346,7 +384,7 @@ def _field_list_from_tuple( for i, child in enumerate(children): default_i = default_instance[i] # type: ignore field_list.append( - FieldDefinition( + FieldDefinition.make( # Ideally we'd have --tuple[0] instead of --tuple.0 as the command-line # argument, but in practice the brackets are annoying because they # require escaping. @@ -354,10 +392,9 @@ def _field_list_from_tuple( typ=child, default=default_i, helptext="", - # This should really be positional=True, but the CLI is more - # intuitive for mixed nested/non-nested types in tuples when we - # stick with kwargs. Tuples are special-cased in _calling.py. - positional=False, + # This should really set the positional marker, but the CLI is more + # intuitive for mixed nested/non-nested types in tuples when we stick + # with kwargs. Tuples are special-cased in _calling.py. ) ) @@ -408,12 +445,11 @@ def _try_field_list_from_sequence( field_list = [] for i, default_i in enumerate(default_instance): # type: ignore field_list.append( - FieldDefinition( + FieldDefinition.make( name=str(i), typ=contained_type, default=default_i, helptext="", - positional=False, ) ) return field_list @@ -430,12 +466,11 @@ def _try_field_list_from_dict( field_list = [] for k, v in cast(dict, default_instance).items(): field_list.append( - FieldDefinition( + FieldDefinition.make( name=str(k) if not isinstance(k, enum.Enum) else k.name, typ=type(v), default=v, helptext=None, - positional=False, # Dictionary specific key: name_override=k, ) @@ -514,13 +549,15 @@ def _field_list_from_params( return out field_list.append( - FieldDefinition( + FieldDefinition.make( name=param.name, # Note that param.annotation does not resolve forward references. typ=hints[param.name], default=default, helptext=helptext, - positional=param.kind is inspect.Parameter.POSITIONAL_ONLY, + markers=(_markers.POSITIONAL,) + if param.kind is inspect.Parameter.POSITIONAL_ONLY + else (), ) ) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index cf5462ecf..d29f03d89 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -4,19 +4,7 @@ import argparse import dataclasses import itertools -from typing import ( - Any, - Callable, - Dict, - List, - Optional, - Set, - Tuple, - Type, - TypeVar, - Union, - cast, -) +from typing import Any, Callable, Dict, List, Optional, Set, Type, TypeVar, Union, cast import termcolor from typing_extensions import get_args, get_origin @@ -29,8 +17,8 @@ _instantiators, _resolver, _strings, - conf, ) +from .conf import _markers, _subcommands T = TypeVar("T") @@ -52,7 +40,6 @@ def from_callable( description: Optional[str], parent_classes: Set[Type], parent_type_from_typevar: Optional[Dict[TypeVar, Type]], - parent_markers: Tuple[conf._markers.Marker, ...], default_instance: Union[ T, _fields.PropagatingMissingType, _fields.NonpropagatingMissingType ], @@ -61,9 +48,6 @@ def from_callable( """Create a parser definition from a callable.""" # Resolve generic types. - markers = ( - parent_markers + _resolver.unwrap_annotated(f, conf._markers.Marker)[1] - ) f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) if parent_type_from_typevar is not None: @@ -95,6 +79,7 @@ def from_callable( for field in field_list: field = dataclasses.replace( field, + # Resolve generic types. typ=_resolver.type_from_typevar_constraints( _resolver.apply_type_from_typevar( field.typ, @@ -102,14 +87,13 @@ def from_callable( ) ), ) - field.markers.extend(markers) # TODO: would be nice to avoid this mutation! if isinstance(field.typ, TypeVar): raise _instantiators.UnsupportedTypeAnnotationError( f"Field {field.name} has an unbound TypeVar: {field.typ}." ) - if conf._markers.FIXED not in field.markers: + if _markers.FIXED not in field.markers: # (1) Handle Unions over callables; these result in subparsers. subparsers_attempt = SubparsersSpecification.from_field( field, @@ -120,7 +104,7 @@ def from_callable( if subparsers_attempt is not None: if ( not subparsers_attempt.required - and conf._markers.AVOID_SUBCOMMANDS in field.markers + and _markers.AVOID_SUBCOMMANDS in field.markers ): # Don't make a subparser. field = dataclasses.replace(field, typ=type(field.default)) @@ -144,7 +128,6 @@ def from_callable( description=None, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, - parent_markers=markers, default_instance=field.default, prefix=_strings.make_field_name([prefix, field.name]), ) @@ -226,7 +209,7 @@ def format_group_name(nested_field_name: str) -> str: # Add each argument. for arg in self.args: - if arg.field.positional: + if arg.field.is_positional(): arg.add_argument(positional_group) continue @@ -291,12 +274,12 @@ def from_field( for option in options_no_none: name = _strings.subparser_name_from_type(prefix, option) option, found_subcommand_configs = _resolver.unwrap_annotated( - option, conf._subcommands._SubcommandConfiguration + option, _subcommands._SubcommandConfiguration ) if len(found_subcommand_configs) == 0: # Make a dummy subcommand config. found_subcommand_configs = ( - conf._subcommands._SubcommandConfiguration( + _subcommands._SubcommandConfiguration( "unused", description=None, default=( @@ -314,7 +297,6 @@ def from_field( description=found_subcommand_configs[0].description, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, - parent_markers=tuple(field.markers), default_instance=found_subcommand_configs[0].default, prefix=prefix, ) @@ -353,7 +335,7 @@ def from_field( ) if default_name not in parser_from_name: # If we can't find the subparser by name, search by type. This is needed - # when the user renames their subcommands. (eg via dcargs.conf.subcommand) + # when the user renames their subcommands. (eg via dcargs.subcommand) default_name = None for name, parser in parser_from_name.items(): if type(field.default) is _resolver.unwrap_origin_strip_extras( diff --git a/dcargs/conf/__init__.py b/dcargs/conf/__init__.py index 7415c82e8..e1a1743ee 100644 --- a/dcargs/conf/__init__.py +++ b/dcargs/conf/__init__.py @@ -1,15 +1,17 @@ """The :mod:`dcargs.conf` submodule contains helpers for attaching parsing-specific -configuration metadata to types via PEP 593 runtime annotations. +configuration metadata to types via [PEP 593](https://peps.python.org/pep-0593/) runtime +annotations. -Features here are supported, but are generally unnecessary and should be used sparingly. +Features here are supported, but generally unnecessary and should be used sparingly. """ -from ._markers import AvoidSubcommands, Fixed, FlagConversionOff +from ._markers import AvoidSubcommands, Fixed, FlagConversionOff, Positional from ._subcommands import subcommand __all__ = [ "AvoidSubcommands", "Fixed", "FlagConversionOff", + "Positional", "subcommand", ] diff --git a/dcargs/conf/_markers.py b/dcargs/conf/_markers.py index 91fa24d49..3e6100651 100644 --- a/dcargs/conf/_markers.py +++ b/dcargs/conf/_markers.py @@ -22,9 +22,15 @@ def __repr__(self): T = TypeVar("T", bound=Type) +POSITIONAL = _make_marker("Positional") +Positional = Annotated[T, POSITIONAL] +"""A type `T` can be annotated as `Positional[T]` if we want to parse it as a positional +argument.""" + + FIXED = _make_marker("Fixed") Fixed = Annotated[T, FIXED] -"""A type `T` can be annotated as `Fixed[T]` if we don't want dcargs to parse it. A +"""A type `T` can be annotated as `Fixed[T]` to prevent `dcargs.cli` from parsing it. A default value should be set instead.""" FLAG_CONVERSION_OFF = _make_marker("FlagConversionOff") From 87d4ad49b0deb62922d5a3d205e68e5286477674 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 5 Sep 2022 21:53:27 +0100 Subject: [PATCH 124/491] Replace `get_parser()` with `--dcargs-print-completion` --- dcargs/__init__.py | 3 +- dcargs/_argparse_formatter.py | 11 +-- dcargs/_cli.py | 144 ++++++++-------------------------- dcargs/_deprecated.py | 1 - dcargs/_instantiators.py | 12 ++- examples/01_functions.py | 0 poetry.lock | 58 +++++++++++--- pyproject.toml | 1 + 8 files changed, 85 insertions(+), 145 deletions(-) mode change 100644 => 100755 examples/01_functions.py diff --git a/dcargs/__init__.py b/dcargs/__init__.py index 3073b8b20..6f2252440 100644 --- a/dcargs/__init__.py +++ b/dcargs/__init__.py @@ -1,5 +1,5 @@ from . import conf, extras -from ._cli import cli, get_parser +from ._cli import cli from ._fields import MISSING_PUBLIC as MISSING from ._instantiators import UnsupportedTypeAnnotationError @@ -7,7 +7,6 @@ "conf", "extras", "cli", - "get_parser", "MISSING", "UnsupportedTypeAnnotationError", ] diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index 0322cc928..a6dac3593 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -184,13 +184,4 @@ def _split_lines(self, text, width): return out def _fill_text(self, text, width, indent): - text = self._whitespace_matcher.sub(" ", text).strip() - import textwrap as textwrap - - # Sketchy, but seems to work. - textwrap.len = monkeypatch_len # type: ignore - out = textwrap.fill( - text, width, initial_indent=indent, subsequent_indent=indent - ) - del textwrap.len # type: ignore - return out + return "".join(indent + line for line in text.splitlines(keepends=True)) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 64dc45e27..913a0ed2e 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -1,10 +1,11 @@ """Core public API.""" import argparse import dataclasses +import sys import warnings from typing import Callable, Optional, Sequence, Type, TypeVar, Union, cast, overload -from typing_extensions import Literal, assert_never +import shtab from . import _argparse_formatter, _calling, _fields, _parsers, _strings, conf @@ -103,110 +104,6 @@ def cli( Returns: The output of `f(...)`. """ - out = _cli_impl( - "f_out", - f, - prog=prog, - description=description, - args=args, - default=default, - **deprecated_kwargs, - ) - return out - - -@overload -def get_parser( - f: Type[OutT], - *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, -) -> argparse.ArgumentParser: - ... - - -@overload -def get_parser( - f: Callable[..., OutT], - *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, -) -> argparse.ArgumentParser: - ... - - -def get_parser( - f: Union[Type[OutT], Callable[..., OutT]], - *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, - **deprecated_kwargs, -) -> argparse.ArgumentParser: - """Returns the argparse parser that would be used under-the-hood if `dcargs.cli()` - was called with the same arguments. - - This can be useful for libraries like argcomplete, pyzshcomplete, or shtab, which - enable autocompletion for argparse parsers. - - To reduce compatibility issues, strips out color formatting.""" - with _argparse_formatter.dummy_termcolor_context(): - out = _cli_impl( - "parser", - f, - prog=prog, - description=description, - args=args, - default=default, - **deprecated_kwargs, - ) - assert isinstance(out, argparse.ArgumentParser) - return out - - -@overload -def _cli_impl( - _return_stage: Literal["parser"], - f: Callable[..., OutT], - *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, - **deprecated_kwargs, -) -> argparse.ArgumentParser: - ... - - -@overload -def _cli_impl( - _return_stage: Literal["f_out"], - f: Callable[..., OutT], - *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, - **deprecated_kwargs, -) -> OutT: - ... - - -def _cli_impl( - _return_stage: Literal["parser", "f_out"], - f: Callable[..., OutT], - *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, - **deprecated_kwargs, -) -> Union[OutT, argparse.ArgumentParser]: if "default_instance" in deprecated_kwargs: warnings.warn( "`default_instance=` is deprecated! use `default=` instead.", stacklevel=2 @@ -255,8 +152,24 @@ def _cli_impl( prefix="", # Used for recursive calls. ) + # If we pass in the --dcargs-print-completion flag: turn termcolor off, and ge the + # shell we want to generate a completion script for (bash/zsh/tcsh). + args = sys.argv[1:] if args is None else args + print_completion = len(args) >= 2 and args[0] == "--dcargs-print-completion" + + formatting_context = _argparse_formatter.ansi_context() + completion_shell = None + if print_completion: + formatting_context = _argparse_formatter.dummy_termcolor_context() + completion_shell = args[1] + assert completion_shell in ( + "bash", + "zsh", + "tcsh", + ), f"Shell should be one `bash`, `zsh`, or `tcsh`, but got {completion_shell}" + # Generate parser! - with _argparse_formatter.ansi_context(): + with formatting_context: parser = argparse.ArgumentParser( prog=prog, formatter_class=_argparse_formatter.make_formatter_class( @@ -264,9 +177,19 @@ def _cli_impl( ), ) parser_definition.apply(parser) - if _return_stage == "parser": - return parser + + if print_completion: + print( + shtab.complete( + parser=parser, + shell=completion_shell, + root_prefix=f"dcargs_{parser.prog}", + ) + ) + raise SystemExit() + value_from_prefixed_field_name = vars(parser.parse_args(args=args)) + value_from_prefixed_field_name.pop("dcargs_print_completion") if dummy_wrapped: value_from_prefixed_field_name = { @@ -297,7 +220,4 @@ def _cli_impl( if dummy_wrapped: out = getattr(out, _strings.dummy_field_name) - if _return_stage == "f_out": - return out - - assert_never(_return_stage) + return out diff --git a/dcargs/_deprecated.py b/dcargs/_deprecated.py index 05e32b46b..c879522a8 100644 --- a/dcargs/_deprecated.py +++ b/dcargs/_deprecated.py @@ -1,3 +1,2 @@ from ._cli import cli as parse # noqa -from ._cli import get_parser as generate_parser # noqa from .extras._serialization import from_yaml, to_yaml # noqa diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index 4a0fd20b4..d7ff78c4f 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -22,13 +22,11 @@ Tuple[int, float] lambda strings: tuple( - [ - typ(x) - for typ, x in zip( - (int, float), - strings, - ) - ] + typ(x) + for typ, x in zip( + (int, float), + strings, + ) ) ``` """ diff --git a/examples/01_functions.py b/examples/01_functions.py old mode 100644 new mode 100755 diff --git a/poetry.lock b/poetry.lock index 2ee845526..14a891815 100644 --- a/poetry.lock +++ b/poetry.lock @@ -23,10 +23,10 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "zope.interface"] +tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six"] [[package]] name = "backports.cached-property" @@ -87,9 +87,9 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] [[package]] name = "iniconfig" @@ -134,6 +134,9 @@ category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +[package.dependencies] +setuptools = "*" + [[package]] name = "numpy" version = "1.21.1" @@ -177,8 +180,8 @@ python-versions = ">=3.6" importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] -testing = ["pytest-benchmark", "pytest"] -dev = ["tox", "pre-commit"] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "py" @@ -197,7 +200,7 @@ optional = false python-versions = ">=3.6.8" [package.extras] -diagrams = ["railroad-diagrams", "jinja2"] +diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pyright" @@ -250,7 +253,7 @@ coverage = {version = ">=5.2.1", extras = ["toml"]} pytest = ">=4.6" [package.extras] -testing = ["virtualenv", "pytest-xdist", "six", "process-tests", "hunter", "fields"] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] [[package]] name = "pyyaml" @@ -260,6 +263,27 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "setuptools" +version = "65.3.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "shtab" +version = "1.5.5" +description = "Automagic shell tab completion for Python CLI applications" +category = "main" +optional = false +python-versions = ">=3.2" + [[package]] name = "termcolor" version = "1.1.0" @@ -312,13 +336,13 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] +docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] +testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "897e78d65e9a00cd292034a23eb35922e934052dbba5b927795e39cf21b3f449" +content-hash = "880ee7ca4570c69fe1fadfac1b13d81c6719eaf7b45420bbb587985f9e9ef5ed" [metadata.files] antlr4-python3-runtime = [ @@ -552,6 +576,14 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +setuptools = [ + {file = "setuptools-65.3.0-py3-none-any.whl", hash = "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82"}, + {file = "setuptools-65.3.0.tar.gz", hash = "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57"}, +] +shtab = [ + {file = "shtab-1.5.5-py2.py3-none-any.whl", hash = "sha256:f4bf7cc122fb434cb65b96db7e3adcf3b5258846e906e60c48b3725d2965c6b5"}, + {file = "shtab-1.5.5.tar.gz", hash = "sha256:f90a6ce64b821002d5881b6212992a27ab40c3bab36aabca8de118b0b78f61f6"}, +] termcolor = [ {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, ] diff --git a/pyproject.toml b/pyproject.toml index bd329a319..9e382d904 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ termcolor = "^1.1.0" "backports.cached-property" = { version = "^1.0.2", python = "~3.7" } colorama = {version = "^0.4.0", platform = "win32"} frozendict = "^2.3.4" +shtab = "^1.5.5" [tool.poetry.dev-dependencies] pytest = "^7.1.2" From 519abfeeae57d1a4e065e7287fdcf162830d5200 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 6 Sep 2022 21:05:02 -0700 Subject: [PATCH 125/491] Update tests with new completion logic --- tests/test_dcargs.py | 2 -- tests/test_helptext.py | 10 ++++---- tests/test_print_completion.py | 43 ++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 tests/test_print_completion.py diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 656f1aa80..ede75c57b 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -40,8 +40,6 @@ class ManyTypes: f: float p: pathlib.Path - assert isinstance(dcargs.get_parser(ManyTypes), argparse.ArgumentParser) - # We can directly pass a dataclass to `dcargs.cli()`: assert dcargs.cli( ManyTypes, diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 56ad2963f..bbfbde399 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -20,14 +20,14 @@ def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: with pytest.raises(SystemExit), contextlib.redirect_stdout(target): dcargs.cli(f, args=args) - # Check against dcargs.generate_parser(); this should return the same underlying - # parser. + # Check helptext with vs without termcolor. This can help catch text wrapping bugs + # caused by ANSI sequences. target2 = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): - with dcargs._argparse_formatter.ansi_context(): - dcargs.get_parser(f).parse_args(args) - assert dcargs._strings.strip_ansi_sequences(target.getvalue()) == target2.getvalue() + with dcargs._argparse_formatter.dummy_termcolor_context(): + dcargs.cli(f, args=args) + assert target2.getvalue() == dcargs._strings.strip_ansi_sequences(target.getvalue()) return target2.getvalue() diff --git a/tests/test_print_completion.py b/tests/test_print_completion.py new file mode 100644 index 000000000..9f4c34d95 --- /dev/null +++ b/tests/test_print_completion.py @@ -0,0 +1,43 @@ +import contextlib +import dataclasses +import io +from typing import Union + +import pytest + +import dcargs + + +# https://github.com/brentyi/dcargs/issues/9 +@dataclasses.dataclass(frozen=True) +class Subtype: + data: int = 1 + + +@dataclasses.dataclass(frozen=True) +class TypeA: + subtype: Subtype = Subtype(1) + + +@dataclasses.dataclass(frozen=True) +class TypeB: + subtype: Subtype = Subtype(2) + + +@dataclasses.dataclass(frozen=True) +class Wrapper: + supertype: Union[TypeA, TypeB] = TypeA() + + +def test_bash(): + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + dcargs.cli(Wrapper, args=["--dcargs-print-completion", "bash"]) + assert "# AUTOMATCALLY GENERATED by `shtab`" in target.getvalue() + + +def test_zsh(): + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + dcargs.cli(Wrapper, args=["--dcargs-print-completion", "zsh"]) + assert "# AUTOMATCALLY GENERATED by `shtab`" in target.getvalue() From c81f6d93b9b6264163c223dcad3410ac71d4bc3e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 6 Sep 2022 21:13:20 -0700 Subject: [PATCH 126/491] Fix errors from old completion logic --- dcargs/_cli.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 913a0ed2e..41e58268c 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -162,11 +162,6 @@ def cli( if print_completion: formatting_context = _argparse_formatter.dummy_termcolor_context() completion_shell = args[1] - assert completion_shell in ( - "bash", - "zsh", - "tcsh", - ), f"Shell should be one `bash`, `zsh`, or `tcsh`, but got {completion_shell}" # Generate parser! with formatting_context: @@ -179,6 +174,11 @@ def cli( parser_definition.apply(parser) if print_completion: + assert completion_shell in ( + "bash", + "zsh", + "tcsh", + ), f"Shell should be one `bash`, `zsh`, or `tcsh`, but got {completion_shell}" print( shtab.complete( parser=parser, @@ -189,7 +189,6 @@ def cli( raise SystemExit() value_from_prefixed_field_name = vars(parser.parse_args(args=args)) - value_from_prefixed_field_name.pop("dcargs_print_completion") if dummy_wrapped: value_from_prefixed_field_name = { From a0e19520418dbb484bcece944b3e4a24dc38aae7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 6 Sep 2022 21:46:53 -0700 Subject: [PATCH 127/491] Improve + deprecate serialization helpers :floppy_disk: --- dcargs/_cli.py | 12 +++-- dcargs/extras/_serialization.py | 60 ++++++++++++++--------- docs/source/serialization.rst | 22 --------- tests/test_generics_and_serialization.py | 61 ++++++++++++++++++------ 4 files changed, 90 insertions(+), 65 deletions(-) delete mode 100644 docs/source/serialization.rst diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 41e58268c..670c4bd92 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -127,6 +127,9 @@ def cli( _fields.MISSING_NONPROP if default is None else default ) + # We wrap our type with a dummy dataclass if it can't be treated as a nested type. + # For example: passing in f=int will result in a dataclass with a single field + # typed as int. if not _fields.is_nested_type(cast(Type, f), default_instance_internal): dummy_field = cast( dataclasses.Field, @@ -174,11 +177,10 @@ def cli( parser_definition.apply(parser) if print_completion: - assert completion_shell in ( - "bash", - "zsh", - "tcsh", - ), f"Shell should be one `bash`, `zsh`, or `tcsh`, but got {completion_shell}" + assert completion_shell in ("bash", "zsh", "tcsh",), ( + "Shell should be one `bash`, `zsh`, or `tcsh`, but got" + f" {completion_shell}" + ) print( shtab.complete( parser=parser, diff --git a/dcargs/extras/_serialization.py b/dcargs/extras/_serialization.py index 6dc9931e4..333f2b324 100644 --- a/dcargs/extras/_serialization.py +++ b/dcargs/extras/_serialization.py @@ -17,20 +17,6 @@ DataclassType = TypeVar("DataclassType") -def _get_contained_special_types_from_instance(instance: Any) -> Set[Type]: - """Takes an object and recursively searches its cihldren for dataclass or enum - types.""" - if issubclass(type(instance), enum.Enum): - return {type(instance)} - elif not dataclasses.is_dataclass(instance): - return set() - - out = {type(instance)} - for v in vars(instance).values(): - out |= _get_contained_special_types_from_instance(v) - return out - - def _get_contained_special_types_from_type( cls: Type, _parent_contained_dataclasses: Optional[Set[Type]] = None, @@ -47,14 +33,14 @@ def _get_contained_special_types_from_type( cls, _ = _resolver.unwrap_annotated(cls) cls, type_from_typevar = _resolver.resolve_generic_types(cls) - contained_dataclasses = {cls} + contained_special_types = {cls} - def handle_type(typ) -> Set[Type]: + def handle_type(typ: Type) -> Set[Type]: # Handle dataclasses. if _resolver.is_dataclass(typ) and typ not in parent_contained_dataclasses: return _get_contained_special_types_from_type( typ, - _parent_contained_dataclasses=contained_dataclasses + _parent_contained_dataclasses=contained_special_types | parent_contained_dataclasses, ) @@ -67,16 +53,20 @@ def handle_type(typ) -> Set[Type]: # Handle generics. for typ in type_from_typevar.values(): - contained_dataclasses |= handle_type(typ) + contained_special_types |= handle_type(typ) if cls in parent_contained_dataclasses: - return contained_dataclasses + return contained_special_types # Handle fields. for field in _resolver.resolved_fields(cls): # type: ignore - contained_dataclasses |= handle_type(field.type) + contained_special_types |= handle_type(field.type) - return contained_dataclasses + # Handle subclasses. + for subclass in cls.__subclasses__(): + contained_special_types |= handle_type(subclass) + + return contained_special_types def _make_loader(cls: Type) -> Type[yaml.Loader]: @@ -134,7 +124,7 @@ class DataclassDumper(yaml.Dumper): def ignore_aliases(self, data): return super().ignore_aliases(data) or data is _fields.MISSING_PROP - contained_types = list(_get_contained_special_types_from_instance(instance)) + contained_types = list(_get_contained_special_types_from_type(type(instance))) contained_type_names = list(map(lambda cls: cls.__name__, contained_types)) # Note: this is currently a stricter than necessary assert. @@ -183,7 +173,19 @@ def from_yaml( stream: Union[str, IO[str], bytes, IO[bytes]], ) -> DataclassType: """Re-construct a dataclass instance from a yaml-compatible string, which should be - generated from `dcargs.extra.to_yaml()`. + generated from `dcargs.extras.to_yaml()`. + + As a secondary feature aimed at enabling the use of :func:`dcargs.cli` for general + configuration use cases, we also introduce functions for human-readable dataclass + serialization: :func:`dcargs.conf.from_yaml` and :func:`dcargs.conf.to_yaml` attempt + to strike a balance between flexibility and robustness — in contrast to naively + dumping or loading dataclass instances (via pickle, PyYAML, etc), explicit type + references enable custom tags that are robust against code reorganization and + refactor, while a PyYAML backend enables serialization of arbitrary Python objects. + + .. warning:: + Serialization functionality is deprecated. It may be removed in a future version + of :code:`dcargs`. Args: cls: Type to reconstruct. @@ -202,6 +204,18 @@ def to_yaml(instance: Any) -> str: """Serialize a dataclass; returns a yaml-compatible string that can be deserialized via `dcargs.extras.from_yaml()`. + As a secondary feature aimed at enabling the use of :func:`dcargs.cli` for general + configuration use cases, we also introduce functions for human-readable dataclass + serialization: :func:`dcargs.conf.from_yaml` and :func:`dcargs.conf.to_yaml` attempt + to strike a balance between flexibility and robustness — in contrast to naively + dumping or loading dataclass instances (via pickle, PyYAML, etc), explicit type + references enable custom tags that are robust against code reorganization and + refactor, while a PyYAML backend enables serialization of arbitrary Python objects. + + .. warning:: + Serialization functionality is deprecated. It may be removed in a future version + of :code:`dcargs`. + Args: instance: Dataclass instance to serialize. diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst deleted file mode 100644 index 6c47af1ef..000000000 --- a/docs/source/serialization.rst +++ /dev/null @@ -1,22 +0,0 @@ -Serialization -==================================== - -As a secondary feature aimed at enabling the use of :func:`dcargs.cli` for general -configuration use cases, we also introduce functions for human-readable -dataclass serialization: - -- :func:`dcargs.extras.to_yaml` and :func:`dcargs.extras.from_yaml` convert - between YAML-style strings and dataclass instances. - -The functions attempt to strike a balance between flexibility and robustness — -in contrast to naively dumping or loading dataclass instances (via pickle, -PyYAML, etc), explicit type references enable custom tags that are robust -against code reorganization and refactor, while a PyYAML backend enables -serialization of arbitrary Python objects. - -Note that we generally prefer to use YAML purely for serialization, as opposed -to a configuration interface that humans are expected to manually write or -modify. Specifying things like loadable base configurations can be done directly -in Python, which enables all of the usual autocompletion and type checking -features. - diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 43a4d8950..4fe599354 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -5,6 +5,7 @@ from typing import Generic, List, Tuple, Type, TypeVar, Union import pytest +import yaml from typing_extensions import Annotated import dcargs @@ -217,7 +218,10 @@ class DataclassGeneric(Generic[T]): DataclassGeneric[Child], args=["--child.a", "5", "--child.b", "7"] ) assert parsed_instance == DataclassGeneric(Child(5, 7)) - _check_serialization_identity(DataclassGeneric[Child], parsed_instance) + + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity(DataclassGeneric[Child], parsed_instance) def test_generic_nested_dataclass_helptext(): @@ -263,14 +267,22 @@ class Subparser(Generic[T1, T2]): args="command:command-one --command.a 5".split(" "), ) assert parsed_instance == Subparser(CommandOne(5)) - _check_serialization_identity(Subparser[CommandOne, CommandTwo], parsed_instance) + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity( + Subparser[CommandOne, CommandTwo], parsed_instance + ) parsed_instance = dcargs.cli( Subparser[CommandOne, CommandTwo], args="command:command-two --command.b 7".split(" "), ) assert parsed_instance == Subparser(CommandTwo(7)) - _check_serialization_identity(Subparser[CommandOne, CommandTwo], parsed_instance) + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity( + Subparser[CommandOne, CommandTwo], parsed_instance + ) def test_generic_subparsers_in_container(): @@ -294,9 +306,11 @@ class Subparser(Generic[T1, T2]): assert parsed_instance == Subparser(Command([5, 3])) and isinstance( parsed_instance.command.a[0], int ) - _check_serialization_identity( - Subparser[Command[int], Command[float]], parsed_instance - ) + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity( + Subparser[Command[int], Command[float]], parsed_instance + ) parsed_instance = dcargs.cli( Subparser[Command[int], Command[float]], @@ -305,9 +319,11 @@ class Subparser(Generic[T1, T2]): assert parsed_instance == Subparser(Command([7.0, 2.0])) and isinstance( parsed_instance.command.a[0], float ) - _check_serialization_identity( - Subparser[Command[int], Command[float]], parsed_instance - ) + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity( + Subparser[Command[int], Command[float]], parsed_instance + ) def test_serialize_missing(): @@ -366,9 +382,7 @@ class Wrapper: subclass: Union[TypeA, TypeB] = TypeA(1) wrapper1 = Wrapper() # Create Wrapper object. - dcargs.extras.from_yaml( - Wrapper, dcargs.extras.to_yaml(wrapper1) - ) # Errors, no constructor for TypeA + assert wrapper1 == dcargs.extras.from_yaml(Wrapper, dcargs.extras.to_yaml(wrapper1)) def test_annotated(): @@ -383,6 +397,23 @@ class Wrapper: subclass: Annotated[TypeA, int] = TypeA(1) wrapper1 = Wrapper() # Create Wrapper object. - dcargs.extras.from_yaml( - Wrapper, dcargs.extras.to_yaml(wrapper1) - ) # Errors, no constructor for TypeA + assert wrapper1 == dcargs.extras.from_yaml(Wrapper, dcargs.extras.to_yaml(wrapper1)) + + +def test_superclass(): + # https://github.com/brentyi/dcargs/issues/7 + + @dataclasses.dataclass + class TypeA: + data: int + + @dataclasses.dataclass + class TypeASubclass(TypeA): + pass + + @dataclasses.dataclass + class Wrapper: + subclass: TypeA + + wrapper1 = Wrapper(TypeASubclass(3)) # Create Wrapper object. + assert wrapper1 == dcargs.extras.from_yaml(Wrapper, dcargs.extras.to_yaml(wrapper1)) From 11bff7cec27e4a76df601bb82598ed9149abe947 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 6 Sep 2022 22:50:00 -0700 Subject: [PATCH 128/491] Helptext improvements, tests, documentation --- dcargs/_docstrings.py | 29 +++++++----- docs/source/helptext_generation.md | 76 ++++++++++++++++++++++++++++++ docs/source/index.rst | 2 +- examples/02_dataclasses.py | 7 ++- tests/test_helptext.py | 52 ++++++++++++++++++++ 5 files changed, 152 insertions(+), 14 deletions(-) create mode 100644 docs/source/helptext_generation.md diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index 492620a63..b236b583d 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -150,6 +150,12 @@ def get_class_tokenization_with_field( def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: """Get docstring for a field in a class.""" + docstring = inspect.getdoc(cls) + if docstring is not None: + for param_doc in docstring_parser.parse(docstring).params: + if param_doc.arg_name == field_name: + return param_doc.description + tokenization = get_class_tokenization_with_field(cls, field_name) if tokenization is None: # Currently only happens for dynamic dataclasses. return None @@ -277,17 +283,16 @@ def get_callable_description(f: Callable) -> str: default_doc = f.__name__ + str(inspect.signature(f)).replace(" -> None", "") if docstring == default_doc: return "" - return docstring - else: - parsed_docstring = docstring_parser.parse(docstring) - return "\n".join( - list( - filter( - lambda x: x is not None, # type: ignore - [ - parsed_docstring.short_description, - parsed_docstring.long_description, - ], - ) + + parsed_docstring = docstring_parser.parse(docstring) + return "\n".join( + list( + filter( + lambda x: x is not None, # type: ignore + [ + parsed_docstring.short_description, + parsed_docstring.long_description, + ], ) ) + ) diff --git a/docs/source/helptext_generation.md b/docs/source/helptext_generation.md new file mode 100644 index 000000000..f341e59d9 --- /dev/null +++ b/docs/source/helptext_generation.md @@ -0,0 +1,76 @@ +# Helptext generation + +In addition to type annotations, :func:`dcargs.cli()` will also parse docstrings +and comments. These are used to automatically generate helptext; see examples +for how these end up being formatted. + +## General callables + +For general callables, field helptext is extracted from the corresponding field +docstring. Our examples use Google-style docstrings, but ReST, Numpydoc-style +and Epydoc docstrings are supported as well. Under the hood, all of these +options use [docstring_parser](https://github.com/rr-/docstring_parser). + +```python +def main( + field1: str, + field2: int = 3, +) -> None: + """Function, whose arguments will be populated from a CLI interface. + + Args: + field1: A string field. + field2: A numeric field, with a default value. + """ + print(field1, field2) +``` + +## Dataclasses, TypedDict, NamedTuple + +For types defined using class attributes, enumerating each argument list in the +class docstring can be cumbersome. + +If they are unavailable, :func:`dcargs.cli` will generate helptext from +docstrings and comments on attributes. These are parsed via source code +inspection. + +**(1) Attribute docstrings** + +As per [PEP 257](https://peps.python.org/pep-0257/#what-is-a-docstring). + +```python +@dataclasses.dataclass +class Args: + field1: str + """A string field.""" + field2: int = 3 + """A numeric field, with a default value.""" +``` + +**(2) Inline comments** + +Inline comments can be more succinct than true attribute docstrings. + +```python +@dataclasses.dataclass +class Args: + field1: str # A string field. + field2: int = 3 # A numeric field, with a default value. +``` + +**(3) Preceding comments** + +These can also be handy for semantically grouping fields, such as the two string +fields below. + +```python +@dataclasses.dataclass +class Args: + # String fields. + field1: str + field2: str + + # An integer field. + # Multi-line comments are supported. + field3: int +``` diff --git a/docs/source/index.rst b/docs/source/index.rst index afa9ad104..901187d25 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -106,7 +106,7 @@ The broader goal is also a replacement for tools like :code:`hydra`, :glob: goals_and_alternatives - serialization + helptext_generation .. toctree:: diff --git a/examples/02_dataclasses.py b/examples/02_dataclasses.py index 4993138bc..6acf06129 100644 --- a/examples/02_dataclasses.py +++ b/examples/02_dataclasses.py @@ -15,7 +15,12 @@ @dataclasses.dataclass class Args: """Description. - This should show up in the helptext!""" + This should show up in the helptext! + + Args: + field1: A string field!!! + field2: An int field!!! + """ field1: str # A string field. field2: int = 3 # A numeric field, with a default value. diff --git a/tests/test_helptext.py b/tests/test_helptext.py index bbfbde399..dd008965f 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -54,6 +54,58 @@ class Helptext: assert "Documentation 3 (default: 3)\n" in helptext +def test_helptext_from_class_docstring(): + @dataclasses.dataclass + class Helptext2: + """This docstring should be printed as a description. + + Attributes: + x: Documentation 1 + y: Documentation 2 + z: Documentation 3 + """ + + x: int + y: Annotated[int, "ignored"] + z: int = 3 + + helptext = _get_helptext(Helptext2) + assert "This docstring should be printed as a description" in helptext + assert "Attributes" not in helptext + assert "x INT" in helptext + assert "y INT" in helptext + assert "z INT" in helptext + assert "Documentation 1 (required)\n" in helptext + assert "Documentation 2 (required)\n" in helptext + assert "Documentation 3 (default: 3)\n" in helptext + + +def test_helptext_from_class_docstring_args(): + @dataclasses.dataclass + class Helptext3: + """This docstring should be printed as a description. + + Args: + x: Documentation 1 + y: Documentation 2 + z: Documentation 3 + """ + + x: int + y: Annotated[int, "ignored"] + z: int = 3 + + helptext = _get_helptext(Helptext3) + assert "This docstring should be printed as a description" in helptext + assert "Args" not in helptext + assert "x INT" in helptext + assert "y INT" in helptext + assert "z INT" in helptext + assert "Documentation 1 (required)\n" in helptext + assert "Documentation 2 (required)\n" in helptext + assert "Documentation 3 (default: 3)\n" in helptext + + def test_helptext_inherited(): class UnrelatedParentClass: pass From b7a3f092b5c68e557e3fee2e61f5912bc47b766f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 7 Sep 2022 01:57:20 -0700 Subject: [PATCH 129/491] Tab completion documentation --- dcargs/_cli.py | 10 ++++- dcargs/extras/_serialization.py | 8 ++-- docs/source/index.rst | 3 +- docs/source/tab_completion.md | 79 +++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 docs/source/tab_completion.md diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 670c4bd92..84a1caee4 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -60,7 +60,7 @@ def cli( `f` is a class, `dcargs.cli()` returns an instance. The parser is generated by populating helptext from docstrings and types from - annotations; a broad range of core type annotations are supported... + annotations; a broad range of core type annotations are supported. - Types natively accepted by `argparse`: str, int, float, pathlib.Path, etc. - Default values for optional parameters. - Booleans, which are automatically converted to flags when provided a default @@ -87,6 +87,9 @@ def cli( - Optional unions over nested structures (optional subparsers). - Generics (including nested generics). + Completion scripts for interactive shells is also provided. To print a script that + can be used for tab completion, pass in `--dcargs-print-completion {bash/zsh/tcsh}`. + Args: f: Callable. prog: The name of the program printed in helptext. Mirrors argument from @@ -155,8 +158,11 @@ def cli( prefix="", # Used for recursive calls. ) - # If we pass in the --dcargs-print-completion flag: turn termcolor off, and ge the + # If we pass in the --dcargs-print-completion flag: turn termcolor off, and get the # shell we want to generate a completion script for (bash/zsh/tcsh). + # + # Note that shtab also offers an add_argument_to() functions that fulfills a similar + # goal, but manual parsing of argv is convenient for turning off colors. args = sys.argv[1:] if args is None else args print_completion = len(args) >= 2 and args[0] == "--dcargs-print-completion" diff --git a/dcargs/extras/_serialization.py b/dcargs/extras/_serialization.py index 333f2b324..9d23f31c2 100644 --- a/dcargs/extras/_serialization.py +++ b/dcargs/extras/_serialization.py @@ -184,8 +184,8 @@ def from_yaml( refactor, while a PyYAML backend enables serialization of arbitrary Python objects. .. warning:: - Serialization functionality is deprecated. It may be removed in a future version - of :code:`dcargs`. + Serialization functionality is stable but deprecated. It may be removed in a + future version of :code:`dcargs`. Args: cls: Type to reconstruct. @@ -213,8 +213,8 @@ def to_yaml(instance: Any) -> str: refactor, while a PyYAML backend enables serialization of arbitrary Python objects. .. warning:: - Serialization functionality is deprecated. It may be removed in a future version - of :code:`dcargs`. + Serialization functionality is stable but deprecated. It may be removed in a + future version of :code:`dcargs`. Args: instance: Dataclass instance to serialize. diff --git a/docs/source/index.rst b/docs/source/index.rst index 901187d25..0ab9f4429 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -105,8 +105,9 @@ The broader goal is also a replacement for tools like :code:`hydra`, :hidden: :glob: - goals_and_alternatives helptext_generation + tab_completion + goals_and_alternatives .. toctree:: diff --git a/docs/source/tab_completion.md b/docs/source/tab_completion.md new file mode 100644 index 000000000..c9fad0a10 --- /dev/null +++ b/docs/source/tab_completion.md @@ -0,0 +1,79 @@ +# Tab completion + +Interfaces built with :func:`dcargs.cli()` can be tab completed in interactive +shells without any source code modification. + +Completion scripts can be generated by passing the +`--dcargs-print-completion {bash/zsh/tcsh}` flag to a dcargs CLI. This generates +a completion script via [shtab](https://docs.iterative.ai/shtab/) and prints it +to stdout. To set up tab completion, the printed script simply needs to be +written somewhere where your shell will find it. + +## Zsh + +For zsh, one option is to emulate the pattern used for completions in +[poetry](https://python-poetry.org/docs): + +``` +# Set up zsh autocompletion for 01_functions.py, which is located in +# dcargs/examples. + +# (1) Make directory for local completions. +mkdir -p ~/.zfunc + +# (2) Write completion script. The name here (_01_functions_py) doesn't matter, +# as long as it's prefixed with an underscore. +python 01_functions.py > ~/.zfunc/_01_functions_py +``` + +And if it's not in your `.zshrc` already: + +``` +# (3) Add .zfunc to our function search path, then initialize completions. +# Ideally this should go in your .zshrc! +fpath+=~/.zfunc +autoload -Uz compinit && compinit +``` + +## Bash + +Local completion scripts for bash can be written as described in the +[bash-complete documentation](https://github.com/scop/bash-completion/blob/master/README.md#FAQ). + +> **Q.** Where should I install my own local completions? +> +> **A.** Put them in the completions subdir of `$BASH_COMPLETION_USER_DIR` +> (defaults to `$XDG_DATA_HOME/bash-completion` or +> `~/.local/share/bash-completion` if `$XDG_DATA_HOME` is not set) to have them +> loaded automatically on demand when the respective command is being completed +> [...] + +Borrowing from the `bash-completion` source[^bash_completion], we can run: + + + +[^bash_completion]: `completion_dir` query is borrowed from [here](https://github.com/scop/bash-completion/blob/966a4e043822613040546e8c003509798c5fae1a/bash_completion#L2440). + + + +```bash +# Set up bash autocompletion for 01_functions.py, which is located in +# dcargs/examples. + +# (1) Find and make completion directory. +completion_dir=${BASH_COMPLETION_USER_DIR:-${XDG_DATA_HOME:-$HOME/.local/share}/bash-completion}/completions/ +mkdir -p $completion_dir + +# (2) Write completion scripts. Note that the name of the completion script must +# match the name of the file. +python 01_functions.py --dcargs-print-completion bash > ${completion_dir}/01_functions.py +``` + +In contrast to zsh, tab completion in bash requires that scripts are either set +up as an entry point or run as :code:`./01_functions.py `, as opposed to +with the `python` command, :code:`python 01_functions.py `. + +Making a script directly executable typically requires: + +1. A permissions update: `chmod +x ./01_functions.py`. +2. A shebang as the first line of your script: `#!/usr/bin/env python` From 016b87bf4b5780f35691563f50bc14962d68139f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 7 Sep 2022 02:36:46 -0700 Subject: [PATCH 130/491] `dcargs.conf` documentation, example --- dcargs/_parsers.py | 4 + dcargs/_strings.py | 12 ++- dcargs/conf/_markers.py | 5 +- dcargs/conf/_subcommands.py | 4 +- docs/source/examples/04_flags.rst | 2 + docs/source/examples/07_positional_args.rst | 2 + docs/source/examples/08_subcommands.rst | 3 + .../examples/16_advanced_configuration.rst | 76 +++++++++++++++++++ examples/02_dataclasses.py | 7 +- examples/04_flags.py | 2 + examples/07_positional_args.py | 2 + examples/08_subcommands.py | 3 + examples/16_advanced_configuration.py | 61 +++++++++++++++ 13 files changed, 170 insertions(+), 13 deletions(-) create mode 100644 docs/source/examples/16_advanced_configuration.rst create mode 100644 examples/16_advanced_configuration.py diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index d29f03d89..31e3ca0e7 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -336,7 +336,11 @@ def from_field( if default_name not in parser_from_name: # If we can't find the subparser by name, search by type. This is needed # when the user renames their subcommands. (eg via dcargs.subcommand) + # + # TODO: this will display some weird behaviors if multiple subcommands + # have the same type. default_name = None + for name, parser in parser_from_name.items(): if type(field.default) is _resolver.unwrap_origin_strip_extras( parser.f diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 15e5bfcd9..13e58face 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -62,13 +62,19 @@ def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: ) # Subparser name from `dcargs.metadata.subcommand()`. + found_name = None + prefix_name = True if len(found_subcommand_configs) > 0: - return found_subcommand_configs[0].name, found_subcommand_configs[0].prefix_name + found_name = found_subcommand_configs[0].name + prefix_name = found_subcommand_configs[0].prefix_name + + if found_name is not None: + return found_name, prefix_name # Subparser name from class name. if len(type_from_typevar) == 0: assert hasattr(cls, "__name__") - return hyphen_separated_from_camel_case(cls.__name__), True # type: ignore + return hyphen_separated_from_camel_case(cls.__name__), prefix_name # type: ignore return ( "-".join( @@ -77,7 +83,7 @@ def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: [cls] + list(type_from_typevar.values()), ) ), - True, + prefix_name, ) diff --git a/dcargs/conf/_markers.py b/dcargs/conf/_markers.py index 3e6100651..6aa83f9c9 100644 --- a/dcargs/conf/_markers.py +++ b/dcargs/conf/_markers.py @@ -30,8 +30,9 @@ def __repr__(self): FIXED = _make_marker("Fixed") Fixed = Annotated[T, FIXED] -"""A type `T` can be annotated as `Fixed[T]` to prevent `dcargs.cli` from parsing it. A -default value should be set instead.""" +"""A type `T` can be annotated as `Fixed[T]` to prevent `dcargs.cli` from parsing it; a +default value should be set instead. Note that fields with defaults that can't be parsed +will also be marked as fixed automatically.""" FLAG_CONVERSION_OFF = _make_marker("FlagConversionOff") FlagConversionOff = Annotated[T, FLAG_CONVERSION_OFF] diff --git a/dcargs/conf/_subcommands.py b/dcargs/conf/_subcommands.py index 03d99661a..92f84b030 100644 --- a/dcargs/conf/_subcommands.py +++ b/dcargs/conf/_subcommands.py @@ -6,7 +6,7 @@ @dataclasses.dataclass(frozen=True) class _SubcommandConfiguration: - name: str + name: Optional[str] default: Any description: Optional[str] prefix_name: bool @@ -16,7 +16,7 @@ def __hash__(self) -> int: def subcommand( - name: str, + name: Optional[str] = None, *, default: Any = MISSING_NONPROP, description: Optional[str] = None, diff --git a/docs/source/examples/04_flags.rst b/docs/source/examples/04_flags.rst index ceece83d1..52de2c13e 100644 --- a/docs/source/examples/04_flags.rst +++ b/docs/source/examples/04_flags.rst @@ -8,6 +8,8 @@ Booleans can either be expected to be explicitly passed in, or, if given a default value, automatically converted to flags. +To turn off conversion, see :func:`dcargs.conf.FlagConversionOff`. + .. code-block:: python diff --git a/docs/source/examples/07_positional_args.rst b/docs/source/examples/07_positional_args.rst index 270c9b9df..0e508d82d 100644 --- a/docs/source/examples/07_positional_args.rst +++ b/docs/source/examples/07_positional_args.rst @@ -7,6 +7,8 @@ Positional-only arguments in functions are converted to positional CLI arguments. +For more general positional arguments, see :func:`dcargs.conf.Positional`. + .. code-block:: python diff --git a/docs/source/examples/08_subcommands.rst b/docs/source/examples/08_subcommands.rst index 841b408f7..0ce749ef3 100644 --- a/docs/source/examples/08_subcommands.rst +++ b/docs/source/examples/08_subcommands.rst @@ -7,6 +7,9 @@ Unions over nested types (classes or dataclasses) are populated using subcommands. +For configuring subcommands beyond what can be expressed with type annotations, see +:func:`dcargs.conf.subcommand()`. + .. code-block:: python diff --git a/docs/source/examples/16_advanced_configuration.rst b/docs/source/examples/16_advanced_configuration.rst new file mode 100644 index 000000000..fdef0bc8b --- /dev/null +++ b/docs/source/examples/16_advanced_configuration.rst @@ -0,0 +1,76 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +16. Advanced Configuration +========================================== + + +The :mod:`dcargs.conf` module contains utilities that can be used to configure +command-line interfaces beyond what is expressible via static type annotations. + +Features here are supported, but generally unnecessary and should be used sparingly. + + + +.. code-block:: python + :linenos: + + import dataclasses + from typing import Union + + from typing_extensions import Annotated + + import dcargs + + + @dataclasses.dataclass(frozen=True) + class CheckoutArgs: + """Checkout a branch.""" + + branch: str + + + @dataclasses.dataclass(frozen=True) + class CommitArgs: + """Commit changes.""" + + message: str + all: bool = False + + + @dataclasses.dataclass + class Args: + # A boolean field with flag conversion turned off. + boolean: dcargs.conf.FlagConversionOff[bool] = False + + # A numeric field parsed as a positional argument. + positional: dcargs.conf.Positional[int] = 3 + + # A numeric field that can't be changed via the CLI. + fixed: dcargs.conf.Fixed[int] = 5 + + # A union over nested structures, but without subcommand generation. When a default + # is provided, the type is simply fixed to that default. + union_without_subcommand: dcargs.conf.AvoidSubcommands[ + Union[CheckoutArgs, CommitArgs] + ] = CheckoutArgs("main") + + # `dcargs.conf.subcommand()` can be used to configure subcommands in a Union. + renamed_subcommand: Union[ + Annotated[ + CheckoutArgs, dcargs.conf.subcommand(name="checkout", prefix_name=False) + ], + Annotated[CommitArgs, dcargs.conf.subcommand(name="commit", prefix_name=False)], + ] = CheckoutArgs("main") + + + if __name__ == "__main__": + print(dcargs.cli(Args)) + +------------ + +.. raw:: html + + python 16_advanced_configuration.py --help + +.. program-output:: python ../../examples/16_advanced_configuration.py --help diff --git a/examples/02_dataclasses.py b/examples/02_dataclasses.py index 6acf06129..4993138bc 100644 --- a/examples/02_dataclasses.py +++ b/examples/02_dataclasses.py @@ -15,12 +15,7 @@ @dataclasses.dataclass class Args: """Description. - This should show up in the helptext! - - Args: - field1: A string field!!! - field2: An int field!!! - """ + This should show up in the helptext!""" field1: str # A string field. field2: int = 3 # A numeric field, with a default value. diff --git a/examples/04_flags.py b/examples/04_flags.py index 233e5fb48..635acea2e 100644 --- a/examples/04_flags.py +++ b/examples/04_flags.py @@ -1,6 +1,8 @@ """Booleans can either be expected to be explicitly passed in, or, if given a default value, automatically converted to flags. +To turn off conversion, see :func:`dcargs.conf.FlagConversionOff`. + Usage: `python ./04_flags.py --help` `python ./04_flags.py --boolean True` diff --git a/examples/07_positional_args.py b/examples/07_positional_args.py index 317eccaa1..fc81f7126 100644 --- a/examples/07_positional_args.py +++ b/examples/07_positional_args.py @@ -1,5 +1,7 @@ """Positional-only arguments in functions are converted to positional CLI arguments. +For more general positional arguments, see :func:`dcargs.conf.Positional`. + Usage: `python ./07_positional_args.py --help` `python ./07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5` diff --git a/examples/08_subcommands.py b/examples/08_subcommands.py index dba66b898..7ae6b5db9 100644 --- a/examples/08_subcommands.py +++ b/examples/08_subcommands.py @@ -1,5 +1,8 @@ """Unions over nested types (classes or dataclasses) are populated using subcommands. +For configuring subcommands beyond what can be expressed with type annotations, see +:func:`dcargs.conf.subcommand()`. + Usage: `python ./08_subcommands.py --help` `python ./08_subcommands.py cmd:commit --help` diff --git a/examples/16_advanced_configuration.py b/examples/16_advanced_configuration.py new file mode 100644 index 000000000..8f0c58151 --- /dev/null +++ b/examples/16_advanced_configuration.py @@ -0,0 +1,61 @@ +"""The :mod:`dcargs.conf` module contains utilities that can be used to configure +command-line interfaces beyond what is expressible via static type annotations. + +Features here are supported, but generally unnecessary and should be used sparingly. + +Usage: +`python ./16_advanced_configuration.py --help` +""" + +import dataclasses +from typing import Union + +from typing_extensions import Annotated + +import dcargs + + +@dataclasses.dataclass(frozen=True) +class CheckoutArgs: + """Checkout a branch.""" + + branch: str + + +@dataclasses.dataclass(frozen=True) +class CommitArgs: + """Commit changes.""" + + message: str + all: bool = False + + +@dataclasses.dataclass +class Args: + # A boolean field with flag conversion turned off. + boolean: dcargs.conf.FlagConversionOff[bool] = False + + # A numeric field parsed as a positional argument. + positional: dcargs.conf.Positional[int] = 3 + + # A numeric field that can't be changed via the CLI. + fixed: dcargs.conf.Fixed[int] = 5 + + # A union over nested structures, but without subcommand generation. When a default + # is provided, the type is simply fixed to that default. + union_without_subcommand: dcargs.conf.AvoidSubcommands[ + Union[CheckoutArgs, CommitArgs] + ] = CheckoutArgs("main") + + # `dcargs.conf.subcommand()` can be used to configure subcommands in a Union. Here, + # we make the subcommand names more succinct. + renamed_subcommand: Union[ + Annotated[ + CheckoutArgs, dcargs.conf.subcommand(name="checkout", prefix_name=False) + ], + Annotated[CommitArgs, dcargs.conf.subcommand(name="commit", prefix_name=False)], + ] = CheckoutArgs("main") + + +if __name__ == "__main__": + print(dcargs.cli(Args)) From 5833870895f6ffacd305912facc38e54b4192518 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 7 Sep 2022 13:35:36 -0700 Subject: [PATCH 131/491] Docs polish --- docs/source/examples/16_advanced_configuration.rst | 3 ++- docs/source/tab_completion.md | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/source/examples/16_advanced_configuration.rst b/docs/source/examples/16_advanced_configuration.rst index fdef0bc8b..12c27e8c8 100644 --- a/docs/source/examples/16_advanced_configuration.rst +++ b/docs/source/examples/16_advanced_configuration.rst @@ -55,7 +55,8 @@ Features here are supported, but generally unnecessary and should be used sparin Union[CheckoutArgs, CommitArgs] ] = CheckoutArgs("main") - # `dcargs.conf.subcommand()` can be used to configure subcommands in a Union. + # `dcargs.conf.subcommand()` can be used to configure subcommands in a Union. Here, + # we make the subcommand names more succinct. renamed_subcommand: Union[ Annotated[ CheckoutArgs, dcargs.conf.subcommand(name="checkout", prefix_name=False) diff --git a/docs/source/tab_completion.md b/docs/source/tab_completion.md index c9fad0a10..559e693d5 100644 --- a/docs/source/tab_completion.md +++ b/docs/source/tab_completion.md @@ -14,7 +14,7 @@ written somewhere where your shell will find it. For zsh, one option is to emulate the pattern used for completions in [poetry](https://python-poetry.org/docs): -``` +```bash # Set up zsh autocompletion for 01_functions.py, which is located in # dcargs/examples. @@ -23,12 +23,12 @@ mkdir -p ~/.zfunc # (2) Write completion script. The name here (_01_functions_py) doesn't matter, # as long as it's prefixed with an underscore. -python 01_functions.py > ~/.zfunc/_01_functions_py +python 01_functions.py --dcargs-print-completion zsh > ~/.zfunc/_01_functions_py ``` And if it's not in your `.zshrc` already: -``` +```bash # (3) Add .zfunc to our function search path, then initialize completions. # Ideally this should go in your .zshrc! fpath+=~/.zfunc From b65d1f151c7289030b590c15a07d2d787f39fbf6 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 7 Sep 2022 13:59:46 -0700 Subject: [PATCH 132/491] Version bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9e382d904..f157f4aa8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.2.8" +version = "0.3.0" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From a24b209cd97c56248cebda77b331c8453f3a9a3e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 7 Sep 2022 16:35:50 -0700 Subject: [PATCH 133/491] Final tweaks for 0.3.0 - Make --field_name and --field-name interchangeable - Add prefix_names option to extras.subcommand_type_from_defaults() --- dcargs/_cli.py | 16 +++++++++++++++- dcargs/_strings.py | 2 +- dcargs/extras/__init__.py | 4 ++-- dcargs/extras/_base_configs.py | 28 +++++++++++++++++++++------- examples/05_hierarchical_configs.py | 2 +- examples/10_base_configs.py | 4 ++-- tests/test_nested.py | 25 +++++++++++++++++++++++++ tests/test_union_from_mapping.py | 4 ++-- 8 files changed, 69 insertions(+), 16 deletions(-) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 84a1caee4..119667370 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -158,12 +158,26 @@ def cli( prefix="", # Used for recursive calls. ) + # Read and fix arguments. If the user passes in --field_name instead of + # --field-name, correct for them. + args = sys.argv[1:] if args is None else args + + def fix_arg(arg: str) -> str: + if not arg.startswith("--"): + return arg + if "=" in arg: + arg, _, val = arg.partition("=") + return arg.replace("_", "-") + "=" + val + else: + return arg.replace("_", "-") + + args = list(map(fix_arg, args)) + # If we pass in the --dcargs-print-completion flag: turn termcolor off, and get the # shell we want to generate a completion script for (bash/zsh/tcsh). # # Note that shtab also offers an add_argument_to() functions that fulfills a similar # goal, but manual parsing of argv is convenient for turning off colors. - args = sys.argv[1:] if args is None else args print_completion = len(args) >= 2 and args[0] == "--dcargs-print-completion" formatting_context = _argparse_formatter.ansi_context() diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 13e58face..6ecbc7b27 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -9,7 +9,7 @@ from . import _resolver -dummy_field_name = "__dcargs_dummy_field_name__" +dummy_field_name = "__dcargs_dummy_field__" def _strip_dummy_field_names(parts: Iterable[str]) -> Iterable[str]: diff --git a/dcargs/extras/__init__.py b/dcargs/extras/__init__.py index 9671a9571..d51a0b898 100644 --- a/dcargs/extras/__init__.py +++ b/dcargs/extras/__init__.py @@ -1,7 +1,7 @@ """The :mod:`dcargs.extras` submodule contains helpers that complement :func:`dcargs.cli()`, but aren't considered part of the core interface.""" -from ._base_configs import subcommand_union_from_mapping +from ._base_configs import subcommand_type_from_defaults from ._serialization import from_yaml, to_yaml -__all__ = ["subcommand_union_from_mapping", "to_yaml", "from_yaml"] +__all__ = ["subcommand_type_from_defaults", "to_yaml", "from_yaml"] diff --git a/dcargs/extras/_base_configs.py b/dcargs/extras/_base_configs.py index 1159df5d9..a178dca3b 100644 --- a/dcargs/extras/_base_configs.py +++ b/dcargs/extras/_base_configs.py @@ -7,12 +7,18 @@ T = TypeVar("T") -def subcommand_union_from_mapping( - default_from_name: Mapping[str, T], descriptions: Mapping[str, str] = {} +def subcommand_type_from_defaults( + defaults: Mapping[str, T], + descriptions: Mapping[str, str] = {}, + *, + prefix_names: bool = True, ) -> Type[T]: - """Returns a Union type for defining subcommands that choose between nested types. + """Construct a Union type for defining subcommands that choose between defaults. - For example, when `default` is set to: + This can most commonly be used to create a "base configuration" pattern: + https://brentyi.github.io/dcargs/examples/10_base_configs/ + + For example, when `defaults` is set to: ```python { @@ -36,7 +42,7 @@ def subcommand_union_from_mapping( ] ``` - This can be used directly in dcargs.cli: + The resulting type can be used directly in dcargs.cli: ```python config = dcargs.cli(subcommand_union_from_mapping(default_from_name)) @@ -70,8 +76,16 @@ def train( return Union.__getitem__( # type: ignore tuple( Annotated.__class_getitem__( # type: ignore - (type(v), subcommand(k, default=v, description=descriptions.get(k))) + ( + type(v), + subcommand( + k, + default=v, + description=descriptions.get(k), + prefix_name=prefix_names, + ), + ) ) - for k, v in default_from_name.items() + for k, v in defaults.items() ) ) diff --git a/examples/05_hierarchical_configs.py b/examples/05_hierarchical_configs.py index c0a1be98c..4f696c366 100644 --- a/examples/05_hierarchical_configs.py +++ b/examples/05_hierarchical_configs.py @@ -34,7 +34,7 @@ class OptimizerConfig: @dataclasses.dataclass class ExperimentConfig: # Various configurable options for our optimizer. - optimizer: OptimizerConfig + optimizer_config: OptimizerConfig # Batch size. batch_size: int = 32 diff --git a/examples/10_base_configs.py b/examples/10_base_configs.py index e87bf945c..b994bac60 100644 --- a/examples/10_base_configs.py +++ b/examples/10_base_configs.py @@ -97,9 +97,9 @@ class ExperimentConfig: if __name__ == "__main__": config = dcargs.cli( - dcargs.extras.subcommand_union_from_mapping(base_configs, descriptions), + dcargs.extras.subcommand_type_from_defaults(base_configs, descriptions), ) - # Note that this is equivalent to: + # ^Note that this is equivalent to: # # config = dcargs.cli( # Union[ diff --git a/tests/test_nested.py b/tests/test_nested.py index db1e5e1b1..982ead888 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -38,6 +38,31 @@ class Nested: dcargs.cli(Nested, args=["--x", "1"]) +def test_nested_accidental_underscores(): + @dataclasses.dataclass + class B: + arg_name: str + + @dataclasses.dataclass + class Nested: + x: int + child_struct: B + + assert ( + dcargs.cli(Nested, args=["--x", "1", "--child-struct.arg-name", "three_five"]) + == dcargs.cli( + Nested, args=["--x", "1", "--child_struct.arg_name", "three_five"] + ) + == dcargs.cli( + Nested, args=["--x", "1", "--child_struct.arg-name", "three_five"] + ) + == dcargs.cli(Nested, args=["--x", "1", "--child_struct.arg_name=three_five"]) + == Nested(x=1, child_struct=B(arg_name="three_five")) + ) + with pytest.raises(SystemExit): + dcargs.cli(Nested, args=["--x", "1"]) + + def test_nested_default(): @dataclasses.dataclass class B: diff --git a/tests/test_union_from_mapping.py b/tests/test_union_from_mapping.py index 14142f4b3..7e9d35ee4 100644 --- a/tests/test_union_from_mapping.py +++ b/tests/test_union_from_mapping.py @@ -15,7 +15,7 @@ def test_union_from_mapping(): "two": A(2), "three": A(3), } - ConfigUnion = dcargs.extras.subcommand_union_from_mapping(base_configs) + ConfigUnion = dcargs.extras.subcommand_type_from_defaults(base_configs) assert dcargs.cli(ConfigUnion, args="one".split(" ")) == A(1) assert dcargs.cli(ConfigUnion, args="two".split(" ")) == A(2) @@ -32,7 +32,7 @@ def test_union_from_mapping_in_function(): # Hack for mypy. Not needed for pyright. ConfigUnion = A - ConfigUnion = dcargs.extras.subcommand_union_from_mapping(base_configs) # type: ignore + ConfigUnion = dcargs.extras.subcommand_type_from_defaults(base_configs) # type: ignore def main(config: ConfigUnion, flag: bool = False) -> Optional[A]: if flag: From acf19dd5e1083469f3fb2093a619dd028e1db827 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 7 Sep 2022 17:16:49 -0700 Subject: [PATCH 134/491] Minor: ignore docstring for subcommand_type_from_defaults() --- dcargs/extras/_base_configs.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dcargs/extras/_base_configs.py b/dcargs/extras/_base_configs.py index a178dca3b..a0984c819 100644 --- a/dcargs/extras/_base_configs.py +++ b/dcargs/extras/_base_configs.py @@ -81,7 +81,7 @@ def train( subcommand( k, default=v, - description=descriptions.get(k), + description=descriptions.get(k, ""), prefix_name=prefix_names, ), ) diff --git a/pyproject.toml b/pyproject.toml index f157f4aa8..4782fbde3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.0" +version = "0.3.1" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From 49ff3af56eb831b73a8cb04919c3ea46b7599432 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 9 Sep 2022 16:06:28 -0700 Subject: [PATCH 135/491] Remove special-casing for positional metavars --- dcargs/_arguments.py | 25 ++++++------------------- pyproject.toml | 2 +- tests/test_positional_ignore_py37.py | 9 --------- 3 files changed, 7 insertions(+), 29 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index dfb8d08d6..73ddb0ef9 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -260,10 +260,6 @@ def _rule_generate_helptext( # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string docstring_help = docstring_help.replace("%", "%%") help_parts.append(docstring_help) - elif arg.field.is_positional() and arg.field.name != _strings.dummy_field_name: - # Place the type in the helptext. Note that we skip this for dummy fields, which - # will still have the type in the metavar. - help_parts.append(str(lowered.metavar)) default = lowered.default if lowered.is_fixed(): @@ -334,28 +330,19 @@ def _rule_positional_special_handling( if not arg.field.is_positional(): return lowered - metavar = _strings.make_field_name([arg.prefix, arg.field.name]).upper() - if lowered.nargs == "+": - metavar = f"{metavar} [{metavar} ...]" - elif isinstance(lowered.nargs, int): - metavar = " ".join((metavar,) * lowered.nargs) - if lowered.required: nargs = lowered.nargs + elif lowered.nargs == 1: + # Optional positional arguments. Note that this needs to be special-cased in + # _calling.py. + nargs = "?" else: - metavar = "[" + metavar + "]" - if lowered.nargs == 1: - # Optional positional arguments. Note that this needs to be special-cased in - # _calling.py. - nargs = "?" - else: - # If lowered.nargs is either + or an int. - nargs = "*" + # If lowered.nargs is either + or an int. + nargs = "*" return dataclasses.replace( lowered, dest=None, required=None, # Can't be passed in for positionals. - metavar=metavar if len(metavar) > 0 else lowered.metavar, nargs=nargs, ) diff --git a/pyproject.toml b/pyproject.toml index 4782fbde3..a4734c23a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.1" +version = "0.3.2" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] diff --git a/tests/test_positional_ignore_py37.py b/tests/test_positional_ignore_py37.py index 5c6af1a59..0c68a7e5c 100644 --- a/tests/test_positional_ignore_py37.py +++ b/tests/test_positional_ignore_py37.py @@ -1,5 +1,3 @@ -import contextlib -import io from typing import List, Optional, Tuple import pytest @@ -48,13 +46,6 @@ def nest1(a: int, b: int, thing: A, /, c: int) -> A: with pytest.raises(SystemExit): dcargs.cli(nest1, args="0 1 2 3 4 --thing.c 4 --c 4".split(" ")) - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - dcargs.cli(nest1, args=["--help"]) - helptext = f.getvalue() - assert "THING.HELLO_WORLD" in helptext - def test_nested_positional_alt(): class B: From 1dcc10a47250469c83339597de2b29ca13c53164 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 9 Sep 2022 17:17:09 -0700 Subject: [PATCH 136/491] Corner case fix for literals in sequences in positionals :grimacing: --- dcargs/_arguments.py | 34 ++++++++++++++++++++++++++++------ tests/test_collections.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 73ddb0ef9..def658eaa 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -34,6 +34,20 @@ from backports.cached_property import cached_property # type: ignore +class _PatchedList(list): + """Custom tuple type, for avoiding "default not in choices" errors when the default + is set to MISSING_NONPROP. + + This solves a choices error raised by argparse in a very specific edge case: + literals in containers as positional arguments.""" + + def __init__(self, li): + super(_PatchedList, self).__init__(li) + + def __contains__(self, x: Any) -> bool: + return list.__contains__(self, x) or x is _fields.MISSING_NONPROP + + @dataclasses.dataclass(frozen=True) class ArgumentDefinition: """Structure containing everything needed to define an argument.""" @@ -61,6 +75,9 @@ def add_argument( # the field default to a string format, then back to the desired type. kwargs["default"] = _fields.MISSING_NONPROP + if "choices" in kwargs: + kwargs["choices"] = _PatchedList(kwargs["choices"]) + # Note that the name must be passed in as a position argument. parser.add_argument(name_or_flag, **kwargs) @@ -330,19 +347,24 @@ def _rule_positional_special_handling( if not arg.field.is_positional(): return lowered + metavar = lowered.metavar if lowered.required: nargs = lowered.nargs - elif lowered.nargs == 1: - # Optional positional arguments. Note that this needs to be special-cased in - # _calling.py. - nargs = "?" else: - # If lowered.nargs is either + or an int. - nargs = "*" + if metavar is not None: + metavar = "[" + metavar + "]" + if lowered.nargs == 1: + # Optional positional arguments. Note that this needs to be special-cased in + # _calling.py. + nargs = "?" + else: + # If lowered.nargs is either + or an int. + nargs = "*" return dataclasses.replace( lowered, dest=None, required=None, # Can't be passed in for positionals. + metavar=metavar, nargs=nargs, ) diff --git a/tests/test_collections.py b/tests/test_collections.py index b6970cbfc..6145ee693 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -1,6 +1,8 @@ import collections +import contextlib import dataclasses import enum +import io from typing import ( Any, Deque, @@ -55,6 +57,33 @@ class A: dcargs.cli(A, args=["--x"]) +def test_tuple_with_literal_and_default(): + @dataclasses.dataclass + class A: + x: Tuple[Literal[1, 2, 3], ...] = (1, 2) + + assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + assert dcargs.cli(A, args=[]) == A(x=(1, 2)) + with pytest.raises(SystemExit): + dcargs.cli(A, args=["--x", "1", "2", "3", "4"]) + with pytest.raises(SystemExit): + dcargs.cli(A, args=["--x"]) + + +def test_positional_tuple_with_literal_and_default(): + @dataclasses.dataclass + class A: + x: dcargs.conf.Positional[Tuple[Literal[1, 2, 3], ...]] = (1, 2) + + assert dcargs.cli(A, args=["1", "2", "3"]) == A(x=(1, 2, 3)) + assert dcargs.cli(A, args=[]) == A(x=(1, 2)) + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): + dcargs.cli(A, args=["1", "2", "3", "4"]) + assert "invalid choice" in target.getvalue() + + def test_tuples_fixed_multitype(): @dataclasses.dataclass class A: From 2aedb648e291c6e7ebe238e58b1072471c705580 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 11 Sep 2022 22:23:58 -0700 Subject: [PATCH 137/491] Mirror `shtab`, dcargs.conf.Suppress[], tests --- dcargs/_arguments.py | 11 +- dcargs/_cli.py | 6 +- dcargs/_parsers.py | 31 +- dcargs/_shtab/LICENCE | 13 + dcargs/_shtab/README.md | 6 + dcargs/_shtab/__init__.py | 962 +++++++++++++++++++++++++++++++++++++ dcargs/_shtab/__main__.py | 10 + dcargs/_shtab/_dist_ver.py | 1 + dcargs/_shtab/main.py | 77 +++ dcargs/_shtab/py.typed | 2 + dcargs/_strings.py | 12 +- dcargs/conf/__init__.py | 3 +- dcargs/conf/_markers.py | 5 + pyproject.toml | 7 +- tests/test_helptext.py | 14 + tests/test_strings.py | 16 +- 16 files changed, 1154 insertions(+), 22 deletions(-) create mode 100644 dcargs/_shtab/LICENCE create mode 100644 dcargs/_shtab/README.md create mode 100644 dcargs/_shtab/__init__.py create mode 100644 dcargs/_shtab/__main__.py create mode 100644 dcargs/_shtab/_dist_ver.py create mode 100644 dcargs/_shtab/main.py create mode 100644 dcargs/_shtab/py.typed diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index def658eaa..8c0cc9e3c 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -268,6 +268,11 @@ def _rule_generate_helptext( lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: """Generate helptext from docstring, argument name, default values.""" + + # If the suppress marker is attached, hide the argument. + if _markers.SUPPRESS in arg.field.markers: + return dataclasses.replace(lowered, help=argparse.SUPPRESS) + help_parts = [] docstring_help = arg.field.helptext @@ -327,11 +332,9 @@ def _rule_set_name_or_flag( elif lowered.action == "store_false": name_or_flag = "--" + _strings.make_field_name( [arg.prefix, "no-" + arg.field.name] - ).replace("_", "-") + ) else: - name_or_flag = "--" + _strings.make_field_name( - [arg.prefix, arg.field.name] - ).replace("_", "-") + name_or_flag = "--" + _strings.make_field_name([arg.prefix, arg.field.name]) return dataclasses.replace( lowered, diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 119667370..aa9a345cf 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -5,9 +5,9 @@ import warnings from typing import Callable, Optional, Sequence, Type, TypeVar, Union, cast, overload -import shtab - -from . import _argparse_formatter, _calling, _fields, _parsers, _strings, conf +from . import _argparse_formatter, _calling, _fields, _parsers +from . import _shtab as shtab +from . import _strings, conf OutT = TypeVar("OutT") diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 31e3ca0e7..c2b7585e0 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -147,11 +147,11 @@ def from_callable( if field.helptext is not None: helptext_from_nested_class_field_name[ - field.name + _strings.make_field_name([field.name]) ] = field.helptext else: helptext_from_nested_class_field_name[ - field.name + _strings.make_field_name([field.name]) ] = _docstrings.get_callable_description(field.typ) continue @@ -207,20 +207,33 @@ def format_group_name(nested_field_name: str) -> str: ) parser._action_groups = parser._action_groups[::-1] - # Add each argument. + # Add each argument group. Note that groups with only suppressed arguments won't + # be added. for arg in self.args: - if arg.field.is_positional(): - arg.add_argument(positional_group) - continue - - if arg.prefix not in group_from_prefix: + if ( + arg.lowered.help is not argparse.SUPPRESS + and arg.prefix not in group_from_prefix + ): group_from_prefix[arg.prefix] = parser.add_argument_group( format_group_name(arg.prefix), description=self.helptext_from_nested_class_field_name.get( arg.prefix ), ) - arg.add_argument(group_from_prefix[arg.prefix]) + + # Add each argument. + for arg in self.args: + if arg.field.is_positional(): + arg.add_argument(positional_group) + continue + + if arg.prefix in group_from_prefix: + arg.add_argument(group_from_prefix[arg.prefix]) + else: + # Suppressed argument: still need to add them, but they won't show up in + # the helptext so it doesn't matter which group. + assert arg.lowered.help is argparse.SUPPRESS + arg.add_argument(group_from_prefix[""]) # Create subparser tree. if len(self.subparsers_from_name) > 0: diff --git a/dcargs/_shtab/LICENCE b/dcargs/_shtab/LICENCE new file mode 100644 index 000000000..aa47a9215 --- /dev/null +++ b/dcargs/_shtab/LICENCE @@ -0,0 +1,13 @@ +Copyright 2020-2021 Casper da Costa-Luis + +Licensed under the Apache Licence, Version 2.0 (the "Licence"); +you may not use this project except in compliance with the Licence. +You may obtain a copy of the Licence at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the Licence is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the Licence for the specific language governing permissions and +limitations under the Licence. diff --git a/dcargs/_shtab/README.md b/dcargs/_shtab/README.md new file mode 100644 index 000000000..f637e63d6 --- /dev/null +++ b/dcargs/_shtab/README.md @@ -0,0 +1,6 @@ +Temporary copy of `shtab` that incorporates two PRs: + +- https://github.com/iterative/shtab/pull/106 +- https://github.com/iterative/shtab/pull/108 + +Can be removed when these are merged. diff --git a/dcargs/_shtab/__init__.py b/dcargs/_shtab/__init__.py new file mode 100644 index 000000000..86d8b8e58 --- /dev/null +++ b/dcargs/_shtab/__init__.py @@ -0,0 +1,962 @@ +from __future__ import print_function + +import logging +import re +import sys +from argparse import ( + SUPPRESS, + Action, + ArgumentParser, + _AppendAction, + _AppendConstAction, + _CountAction, + _HelpAction, + _StoreConstAction, + _VersionAction, +) +from collections import defaultdict +from functools import total_ordering +from itertools import starmap +from string import Template +from typing import Any, Dict, List +from typing import Optional as Opt +from typing import Union + +# version detector. Precedence: installed dist, git, 'UNKNOWN' +try: + from ._dist_ver import __version__ +except ImportError: + try: + from setuptools_scm import get_version + + __version__ = get_version(root="..", relative_to=__file__) + except (ImportError, LookupError): + __version__ = "UNKNOWN" +__all__ = [ + "complete", + "add_argument_to", + "SUPPORTED_SHELLS", + "FILE", + "DIRECTORY", + "DIR", +] +log = logging.getLogger(__name__) + +SUPPORTED_SHELLS: List[str] = [] +_SUPPORTED_COMPLETERS = {} +CHOICE_FUNCTIONS: Dict[str, Dict[str, str]] = { + "file": {"bash": "_shtab_compgen_files", "zsh": "_files", "tcsh": "f"}, + "directory": {"bash": "_shtab_compgen_dirs", "zsh": "_files -/", "tcsh": "d"}, +} +FILE = CHOICE_FUNCTIONS["file"] +DIRECTORY = DIR = CHOICE_FUNCTIONS["directory"] +FLAG_OPTION = ( + _StoreConstAction, + _HelpAction, + _VersionAction, + _AppendConstAction, + _CountAction, +) +OPTION_END = _HelpAction, _VersionAction +OPTION_MULTI = _AppendAction, _AppendConstAction, _CountAction +RE_ZSH_SPECIAL_CHARS = re.compile(r"([^\w\s.,()-])") # excessive but safe + + +def mark_completer(shell): + def wrapper(func): + if shell not in SUPPORTED_SHELLS: + SUPPORTED_SHELLS.append(shell) + _SUPPORTED_COMPLETERS[shell] = func + return func + + return wrapper + + +def get_completer(shell): + try: + return _SUPPORTED_COMPLETERS[shell] + except KeyError: + raise NotImplementedError( + "shell (%s) must be in {%s}" % (shell, ",".join(SUPPORTED_SHELLS)) + ) + + +@total_ordering +class Choice(object): + """ + Placeholder to mark a special completion ``. + + >>> ArgumentParser.add_argument(..., choices=[Choice("")]) + """ + + def __init__(self, choice_type: str, required: bool = False) -> None: + """ + See below for parameters. + + choice_type : internal `type` name + required : controls result of comparison to empty strings + """ + self.required = required + self.type = choice_type + + def __repr__(self) -> str: + return self.type + ("" if self.required else "?") + + def __cmp__(self, other: object) -> int: + if self.required: + return 0 if other else -1 + return 0 + + def __eq__(self, other: object) -> bool: + return self.__cmp__(other) == 0 + + def __lt__(self, other: object) -> bool: + return self.__cmp__(other) < 0 + + +class Optional(object): + """Example: `ArgumentParser.add_argument(..., choices=Optional.FILE)`.""" + + FILE = [Choice("file")] + DIR = DIRECTORY = [Choice("directory")] + + +class Required(object): + """Example: `ArgumentParser.add_argument(..., choices=Required.FILE)`.""" + + FILE = [Choice("file", True)] + DIR = DIRECTORY = [Choice("directory", True)] + + +def complete2pattern(opt_complete, shell, choice_type2fn) -> bool: + return ( + opt_complete.get(shell, "") + if isinstance(opt_complete, dict) + else choice_type2fn[opt_complete] + ) + + +def wordify(string: str) -> str: + """Replace non-word chars [-. ] with underscores [_]""" + return string.replace("-", "_").replace(".", " ").replace(" ", "_") + + +def get_public_subcommands(sub): + """Get all the publicly-visible subcommands for a given subparser.""" + public_parsers = {id(sub.choices[i.dest]) for i in sub._get_subactions()} + return {k for k, v in sub.choices.items() if id(v) in public_parsers} + + +def get_bash_commands(root_parser, root_prefix, choice_functions=None): + """ + Recursive subcommand parser traversal, returning lists of information on + commands (formatted for output to the completions script). + printing bash helper syntax. + + Returns: + subparsers : list of subparsers for each parser + option_strings : list of options strings for each parser + compgens : list of shtab `.complete` functions corresponding to actions + choices : list of choices corresponding to actions + nargs : list of number of args allowed for each action (if not 0 or 1) + """ + choice_type2fn = {k: v["bash"] for k, v in CHOICE_FUNCTIONS.items()} + if choice_functions: + choice_type2fn.update(choice_functions) + + def get_option_strings(parser): + """Flattened list of all `parser`'s option strings.""" + return sum( + ( + opt.option_strings + for opt in parser._get_optional_actions() + if opt.help != SUPPRESS + ), + [], + ) + + def recurse(parser, prefix): + """recurse through subparsers, appending to the return lists""" + subparsers = [] + option_strings = [] + compgens = [] + choices = [] + nargs = [] + + # temp lists for recursion results + sub_subparsers = [] + sub_option_strings = [] + sub_compgens = [] + sub_choices = [] + sub_nargs = [] + + # positional arguments + discovered_subparsers = [] + for i, positional in enumerate(parser._get_positional_actions()): + if positional.help == SUPPRESS: + continue + + if hasattr(positional, "complete"): + # shtab `.complete = ...` functions + compgens.append( + "{}_pos_{}_COMPGEN={}".format( + prefix, + i, + complete2pattern(positional.complete, "bash", choice_type2fn), + ) + ) + + if positional.choices: + # choices (including subparsers & shtab `.complete` functions) + log.debug("choices:{}:{}".format(prefix, sorted(positional.choices))) + + this_positional_choices = [] + for choice in positional.choices: + if isinstance(choice, Choice): + # append special completion type to `compgens` + # NOTE: overrides `.complete` attribute + log.debug( + "Choice.{}:{}:{}".format( + choice.type, prefix, positional.dest + ) + ) + compgens.append( + "{}_pos_{}_COMPGEN={}".format( + prefix, i, choice_type2fn[choice.type] + ) + ) + elif isinstance(positional.choices, dict): + # subparser, so append to list of subparsers & recurse + log.debug("subcommand:%s", choice) + public_cmds = get_public_subcommands(positional) + if choice in public_cmds: + discovered_subparsers.append(str(choice)) + this_positional_choices.append(str(choice)) + ( + new_subparsers, + new_option_strings, + new_compgens, + new_choices, + new_nargs, + ) = recurse( + positional.choices[choice], + prefix + "_" + wordify(choice), + ) + sub_subparsers.extend(new_subparsers) + sub_option_strings.extend(new_option_strings) + sub_compgens.extend(new_compgens) + sub_choices.extend(new_choices) + sub_nargs.extend(new_nargs) + else: + log.debug("skip:subcommand:%s", choice) + else: + # simple choice + this_positional_choices.append(str(choice)) + + if this_positional_choices: + choices.append( + "{}_pos_{}_choices=('{}')".format( + prefix, i, "' '".join(this_positional_choices) + ) + ) + + # skip default `nargs` values + if positional.nargs not in (None, "1", "?"): + nargs.append("{}_pos_{}_nargs={}".format(prefix, i, positional.nargs)) + + if discovered_subparsers: + subparsers.append( + "{}_subparsers=('{}')".format(prefix, "' '".join(discovered_subparsers)) + ) + log.debug("subcommands:{}:{}".format(prefix, discovered_subparsers)) + + # optional arguments + option_strings.append( + "{}_option_strings=('{}')".format( + prefix, "' '".join(get_option_strings(parser)) + ) + ) + for optional in parser._get_optional_actions(): + if optional == SUPPRESS: + continue + + for option_string in optional.option_strings: + if hasattr(optional, "complete"): + # shtab `.complete = ...` functions + compgens.append( + "{}_{}_COMPGEN={}".format( + prefix, + wordify(option_string), + complete2pattern(optional.complete, "bash", choice_type2fn), + ) + ) + + if optional.choices: + # choices (including shtab `.complete` functions) + this_optional_choices = [] + for choice in optional.choices: + # append special completion type to `compgens` + # NOTE: overrides `.complete` attribute + if isinstance(choice, Choice): + log.debug( + "Choice.{}:{}:{}".format( + choice.type, prefix, optional.dest + ) + ) + compgens.append( + "{}_{}_COMPGEN={}".format( + prefix, + wordify(option_string), + choice_type2fn[choice.type], + ) + ) + else: + # simple choice + this_optional_choices.append(str(choice)) + + if this_optional_choices: + choices.append( + "{}_{}_choices=('{}')".format( + prefix, + wordify(option_string), + "' '".join(this_optional_choices), + ) + ) + + # Check for nargs. + if optional.nargs is not None and optional.nargs != 1: + nargs.append( + "{}_{}_nargs={}".format( + prefix, wordify(option_string), optional.nargs + ) + ) + + # append recursion results + subparsers.extend(sub_subparsers) + option_strings.extend(sub_option_strings) + compgens.extend(sub_compgens) + choices.extend(sub_choices) + nargs.extend(sub_nargs) + + return subparsers, option_strings, compgens, choices, nargs + + return recurse(root_parser, root_prefix) + + +@mark_completer("bash") +def complete_bash(parser, root_prefix=None, preamble="", choice_functions=None): + """ + Returns bash syntax autocompletion script. + + See `complete` for arguments. + """ + root_prefix = wordify("_shtab_" + (root_prefix or parser.prog)) + subparsers, option_strings, compgens, choices, nargs = get_bash_commands( + parser, root_prefix, choice_functions=choice_functions + ) + + # References: + # - https://www.gnu.org/software/bash/manual/html_node/ + # Programmable-Completion.html + # - https://opensource.com/article/18/3/creating-bash-completion-script + # - https://stackoverflow.com/questions/12933362 + return Template( + """\ +# AUTOMATCALLY GENERATED by `shtab` + +${subparsers} + +${option_strings} + +${compgens} + +${choices} + +${nargs} + +${preamble} +# $1=COMP_WORDS[1] +_shtab_compgen_files() { + compgen -f -- $1 # files +} + +# $1=COMP_WORDS[1] +_shtab_compgen_dirs() { + compgen -d -- $1 # recurse into subdirs +} + +# $1=COMP_WORDS[1] +_shtab_replace_nonword() { + echo "${1//[^[:word:]]/_}" +} + +# set default values (called for the initial parser & any subparsers) +_set_parser_defaults() { + local subparsers_var="${prefix}_subparsers[@]" + sub_parsers=${!subparsers_var} + + local current_option_strings_var="${prefix}_option_strings[@]" + current_option_strings=${!current_option_strings_var} + + completed_positional_actions=0 + + _set_new_action "pos_${completed_positional_actions}" true +} + +# $1=action identifier +# $2=positional action (bool) +# set all identifiers for an action's parameters +_set_new_action() { + current_action="${prefix}_$(_shtab_replace_nonword $1)" + + local current_action_compgen_var=${current_action}_COMPGEN + current_action_compgen="${!current_action_compgen_var}" + + local current_action_choices_var="${current_action}_choices[@]" + current_action_choices="${!current_action_choices_var}" + + local current_action_nargs_var="${current_action}_nargs" + if [ -n "${!current_action_nargs_var}" ]; then + current_action_nargs="${!current_action_nargs_var}" + else + current_action_nargs=1 + fi + + current_action_args_start_index=$(( $word_index + 1 )) + + current_action_is_positional=$2 +} + +# Notes: +# `COMPREPLY`: what will be rendered after completion is triggered +# `completing_word`: currently typed word to generate completions for +# `${!var}`: evaluates the content of `var` and expand its content as a variable +# hello="world" +# x="hello" +# ${!x} -> ${hello} -> "world" +${root_prefix}() { + local completing_word="${COMP_WORDS[COMP_CWORD]}" + COMPREPLY=() + + prefix=${root_prefix} + word_index=0 + _set_parser_defaults + word_index=1 + + # determine what arguments are appropriate for the current state + # of the arg parser + while [ $word_index -ne $COMP_CWORD ]; do + local this_word="${COMP_WORDS[$word_index]}" + + if [[ -n $sub_parsers && " ${sub_parsers[@]} " =~ " ${this_word} " ]]; then + # valid subcommand: add it to the prefix & reset the current action + prefix="${prefix}_$(_shtab_replace_nonword $this_word)" + _set_parser_defaults + fi + + if [[ " ${current_option_strings[@]} " =~ " ${this_word} " ]]; then + # a new action should be acquired (due to recognised option string or + # no more input expected from current action); + # the next positional action can fill in here + _set_new_action $this_word false + fi + + if [[ "$current_action_nargs" != "*" ]] && \\ + [[ "$current_action_nargs" != "+" ]] && \\ + [[ "$current_action_nargs" != *"..." ]] && \\ + (( $word_index + 1 - $current_action_args_start_index >= \\ + $current_action_nargs )); then + $current_action_is_positional && let "completed_positional_actions += 1" + _set_new_action "pos_${completed_positional_actions}" true + fi + + let "word_index+=1" + done + + # Generate the completions + + if [[ "${completing_word}" == -* ]]; then + # optional argument started: use option strings + COMPREPLY=( $(compgen -W "${current_option_strings[*]}" -- "${completing_word}") ) + else + # use choices & compgen + COMPREPLY=( $(compgen -W "${current_action_choices[*]}" -- "${completing_word}") \\ + $([ -n "${current_action_compgen}" ] \\ + && "${current_action_compgen}" "${completing_word}") ) + fi + + return 0 +} + +complete -o filenames -F ${root_prefix} ${prog}""" + ).safe_substitute( + subparsers="\n".join(subparsers), + option_strings="\n".join(option_strings), + compgens="\n".join(compgens), + choices="\n".join(choices), + nargs="\n".join(nargs), + preamble=( + "\n# Custom Preamble\n" + preamble + "\n# End Custom Preamble\n" + if preamble + else "" + ), + root_prefix=root_prefix, + prog=parser.prog, + ) + + +def escape_zsh(string): + return RE_ZSH_SPECIAL_CHARS.sub(r"\\\1", str(string)) + + +@mark_completer("zsh") +def complete_zsh(parser, root_prefix=None, preamble="", choice_functions=None): + """ + Returns zsh syntax autocompletion script. + + See `complete` for arguments. + """ + prog = parser.prog + root_prefix = wordify("_shtab_" + (root_prefix or prog)) + + choice_type2fn = {k: v["zsh"] for k, v in CHOICE_FUNCTIONS.items()} + if choice_functions: + choice_type2fn.update(choice_functions) + + def format_optional(opt): + return ( + ( + '{nargs}{options}"[{help}]"' + if isinstance(opt, FLAG_OPTION) + else '{nargs}{options}"[{help}]:{dest}:{pattern}"' + ) + .format( + nargs=( + '"(- :)"' + if isinstance(opt, OPTION_END) + else '"*"' + if isinstance(opt, OPTION_MULTI) + else "" + ), + options=( + "{{{}}}".format(",".join(opt.option_strings)) + if len(opt.option_strings) > 1 + else '"{}"'.format("".join(opt.option_strings)) + ), + help=escape_zsh(opt.help or ""), + dest=opt.dest, + pattern=complete2pattern(opt.complete, "zsh", choice_type2fn) + if hasattr(opt, "complete") + else ( + choice_type2fn[opt.choices[0].type] + if isinstance(opt.choices[0], Choice) + else "({})".format(" ".join(map(str, opt.choices))) + ) + if opt.choices + else "", + ) + .replace('""', "") + ) + + def format_positional(opt): + return '"{nargs}:{help}:{pattern}"'.format( + nargs={"+": "(*)", "*": "(*):"}.get(opt.nargs, ""), + help=escape_zsh((opt.help or opt.dest).strip().split("\n")[0]), + pattern=complete2pattern(opt.complete, "zsh", choice_type2fn) + if hasattr(opt, "complete") + else ( + choice_type2fn[opt.choices[0].type] + if isinstance(opt.choices[0], Choice) + else "({})".format(" ".join(map(str, opt.choices))) + ) + if opt.choices + else "", + ) + + # {cmd: {"help": help, "arguments": [arguments]}} + all_commands = { + root_prefix: { + "cmd": prog, + "arguments": [ + format_optional(opt) + for opt in parser._get_optional_actions() + if opt.help != SUPPRESS + ], + "help": (parser.description or "").strip().split("\n")[0], + "commands": [], + "paths": [], + } + } + + def recurse(parser, prefix, paths=None): + paths = paths or [] + subcmds = [] + for sub in parser._get_positional_actions(): + if sub.help == SUPPRESS or not sub.choices: + continue + if not sub.choices or not isinstance(sub.choices, dict): + # positional argument + all_commands[prefix]["arguments"].append(format_positional(sub)) + else: # subparser + log.debug("choices:{}:{}".format(prefix, sorted(sub.choices))) + public_cmds = get_public_subcommands(sub) + for cmd, subparser in sub.choices.items(): + if cmd not in public_cmds: + log.debug("skip:subcommand:%s", cmd) + continue + log.debug("subcommand:%s", cmd) + + # optionals + arguments = [ + format_optional(opt) + for opt in subparser._get_optional_actions() + if opt.help != SUPPRESS + ] + + # positionals + arguments.extend( + format_positional(opt) + for opt in subparser._get_positional_actions() + if not isinstance(opt.choices, dict) + if opt.help != SUPPRESS + ) + + new_pref = prefix + "_" + wordify(cmd) + options = all_commands[new_pref] = { + "cmd": cmd, + "help": (subparser.description or "").strip().split("\n")[0], + "arguments": arguments, + "paths": [*paths, cmd], + } + new_subcmds = recurse(subparser, new_pref, [*paths, cmd]) + options["commands"] = { + all_commands[pref]["cmd"]: all_commands[pref] + for pref in new_subcmds + if pref in all_commands + } + subcmds.extend([*new_subcmds, new_pref]) + log.debug("subcommands:%s:%s", cmd, options) + return subcmds + + recurse(parser, root_prefix) + all_commands[root_prefix]["commands"] = { + options["cmd"]: options + for prefix, options in sorted(all_commands.items()) + if len(options.get("paths", [])) < 2 and prefix != root_prefix + } + subcommands = { + prefix: options + for prefix, options in all_commands.items() + if options.get("commands") + } + subcommands.setdefault(root_prefix, all_commands[root_prefix]) + log.debug("subcommands:%s:%s", root_prefix, sorted(all_commands)) + + def command_case(prefix, options): + name = options["cmd"] + commands = options["commands"] + case_fmt_on_no_sub = ( + """{name}) _arguments -C ${prefix}_{name_wordify}_options ;;""" + ) + case_fmt_on_sub = """{name}) {prefix}_{name_wordify} ;;""" + + cases = [] + for _, options in sorted(commands.items()): + fmt = case_fmt_on_sub if options.get("commands") else case_fmt_on_no_sub + cases.append( + fmt.format( + name=options["cmd"], + name_wordify=wordify(options["cmd"]), + prefix=prefix, + ) + ) + cases = "\n\t".expandtabs(8).join(cases) + + return """\ +{prefix}() {{ + local context state line curcontext="$curcontext" + + _arguments -C ${prefix}_options \\ + ': :{prefix}_commands' \\ + '*::: :->{name}' + + case $state in + {name}) + words=($line[1] "${{words[@]}}") + (( CURRENT += 1 )) + curcontext="${{curcontext%:*:*}}:{prefix}-$line[1]:" + case $line[1] in + {cases} + esac + esac +}} +""".format( + prefix=prefix, name=name, cases=cases + ) + + def command_option(prefix, options): + return """\ +{prefix}_options=( + {arguments} +) +""".format( + prefix=prefix, arguments="\n ".join(options["arguments"]) + ) + + def command_list(prefix, options): + name = " ".join([prog, *options["paths"]]) + commands = "\n ".join( + '"{}:{}"'.format(cmd, escape_zsh(opt["help"])) + for cmd, opt in sorted(options["commands"].items()) + ) + return """ +{prefix}_commands() {{ + local _commands=( + {commands} + ) + _describe '{name} commands' _commands +}}""".format( + prefix=prefix, name=name, commands=commands + ) + + preamble = ( + """\ +# Custom Preamble +{} + +# End Custom Preamble +""".format( + preamble.rstrip() + ) + if preamble + else "" + ) + # References: + # - https://github.com/zsh-users/zsh-completions + # - http://zsh.sourceforge.net/Doc/Release/Completion-System.html + # - https://mads-hartmann.com/2017/08/06/ + # writing-zsh-completion-scripts.html + # - http://www.linux-mag.com/id/1106/ + return Template( + """\ +#compdef ${prog} + +# AUTOMATCALLY GENERATED by `shtab` + +${command_commands} + +${command_options} + +${command_cases} +${preamble} + +typeset -A opt_args +${root_prefix} "$@\"""" + ).safe_substitute( + prog=prog, + root_prefix=root_prefix, + command_cases="\n".join(starmap(command_case, sorted(subcommands.items()))), + command_commands="\n".join(starmap(command_list, sorted(subcommands.items()))), + command_options="\n".join( + starmap(command_option, sorted(all_commands.items())) + ), + preamble=preamble, + ) + + +@mark_completer("tcsh") +def complete_tcsh(parser, root_prefix=None, preamble="", choice_functions=None): + """ + Return tcsh syntax autocompletion script. + + See `complete` for arguments. + """ + optionals_single = set() + optionals_double = set() + specials = [] + index_choices = defaultdict(dict) + + choice_type2fn = {k: v["tcsh"] for k, v in CHOICE_FUNCTIONS.items()} + if choice_functions: + choice_type2fn.update(choice_functions) + + def get_specials(arg, arg_type, arg_sel): + if arg.choices: + choice_strs = " ".join(map(str, arg.choices)) + yield "'{}/{}/({})/'".format( + arg_type, + arg_sel, + choice_strs, + ) + elif hasattr(arg, "complete"): + complete_fn = complete2pattern(arg.complete, "tcsh", choice_type2fn) + if complete_fn: + yield "'{}/{}/{}/'".format( + arg_type, + arg_sel, + complete_fn, + ) + + def recurse_parser(cparser, positional_idx, requirements=None): + log_prefix = "| " * positional_idx + log.debug("%sParser @ %d", log_prefix, positional_idx) + if requirements: + log.debug("%s- Requires: %s", log_prefix, " ".join(requirements)) + else: + requirements = [] + + for optional in cparser._get_optional_actions(): + log.debug("%s| Optional: %s", log_prefix, optional.dest) + if optional.help != SUPPRESS: + # Mingle all optional arguments for all subparsers + for optional_str in optional.option_strings: + log.debug("%s| | %s", log_prefix, optional_str) + if optional_str.startswith("--"): + optionals_double.add(optional_str[2:]) + elif optional_str.startswith("-"): + optionals_single.add(optional_str[1:]) + specials.extend(get_specials(optional, "n", optional_str)) + + for positional in cparser._get_positional_actions(): + if positional.help != SUPPRESS: + positional_idx += 1 + log.debug( + "%s| Positional #%d: %s", + log_prefix, + positional_idx, + positional.dest, + ) + index_choices[positional_idx][tuple(requirements)] = positional + if not requirements and isinstance(positional.choices, dict): + for subcmd, subparser in positional.choices.items(): + log.debug("%s| | SubParser: %s", log_prefix, subcmd) + recurse_parser( + subparser, positional_idx, requirements + [subcmd] + ) + + recurse_parser(parser, 0) + + for idx, ndict in index_choices.items(): + if len(ndict) == 1: + # Single choice, no requirements + arg = list(ndict.values())[0] + specials.extend(get_specials(arg, "p", str(idx))) + else: + # Multiple requirements + nlist = [] + for nn, arg in ndict.items(): + checks = [ + '[ "$cmd[{}]" == "{}" ]'.format(iidx, n) + for iidx, n in enumerate(nn, start=2) + ] + if arg.choices: + nlist.append( + '( {}echo "{}" || false )'.format( + " && ".join(checks + [""]), # Append the separator + "\\n".join(arg.choices), + ) + ) + + # Ugly hack + specials.append( + "'p@{}@`set cmd=($COMMAND_LINE); {}`@'".format( + str(idx), " || ".join(nlist) + ) + ) + + return Template( + """\ +# AUTOMATICALLY GENERATED by `shtab` + +${preamble} + +complete ${prog} \\ + 'c/--/(${optionals_double_str})/' \\ + 'c/-/(${optionals_single_str} -)/' \\ + ${optionals_special_str} \\ + 'p/*/()/'""" + ).safe_substitute( + preamble=( + "\n# Custom Preamble\n" + preamble + "\n# End Custom Preamble\n" + if preamble + else "" + ), + root_prefix=root_prefix, + prog=parser.prog, + optionals_double_str=" ".join(optionals_double), + optionals_single_str=" ".join(optionals_single), + optionals_special_str=" \\\n ".join(specials), + ) + + +def complete( + parser: ArgumentParser, + shell: str = "bash", + root_prefix: Opt[str] = None, + preamble: Union[str, Dict] = "", + choice_functions: Opt[Any] = None, +) -> str: + """ + parser : argparse.ArgumentParser + shell : str (bash/zsh) + root_prefix : str or `None` + prefix for shell functions to avoid clashes (default: "_{parser.prog}") + preamble : dict or str + mapping shell to text to prepend to generated script + (e.g. `{"bash": "_myprog_custom_function(){ echo hello }"}`) + choice_functions : deprecated + + N.B. `parser.add_argument().complete = ...` can be used to define custom + completions (e.g. filenames). See <../examples/pathcomplete.py>. + """ + if isinstance(preamble, dict): + preamble = preamble.get(shell, "") + completer = get_completer(shell) + return completer( + parser, + root_prefix=root_prefix, + preamble=preamble, + choice_functions=choice_functions, + ) + + +def completion_action(parent=None, preamble=""): + class PrintCompletionAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + print(complete(parent or parser, values, preamble=preamble)) + parser.exit(0) + + return PrintCompletionAction + + +def add_argument_to( + parser, + option_string="--print-completion", + help="print shell completion script", + parent=None, + preamble="", +): + """ + parser : argparse.ArgumentParser + option_string : str or list[str] + iff positional (no `-` prefix) then `parser` is assumed to actually be + a subparser (subcommand mode) + help : str + parent : argparse.ArgumentParser + required in subcommand mode + """ + if isinstance( + option_string, str if sys.version_info[0] > 2 else basestring # NOQA + ): + option_string = [option_string] + kwargs = { + "choices": SUPPORTED_SHELLS, + "default": None, + "help": help, + "action": completion_action(parent, preamble), + } + if option_string[0][0] != "-": # subparser mode + kwargs.update(default=SUPPORTED_SHELLS[0], nargs="?") + assert parent is not None, "subcommand mode: parent required" + parser.add_argument(*option_string, **kwargs) + return parser diff --git a/dcargs/_shtab/__main__.py b/dcargs/_shtab/__main__.py new file mode 100644 index 000000000..180d25377 --- /dev/null +++ b/dcargs/_shtab/__main__.py @@ -0,0 +1,10 @@ +from __future__ import absolute_import + +import logging +import sys + +from .main import main + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + sys.exit(main(sys.argv[1:]) or 0) diff --git a/dcargs/_shtab/_dist_ver.py b/dcargs/_shtab/_dist_ver.py new file mode 100644 index 000000000..5ad5a81bd --- /dev/null +++ b/dcargs/_shtab/_dist_ver.py @@ -0,0 +1 @@ +__version__ = "1.5.6.dev1+g4c0170c.d20220909" diff --git a/dcargs/_shtab/main.py b/dcargs/_shtab/main.py new file mode 100644 index 000000000..75b80926b --- /dev/null +++ b/dcargs/_shtab/main.py @@ -0,0 +1,77 @@ +from __future__ import absolute_import, print_function + +import argparse +import logging +import os +import sys +from importlib import import_module + +from . import SUPPORTED_SHELLS, __version__, complete + +log = logging.getLogger(__name__) + + +def get_main_parser(): + parser = argparse.ArgumentParser(prog="shtab") + parser.add_argument( + "parser", help="importable parser (or fuction returning parser)" + ) + parser.add_argument( + "--version", action="version", version="%(prog)s " + __version__ + ) + parser.add_argument( + "-s", "--shell", default=SUPPORTED_SHELLS[0], choices=SUPPORTED_SHELLS + ) + parser.add_argument( + "--prefix", help="prepended to generated functions to avoid clashes" + ) + parser.add_argument("--preamble", help="prepended to generated script") + parser.add_argument("--prog", help="custom program name (overrides `parser.prog`)") + parser.add_argument( + "-u", + "--error-unimportable", + default=False, + action="store_true", + help="raise errors if `parser` is not found in $PYTHONPATH", + ) + parser.add_argument( + "--verbose", + dest="loglevel", + action="store_const", + default=logging.INFO, + const=logging.DEBUG, + help="Log debug information", + ) + return parser + + +def main(argv=None): + parser = get_main_parser() + args = parser.parse_args(argv) + logging.basicConfig(level=args.loglevel) + log.debug(args) + + module, other_parser = args.parser.rsplit(".", 1) + if sys.path and sys.path[0]: + # not blank so not searching curdir + sys.path.insert(1, os.curdir) + try: + module = import_module(module) + except ImportError as err: + if args.error_unimportable: + raise + log.debug(str(err)) + return + other_parser = getattr(module, other_parser) + if callable(other_parser): + other_parser = other_parser() + if args.prog: + other_parser.prog = args.prog + print( + complete( + other_parser, + shell=args.shell, + root_prefix=args.prefix or args.parser.split(".", 1)[0], + preamble=args.preamble, + ) + ) diff --git a/dcargs/_shtab/py.typed b/dcargs/_shtab/py.typed new file mode 100644 index 000000000..92ab5d63a --- /dev/null +++ b/dcargs/_shtab/py.typed @@ -0,0 +1,2 @@ +This file exists solely to signal that the `shtab` package carries inline types. +Do not delete it. diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 6ecbc7b27..8df3ed826 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -19,14 +19,22 @@ def _strip_dummy_field_names(parts: Iterable[str]) -> Iterable[str]: def make_field_name(parts: Sequence[str]) -> str: """Join parts of a field name together. Used for nesting. - ('parent', 'child') => 'parent.child' - ('parents', '1', 'child') => 'parents:1.child' + ('parent_1', 'child') => 'parent-1.child' + ('parents', '1', '_child_node') => 'parents.1._child-node' """ out: List[str] = [] for i, p in enumerate(_strip_dummy_field_names(parts)): if i > 0: out.append(".") + # Replace all underscores with hyphens, except ones at the start of a string. + num_underscore_prefix = 0 + for i in range(len(p)): + if p[i] == "_": + num_underscore_prefix += 1 + else: + break + p = "_" * num_underscore_prefix + p[num_underscore_prefix:].replace("_", "-") out.append(p) return "".join(out) diff --git a/dcargs/conf/__init__.py b/dcargs/conf/__init__.py index e1a1743ee..743339f8f 100644 --- a/dcargs/conf/__init__.py +++ b/dcargs/conf/__init__.py @@ -5,7 +5,7 @@ Features here are supported, but generally unnecessary and should be used sparingly. """ -from ._markers import AvoidSubcommands, Fixed, FlagConversionOff, Positional +from ._markers import AvoidSubcommands, Fixed, FlagConversionOff, Positional, Suppress from ._subcommands import subcommand __all__ = [ @@ -13,5 +13,6 @@ "Fixed", "FlagConversionOff", "Positional", + "Suppress", "subcommand", ] diff --git a/dcargs/conf/_markers.py b/dcargs/conf/_markers.py index 6aa83f9c9..4b8c146da 100644 --- a/dcargs/conf/_markers.py +++ b/dcargs/conf/_markers.py @@ -34,6 +34,11 @@ def __repr__(self): default value should be set instead. Note that fields with defaults that can't be parsed will also be marked as fixed automatically.""" +SUPPRESS = _make_marker("Suppress") +Suppress = Annotated[T, FIXED, SUPPRESS] +"""A type `T` can be annotated as `Suppress[T]` to prevent `dcargs.cli` from parsing it, and +to prevent it from showing up in helptext.""" + FLAG_CONVERSION_OFF = _make_marker("FlagConversionOff") FlagConversionOff = Annotated[T, FLAG_CONVERSION_OFF] """Turn off flag conversion for booleans with default values. Instead, types annotated diff --git a/pyproject.toml b/pyproject.toml index a4734c23a..e0aade281 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ termcolor = "^1.1.0" "backports.cached-property" = { version = "^1.0.2", python = "~3.7" } colorama = {version = "^0.4.0", platform = "win32"} frozendict = "^2.3.4" -shtab = "^1.5.5" +# shtab = "^1.5.5" [tool.poetry.dev-dependencies] pytest = "^7.1.2" @@ -43,6 +43,11 @@ python_version = "3.8" ignore_missing_imports = true warn_unused_configs = true +[tool.coverage.run] +omit = [ + "dcargs/_shtab/*", +] + [tool.coverage.report] exclude_lines = [ # Have to re-enable the standard pragma diff --git a/tests/test_helptext.py b/tests/test_helptext.py index dd008965f..a3b1a907c 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -579,3 +579,17 @@ def main2(x: Callable = nn.ReLU): helptext = _get_helptext(main2) assert "--x {fixed}" in helptext assert "(fixed to: Date: Thu, 15 Sep 2022 12:01:37 -0700 Subject: [PATCH 138/491] Fix `subcommand_type_from_defaults` docs (#11) * rename `subcommand_type_from_defaults` to `subcommand_type_from_defaults` * Fix `update_example_docs.py` * rstrip Co-authored-by: Brent Yi --- dcargs/extras/_base_configs.py | 6 +- docs/source/examples/01_functions.rst | 41 ++-- docs/source/examples/02_dataclasses.rst | 39 ++-- .../examples/03_enums_and_containers.rst | 77 ++++--- docs/source/examples/04_flags.rst | 53 ++--- .../examples/05_hierarchical_configs.rst | 131 +++++------ .../examples/06_literals_and_unions.rst | 89 +++---- docs/source/examples/07_positional_args.rst | 109 ++++----- docs/source/examples/08_subcommands.rst | 69 +++--- .../examples/09_multiple_subcommands.rst | 121 +++++----- docs/source/examples/10_base_configs.rst | 217 +++++++++--------- docs/source/examples/11_dictionaries.rst | 85 +++---- docs/source/examples/12_tuples.rst | 59 ++--- docs/source/examples/13_standard_classes.rst | 51 ++-- docs/source/examples/14_generics.rst | 69 +++--- .../examples/15_nesting_in_containers.rst | 103 +++++---- .../examples/16_advanced_configuration.rst | 109 ++++----- docs/update_example_docs.py | 7 +- 18 files changed, 727 insertions(+), 708 deletions(-) diff --git a/dcargs/extras/_base_configs.py b/dcargs/extras/_base_configs.py index a0984c819..36d06bd8b 100644 --- a/dcargs/extras/_base_configs.py +++ b/dcargs/extras/_base_configs.py @@ -45,14 +45,14 @@ def subcommand_type_from_defaults( The resulting type can be used directly in dcargs.cli: ```python - config = dcargs.cli(subcommand_union_from_mapping(default_from_name)) + config = dcargs.cli(subcommand_type_from_defaults(default_from_name)) reveal_type(config) # Should be correct! ``` Or to generate annotations for classes and functions: ```python - SelectableConfig = subcommand_union_from_mapping(default_from_name) + SelectableConfig = subcommand_type_from_defaults(default_from_name) def train( config: SelectableConfig, @@ -70,7 +70,7 @@ def train( if TYPE_CHECKING: SelectableConfig = ExperimentConfig else: - SelectableConfig = subcommand_union_from_mapping(base_mapping) + SelectableConfig = subcommand_type_from_defaults(base_mapping) ``` """ return Union.__getitem__( # type: ignore diff --git a/docs/source/examples/01_functions.rst b/docs/source/examples/01_functions.rst index def4a7c8d..324e58d32 100644 --- a/docs/source/examples/01_functions.rst +++ b/docs/source/examples/01_functions.rst @@ -11,26 +11,27 @@ populated from the CLI. .. code-block:: python - :linenos: - - import dcargs - - - def main( - field1: str, - field2: int = 3, - ) -> None: - """Function, whose arguments will be populated from a CLI interface. - - Args: - field1: A string field. - field2: A numeric field, with a default value. - """ - print(field1, field2) - - - if __name__ == "__main__": - dcargs.cli(main) + :linenos: + + + import dcargs + + + def main( + field1: str, + field2: int = 3, + ) -> None: + """Function, whose arguments will be populated from a CLI interface. + + Args: + field1: A string field. + field2: A numeric field, with a default value. + """ + print(field1, field2) + + + if __name__ == "__main__": + dcargs.cli(main) ------------ diff --git a/docs/source/examples/02_dataclasses.rst b/docs/source/examples/02_dataclasses.rst index 42400276c..f8c7c525e 100644 --- a/docs/source/examples/02_dataclasses.rst +++ b/docs/source/examples/02_dataclasses.rst @@ -11,25 +11,26 @@ can be used as a typed alternative for an argparse namespace. .. code-block:: python - :linenos: - - import dataclasses - - import dcargs - - - @dataclasses.dataclass - class Args: - """Description. - This should show up in the helptext!""" - - field1: str # A string field. - field2: int = 3 # A numeric field, with a default value. - - - if __name__ == "__main__": - args = dcargs.cli(Args) - print(args) + :linenos: + + + import dataclasses + + import dcargs + + + @dataclasses.dataclass + class Args: + """Description. + This should show up in the helptext!""" + + field1: str # A string field. + field2: int = 3 # A numeric field, with a default value. + + + if __name__ == "__main__": + args = dcargs.cli(Args) + print(args) ------------ diff --git a/docs/source/examples/03_enums_and_containers.rst b/docs/source/examples/03_enums_and_containers.rst index 9d98a95d9..45fd7da3e 100644 --- a/docs/source/examples/03_enums_and_containers.rst +++ b/docs/source/examples/03_enums_and_containers.rst @@ -11,44 +11,45 @@ container types. .. code-block:: python - :linenos: - - import dataclasses - import enum - import pathlib - from typing import Optional, Tuple - - import dcargs - - - class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - - @dataclasses.dataclass(frozen=True) - class TrainConfig: - # Example of a variable-length tuple. `typing.List`, `typing.Sequence`, - # `typing.Set`, `typing.Dict`, etc are all supported as well. - dataset_sources: Tuple[pathlib.Path, ...] - """Paths to load training data from. This can be multiple!""" - - # Fixed-length tuples are also okay. - image_dimensions: Tuple[int, int] = (32, 32) - """Height and width of some image data.""" - - # Enums are handled seamlessly. - optimizer_type: OptimizerType = OptimizerType.ADAM - """Gradient-based optimizer to use.""" - - # We can also explicitly mark arguments as optional. - checkpoint_interval: Optional[int] = None - """Interval to save checkpoints at.""" - - - if __name__ == "__main__": - config = dcargs.cli(TrainConfig) - print(config) + :linenos: + + + import dataclasses + import enum + import pathlib + from typing import Optional, Tuple + + import dcargs + + + class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + + @dataclasses.dataclass(frozen=True) + class TrainConfig: + # Example of a variable-length tuple. `typing.List`, `typing.Sequence`, + # `typing.Set`, `typing.Dict`, etc are all supported as well. + dataset_sources: Tuple[pathlib.Path, ...] + """Paths to load training data from. This can be multiple!""" + + # Fixed-length tuples are also okay. + image_dimensions: Tuple[int, int] = (32, 32) + """Height and width of some image data.""" + + # Enums are handled seamlessly. + optimizer_type: OptimizerType = OptimizerType.ADAM + """Gradient-based optimizer to use.""" + + # We can also explicitly mark arguments as optional. + checkpoint_interval: Optional[int] = None + """Interval to save checkpoints at.""" + + + if __name__ == "__main__": + config = dcargs.cli(TrainConfig) + print(config) ------------ diff --git a/docs/source/examples/04_flags.rst b/docs/source/examples/04_flags.rst index 52de2c13e..a77411878 100644 --- a/docs/source/examples/04_flags.rst +++ b/docs/source/examples/04_flags.rst @@ -13,32 +13,33 @@ To turn off conversion, see :func:`dcargs.conf.FlagConversionOff`. .. code-block:: python - :linenos: - - import dataclasses - from typing import Optional - - import dcargs - - - @dataclasses.dataclass - class Args: - # Boolean. This expects an explicit "True" or "False". - boolean: bool - - # Optional boolean. Same as above, but can be omitted. - optional_boolean: Optional[bool] = None - - # Pass --flag-a in to set this value to True. - flag_a: bool = False - - # Pass --no-flag-b in to set this value to False. - flag_b: bool = True - - - if __name__ == "__main__": - args = dcargs.cli(Args) - print(args) + :linenos: + + + import dataclasses + from typing import Optional + + import dcargs + + + @dataclasses.dataclass + class Args: + # Boolean. This expects an explicit "True" or "False". + boolean: bool + + # Optional boolean. Same as above, but can be omitted. + optional_boolean: Optional[bool] = None + + # Pass --flag-a in to set this value to True. + flag_a: bool = False + + # Pass --no-flag-b in to set this value to False. + flag_b: bool = True + + + if __name__ == "__main__": + args = dcargs.cli(Args) + print(args) ------------ diff --git a/docs/source/examples/05_hierarchical_configs.rst b/docs/source/examples/05_hierarchical_configs.rst index 006bd1ab5..253ac14bd 100644 --- a/docs/source/examples/05_hierarchical_configs.rst +++ b/docs/source/examples/05_hierarchical_configs.rst @@ -11,71 +11,72 @@ objects. This helps with modularity and grouping in larger projects. .. code-block:: python - :linenos: - - import dataclasses - import enum - import pathlib - - import dcargs - - - class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - - @dataclasses.dataclass - class OptimizerConfig: - # Gradient-based optimizer to use. - algorithm: OptimizerType = OptimizerType.ADAM - - # Learning rate to use. - learning_rate: float = 3e-4 - - # Coefficient for L2 regularization. - weight_decay: float = 1e-2 - - - @dataclasses.dataclass - class ExperimentConfig: - # Various configurable options for our optimizer. - optimizer: OptimizerConfig - - # Batch size. - batch_size: int = 32 - - # Total number of training steps. - train_steps: int = 100_000 - - # Random seed. This is helpful for making sure that our experiments are all - # reproducible! - seed: int = 0 - - - def train( - out_dir: pathlib.Path, - config: ExperimentConfig, - restore_checkpoint: bool = False, - checkpoint_interval: int = 1000, - ) -> None: - """Train a model. - - Args: - out_dir: Where to save logs and checkpoints. - config: Experiment configuration. - restore_checkpoint: Set to restore an existing checkpoint. - checkpoint_interval: Training steps between each checkpoint save. - """ - print(f"{out_dir=}, {restore_checkpoint=}, {checkpoint_interval=}") - print() - print(f"{config=}") - print() - print(dcargs.to_yaml(config)) - - - if __name__ == "__main__": - dcargs.cli(train) + :linenos: + + + import dataclasses + import enum + import pathlib + + import dcargs + + + class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + + @dataclasses.dataclass + class OptimizerConfig: + # Gradient-based optimizer to use. + algorithm: OptimizerType = OptimizerType.ADAM + + # Learning rate to use. + learning_rate: float = 3e-4 + + # Coefficient for L2 regularization. + weight_decay: float = 1e-2 + + + @dataclasses.dataclass + class ExperimentConfig: + # Various configurable options for our optimizer. + optimizer_config: OptimizerConfig + + # Batch size. + batch_size: int = 32 + + # Total number of training steps. + train_steps: int = 100_000 + + # Random seed. This is helpful for making sure that our experiments are all + # reproducible! + seed: int = 0 + + + def train( + out_dir: pathlib.Path, + config: ExperimentConfig, + restore_checkpoint: bool = False, + checkpoint_interval: int = 1000, + ) -> None: + """Train a model. + + Args: + out_dir: Where to save logs and checkpoints. + config: Experiment configuration. + restore_checkpoint: Set to restore an existing checkpoint. + checkpoint_interval: Training steps between each checkpoint save. + """ + print(f"{out_dir=}, {restore_checkpoint=}, {checkpoint_interval=}") + print() + print(f"{config=}") + print() + print(dcargs.to_yaml(config)) + + + if __name__ == "__main__": + dcargs.cli(train) ------------ diff --git a/docs/source/examples/06_literals_and_unions.rst b/docs/source/examples/06_literals_and_unions.rst index f5bbf58c9..e85d02cc2 100644 --- a/docs/source/examples/06_literals_and_unions.rst +++ b/docs/source/examples/06_literals_and_unions.rst @@ -11,50 +11,51 @@ .. code-block:: python - :linenos: - - import dataclasses - import enum - from typing import Literal, Optional, Tuple, Union - - import dcargs - - - class Color(enum.Enum): - RED = enum.auto() - GREEN = enum.auto() - BLUE = enum.auto() - - - @dataclasses.dataclass(frozen=True) - class Args: - # We can use Literal[] to restrict the set of allowable inputs, for example, over - # enums. - restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED - - # Or mix them with other types! - mixed: Literal[Color.RED, Color.GREEN, "blue"] = "blue" - - # Literals can also be marked Optional. - integer: Optional[Literal[0, 1, 2, 3]] = None - - # Unions can be used to specify multiple allowable types. - union_over_types: Union[int, str] = 0 - string_or_enum: Union[Literal["red", "green"], Color] = "red" - - # Unions also work over more complex nested types. - union_over_tuples: Union[Tuple[int, int], Tuple[str]] = ("1",) - - # And can be nested in other types. - tuple_of_string_or_enum: Tuple[Union[Literal["red", "green"], Color], ...] = ( - "red", - Color.RED, - ) - - - if __name__ == "__main__": - args = dcargs.cli(Args) - print(args) + :linenos: + + + import dataclasses + import enum + from typing import Literal, Optional, Tuple, Union + + import dcargs + + + class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + + @dataclasses.dataclass(frozen=True) + class Args: + # We can use Literal[] to restrict the set of allowable inputs, for example, over + # enums. + restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED + + # Or mix them with other types! + mixed: Literal[Color.RED, Color.GREEN, "blue"] = "blue" + + # Literals can also be marked Optional. + integer: Optional[Literal[0, 1, 2, 3]] = None + + # Unions can be used to specify multiple allowable types. + union_over_types: Union[int, str] = 0 + string_or_enum: Union[Literal["red", "green"], Color] = "red" + + # Unions also work over more complex nested types. + union_over_tuples: Union[Tuple[int, int], Tuple[str]] = ("1",) + + # And can be nested in other types. + tuple_of_string_or_enum: Tuple[Union[Literal["red", "green"], Color], ...] = ( + "red", + Color.RED, + ) + + + if __name__ == "__main__": + args = dcargs.cli(Args) + print(args) ------------ diff --git a/docs/source/examples/07_positional_args.rst b/docs/source/examples/07_positional_args.rst index 0e508d82d..7bf7cd260 100644 --- a/docs/source/examples/07_positional_args.rst +++ b/docs/source/examples/07_positional_args.rst @@ -12,60 +12,61 @@ For more general positional arguments, see :func:`dcargs.conf.Positional`. .. code-block:: python - :linenos: - - from __future__ import annotations - - import dataclasses - import enum - import pathlib - from typing import Tuple - - import dcargs - - - def main( - source: pathlib.Path, - dest: pathlib.Path, - /, # Mark the end of positional arguments. - optimizer: OptimizerConfig, - force: bool = False, - verbose: bool = False, - background_rgb: Tuple[float, float, float] = (1.0, 0.0, 0.0), - ) -> None: - """Command-line interface defined using a function signature. Note that this - docstring is parsed to generate helptext. - - Args: - source: Source path. - dest: Destination path. - optimizer: Configuration for our optimizer object. - force: Do not prompt before overwriting. - verbose: Explain what is being done. - background_rgb: Background color. Red by default. - """ - print(f"{source=}\n{dest=}\n{optimizer=}\n{force=}\n{verbose=}\n{background_rgb=}") - - - class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - - @dataclasses.dataclass(frozen=True) - class OptimizerConfig: - algorithm: OptimizerType = OptimizerType.ADAM - """Gradient-based optimizer to use.""" - - learning_rate: float = 3e-4 - """Learning rate to use.""" - - weight_decay: float = 1e-2 - """Coefficient for L2 regularization.""" - - - if __name__ == "__main__": - dcargs.cli(main) + :linenos: + + + from __future__ import annotations + + import dataclasses + import enum + import pathlib + from typing import Tuple + + import dcargs + + + def main( + source: pathlib.Path, + dest: pathlib.Path, + /, # Mark the end of positional arguments. + optimizer: OptimizerConfig, + force: bool = False, + verbose: bool = False, + background_rgb: Tuple[float, float, float] = (1.0, 0.0, 0.0), + ) -> None: + """Command-line interface defined using a function signature. Note that this + docstring is parsed to generate helptext. + + Args: + source: Source path. + dest: Destination path. + optimizer: Configuration for our optimizer object. + force: Do not prompt before overwriting. + verbose: Explain what is being done. + background_rgb: Background color. Red by default. + """ + print(f"{source=}\n{dest=}\n{optimizer=}\n{force=}\n{verbose=}\n{background_rgb=}") + + + class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + + @dataclasses.dataclass(frozen=True) + class OptimizerConfig: + algorithm: OptimizerType = OptimizerType.ADAM + """Gradient-based optimizer to use.""" + + learning_rate: float = 3e-4 + """Learning rate to use.""" + + weight_decay: float = 1e-2 + """Coefficient for L2 regularization.""" + + + if __name__ == "__main__": + dcargs.cli(main) ------------ diff --git a/docs/source/examples/08_subcommands.rst b/docs/source/examples/08_subcommands.rst index 0ce749ef3..eae5801e5 100644 --- a/docs/source/examples/08_subcommands.rst +++ b/docs/source/examples/08_subcommands.rst @@ -13,40 +13,41 @@ For configuring subcommands beyond what can be expressed with type annotations, .. code-block:: python - :linenos: - - from __future__ import annotations - - import dataclasses - from typing import Union - - import dcargs - - - @dataclasses.dataclass(frozen=True) - class Checkout: - """Checkout a branch.""" - - branch: str - - - @dataclasses.dataclass(frozen=True) - class Commit: - """Commit changes.""" - - message: str - all: bool = False - - - def main(cmd: Union[Checkout, Commit]) -> None: - print(cmd) - - - if __name__ == "__main__": - # Note that we can also pass `Union[Checkout, Command]` directly into - # `dcargs.cli()`; this is understood by dcargs and pyright, but unfortunately not by - # mypy. - dcargs.cli(main) + :linenos: + + + from __future__ import annotations + + import dataclasses + from typing import Union + + import dcargs + + + @dataclasses.dataclass(frozen=True) + class Checkout: + """Checkout a branch.""" + + branch: str + + + @dataclasses.dataclass(frozen=True) + class Commit: + """Commit changes.""" + + message: str + all: bool = False + + + def main(cmd: Union[Checkout, Commit]) -> None: + print(cmd) + + + if __name__ == "__main__": + # Note that we can also pass `Union[Checkout, Command]` directly into + # `dcargs.cli()`; this is understood by dcargs and pyright, but unfortunately not by + # mypy. + dcargs.cli(main) ------------ diff --git a/docs/source/examples/09_multiple_subcommands.rst b/docs/source/examples/09_multiple_subcommands.rst index bf44d9356..7522d6a8c 100644 --- a/docs/source/examples/09_multiple_subcommands.rst +++ b/docs/source/examples/09_multiple_subcommands.rst @@ -10,66 +10,67 @@ Multiple unions over nested types are populated using a series of subcommands. .. code-block:: python - :linenos: - - from __future__ import annotations - - import dataclasses - from typing import Literal, Tuple, Union - - import dcargs - - # Possible dataset configurations. - - - @dataclasses.dataclass - class Mnist: - binary: bool = False - """Set to load binary version of MNIST dataset.""" - - - @dataclasses.dataclass - class ImageNet: - subset: Literal[50, 100, 1000] - """Choose between ImageNet-50, ImageNet-100, ImageNet-1000, etc.""" - - - # Possible optimizer configurations. - - - @dataclasses.dataclass - class Adam: - learning_rate: float = 1e-3 - betas: Tuple[float, float] = (0.9, 0.999) - - - @dataclasses.dataclass - class Sgd: - learning_rate: float = 3e-4 - - - # Train script. - - - def train( - dataset: Union[Mnist, ImageNet] = Mnist(), - optimizer: Union[Adam, Sgd] = Adam(), - ) -> None: - """Example training script. - - Args: - dataset: Dataset to train on. - optimizer: Optimizer to train with. - - Returns: - None: - """ - print(dataset) - print(optimizer) - - - if __name__ == "__main__": - dcargs.cli(train) + :linenos: + + + from __future__ import annotations + + import dataclasses + from typing import Literal, Tuple, Union + + import dcargs + + # Possible dataset configurations. + + + @dataclasses.dataclass + class Mnist: + binary: bool = False + """Set to load binary version of MNIST dataset.""" + + + @dataclasses.dataclass + class ImageNet: + subset: Literal[50, 100, 1000] + """Choose between ImageNet-50, ImageNet-100, ImageNet-1000, etc.""" + + + # Possible optimizer configurations. + + + @dataclasses.dataclass + class Adam: + learning_rate: float = 1e-3 + betas: Tuple[float, float] = (0.9, 0.999) + + + @dataclasses.dataclass + class Sgd: + learning_rate: float = 3e-4 + + + # Train script. + + + def train( + dataset: Union[Mnist, ImageNet] = Mnist(), + optimizer: Union[Adam, Sgd] = Adam(), + ) -> None: + """Example training script. + + Args: + dataset: Dataset to train on. + optimizer: Optimizer to train with. + + Returns: + None: + """ + print(dataset) + print(optimizer) + + + if __name__ == "__main__": + dcargs.cli(train) ------------ diff --git a/docs/source/examples/10_base_configs.rst b/docs/source/examples/10_base_configs.rst index 5e1a3b700..e3135ac79 100644 --- a/docs/source/examples/10_base_configs.rst +++ b/docs/source/examples/10_base_configs.rst @@ -18,114 +18,115 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. .. code-block:: python - :linenos: - - from dataclasses import dataclass - from typing import Callable, Literal, Tuple, Union - - from torch import nn - - import dcargs - - - @dataclass(frozen=True) - class AdamOptimizer: - learning_rate: float = 1e-3 - betas: Tuple[float, float] = (0.9, 0.999) - - - @dataclass(frozen=True) - class SgdOptimizer: - learning_rate: float = 3e-4 - - - @dataclass(frozen=True) - class ExperimentConfig: - # Dataset to run experiment on. - dataset: Literal["mnist", "imagenet-50"] - - # Optimizer parameters. - optimizer: Union[AdamOptimizer, SgdOptimizer] - - # Model size. - num_layers: int - units: int - - # Batch size. - batch_size: int - - # Total number of training steps. - train_steps: int - - # Random seed. This is helpful for making sure that our experiments are all - # reproducible! - seed: int - - # Activation to use. Not specifiable via the commandline. - activation: Callable[[], nn.Module] - - - # Note that we could also define this library using separate YAML files (similar to - # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type - # checking + IDE support. - descriptions = {} - base_configs = {} - - descriptions["small"] = "Train a smaller model." - base_configs["small"] = ExperimentConfig( - dataset="mnist", - optimizer=SgdOptimizer(), - batch_size=2048, - num_layers=4, - units=64, - train_steps=30_000, - # The dcargs.MISSING sentinel allows us to specify that the seed should have no - # default, and needs to be populated from the CLI. - seed=dcargs.MISSING, - activation=nn.ReLU, - ) - - - descriptions["big"] = "Train a bigger model." - base_configs["big"] = ExperimentConfig( - dataset="imagenet-50", - optimizer=AdamOptimizer(), - batch_size=32, - num_layers=8, - units=256, - train_steps=100_000, - seed=dcargs.MISSING, - activation=nn.GELU, - ) - - - if __name__ == "__main__": - config = dcargs.cli( - dcargs.extras.subcommand_union_from_mapping(base_configs, descriptions), - ) - # Note that this is equivalent to: - # - # config = dcargs.cli( - # Union[ - # Annotated[ - # ExperimentConfig, - # dcargs.conf.subcommand( - # "small", - # default=base_configs["small"], - # description=descriptions["small"], - # ), - # ], - # Annotated[ - # ExperimentConfig, - # dcargs.conf.subcommand( - # "big", - # default=base_configs["big"], - # description=descriptions["big"], - # ), - # ], - # ] - # ) - print(config) + :linenos: + + + from dataclasses import dataclass + from typing import Callable, Literal, Tuple, Union + + from torch import nn + + import dcargs + + + @dataclass(frozen=True) + class AdamOptimizer: + learning_rate: float = 1e-3 + betas: Tuple[float, float] = (0.9, 0.999) + + + @dataclass(frozen=True) + class SgdOptimizer: + learning_rate: float = 3e-4 + + + @dataclass(frozen=True) + class ExperimentConfig: + # Dataset to run experiment on. + dataset: Literal["mnist", "imagenet-50"] + + # Optimizer parameters. + optimizer: Union[AdamOptimizer, SgdOptimizer] + + # Model size. + num_layers: int + units: int + + # Batch size. + batch_size: int + + # Total number of training steps. + train_steps: int + + # Random seed. This is helpful for making sure that our experiments are all + # reproducible! + seed: int + + # Activation to use. Not specifiable via the commandline. + activation: Callable[[], nn.Module] + + + # Note that we could also define this library using separate YAML files (similar to + # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type + # checking + IDE support. + descriptions = {} + base_configs = {} + + descriptions["small"] = "Train a smaller model." + base_configs["small"] = ExperimentConfig( + dataset="mnist", + optimizer=SgdOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + # The dcargs.MISSING sentinel allows us to specify that the seed should have no + # default, and needs to be populated from the CLI. + seed=dcargs.MISSING, + activation=nn.ReLU, + ) + + + descriptions["big"] = "Train a bigger model." + base_configs["big"] = ExperimentConfig( + dataset="imagenet-50", + optimizer=AdamOptimizer(), + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=dcargs.MISSING, + activation=nn.GELU, + ) + + + if __name__ == "__main__": + config = dcargs.cli( + dcargs.extras.subcommand_type_from_defaults(base_configs, descriptions), + ) + # ^Note that this is equivalent to: + # + # config = dcargs.cli( + # Union[ + # Annotated[ + # ExperimentConfig, + # dcargs.conf.subcommand( + # "small", + # default=base_configs["small"], + # description=descriptions["small"], + # ), + # ], + # Annotated[ + # ExperimentConfig, + # dcargs.conf.subcommand( + # "big", + # default=base_configs["big"], + # description=descriptions["big"], + # ), + # ], + # ] + # ) + print(config) ------------ diff --git a/docs/source/examples/11_dictionaries.rst b/docs/source/examples/11_dictionaries.rst index cd3400151..a5092ed5a 100644 --- a/docs/source/examples/11_dictionaries.rst +++ b/docs/source/examples/11_dictionaries.rst @@ -11,48 +11,49 @@ or a ``TypedDict`` subclass. .. code-block:: python - :linenos: - - from typing import Dict, Mapping, Tuple, TypedDict - - from frozendict import frozendict # type: ignore - - import dcargs - - - class DictionarySchema( - TypedDict, - # Setting `total=False` specifies that not all keys need to exist. - total=False, - ): - learning_rate: float - betas: Tuple[float, float] - - - def main( - typed_dict: DictionarySchema, - standard_dict: Dict[str, float] = { - "learning_rate": 3e-4, - "beta1": 0.9, - "beta2": 0.999, - }, - frozen_dict: Mapping[str, float] = frozendict( - { - "num_epochs": 20, - "batch_size": 64, - } - ), - ) -> None: - assert isinstance(typed_dict, dict) - assert isinstance(standard_dict, dict) - assert isinstance(frozen_dict, frozendict) - print("Typed dict:", typed_dict) - print("Standard dict:", standard_dict) - print("Frozen dict:", frozen_dict) - - - if __name__ == "__main__": - dcargs.cli(main) + :linenos: + + + from typing import Dict, Mapping, Tuple, TypedDict + + from frozendict import frozendict # type: ignore + + import dcargs + + + class DictionarySchema( + TypedDict, + # Setting `total=False` specifies that not all keys need to exist. + total=False, + ): + learning_rate: float + betas: Tuple[float, float] + + + def main( + typed_dict: DictionarySchema, + standard_dict: Dict[str, float] = { + "learning_rate": 3e-4, + "beta1": 0.9, + "beta2": 0.999, + }, + frozen_dict: Mapping[str, float] = frozendict( + { + "num_epochs": 20, + "batch_size": 64, + } + ), + ) -> None: + assert isinstance(typed_dict, dict) + assert isinstance(standard_dict, dict) + assert isinstance(frozen_dict, frozendict) + print("Typed dict:", typed_dict) + print("Standard dict:", standard_dict) + print("Frozen dict:", frozen_dict) + + + if __name__ == "__main__": + dcargs.cli(main) ------------ diff --git a/docs/source/examples/12_tuples.rst b/docs/source/examples/12_tuples.rst index b3303daf2..46381befa 100644 --- a/docs/source/examples/12_tuples.rst +++ b/docs/source/examples/12_tuples.rst @@ -10,35 +10,36 @@ Example using ``dcargs.cli()`` to instantiate tuple types. .. code-block:: python - :linenos: - - from typing import NamedTuple, Tuple - - import dcargs - - - # Named tuples are interpreted as nested structures. - class Color(NamedTuple): - r: int - g: int - b: int - - - class TupleType(NamedTuple): - """Description. - This should show up in the helptext!""" - - # Tuple types can contain raw values. - color: Tuple[int, int, int] = (255, 0, 0) - - # Tuple types can contain nested structures. - two_colors: Tuple[Color, Color] = (Color(255, 0, 0), Color(0, 255, 0)) - - - if __name__ == "__main__": - x = dcargs.cli(TupleType) - assert isinstance(x, tuple) - print(x) + :linenos: + + + from typing import NamedTuple, Tuple + + import dcargs + + + # Named tuples are interpreted as nested structures. + class Color(NamedTuple): + r: int + g: int + b: int + + + class TupleType(NamedTuple): + """Description. + This should show up in the helptext!""" + + # Tuple types can contain raw values. + color: Tuple[int, int, int] = (255, 0, 0) + + # Tuple types can contain nested structures. + two_colors: Tuple[Color, Color] = (Color(255, 0, 0), Color(0, 255, 0)) + + + if __name__ == "__main__": + x = dcargs.cli(TupleType) + assert isinstance(x, tuple) + print(x) ------------ diff --git a/docs/source/examples/13_standard_classes.rst b/docs/source/examples/13_standard_classes.rst index 784c45aaa..9e0da5d03 100644 --- a/docs/source/examples/13_standard_classes.rst +++ b/docs/source/examples/13_standard_classes.rst @@ -11,31 +11,32 @@ constructors of) standard Python classes. .. code-block:: python - :linenos: - - import dcargs - - - class Args: - def __init__( - self, - field1: str, - field2: int, - flag: bool = False, - ): - """Arguments. - - Args: - field1: A string field. - field2: A numeric field. - flag: A boolean flag. - """ - self.data = [field1, field2, flag] - - - if __name__ == "__main__": - args = dcargs.cli(Args) - print(args.data) + :linenos: + + + import dcargs + + + class Args: + def __init__( + self, + field1: str, + field2: int, + flag: bool = False, + ): + """Arguments. + + Args: + field1: A string field. + field2: A numeric field. + flag: A boolean flag. + """ + self.data = [field1, field2, flag] + + + if __name__ == "__main__": + args = dcargs.cli(Args) + print(args.data) ------------ diff --git a/docs/source/examples/14_generics.rst b/docs/source/examples/14_generics.rst index 7216c5f98..48145e493 100644 --- a/docs/source/examples/14_generics.rst +++ b/docs/source/examples/14_generics.rst @@ -10,40 +10,41 @@ Example of parsing for generic dataclasses. .. code-block:: python - :linenos: - - import dataclasses - from typing import Generic, TypeVar - - import dcargs - - ScalarType = TypeVar("ScalarType", int, float) - ShapeType = TypeVar("ShapeType") - - - @dataclasses.dataclass(frozen=True) - class Point3(Generic[ScalarType]): - x: ScalarType - y: ScalarType - z: ScalarType - frame_id: str - - - @dataclasses.dataclass(frozen=True) - class Triangle: - a: Point3[float] - b: Point3[float] - c: Point3[float] - - - @dataclasses.dataclass(frozen=True) - class Args(Generic[ShapeType]): - shape: ShapeType - - - if __name__ == "__main__": - args = dcargs.cli(Args[Triangle]) - print(args) + :linenos: + + + import dataclasses + from typing import Generic, TypeVar + + import dcargs + + ScalarType = TypeVar("ScalarType", int, float) + ShapeType = TypeVar("ShapeType") + + + @dataclasses.dataclass(frozen=True) + class Point3(Generic[ScalarType]): + x: ScalarType + y: ScalarType + z: ScalarType + frame_id: str + + + @dataclasses.dataclass(frozen=True) + class Triangle: + a: Point3[float] + b: Point3[float] + c: Point3[float] + + + @dataclasses.dataclass(frozen=True) + class Args(Generic[ShapeType]): + shape: ShapeType + + + if __name__ == "__main__": + args = dcargs.cli(Args[Triangle]) + print(args) ------------ diff --git a/docs/source/examples/15_nesting_in_containers.rst b/docs/source/examples/15_nesting_in_containers.rst index f669bc8c1..148536f4a 100644 --- a/docs/source/examples/15_nesting_in_containers.rst +++ b/docs/source/examples/15_nesting_in_containers.rst @@ -13,57 +13,58 @@ parsing default values. .. code-block:: python - :linenos: - - import dataclasses - from typing import Dict, Tuple - - import dcargs - - - class Color: - pass - - - @dataclasses.dataclass - class RGB(Color): - r: int - g: int - b: int - - - @dataclasses.dataclass - class HSL(Color): - h: int - s: int - l: int - - - @dataclasses.dataclass - class Args: - # Example of specifying nested structures via a fixed-length tuple. - color_tuple: Tuple[RGB, HSL] - - # Examples of nested structures in variable-length containers. These need a default - # provided for length inference; we don't currently support specifying dynamic - # container lengths directly from the commandline. - color_tuple_alt: Tuple[Color, ...] = ( - RGB(255, 0, 0), - HSL(0, 255, 0), - ) - color_map: Dict[str, RGB] = dataclasses.field( - # We can't use mutable values as defaults directly. - default_factory={ - "red": RGB(255, 0, 0), - "green": RGB(0, 255, 0), - "blue": RGB(0, 0, 255), - }.copy - ) - - - if __name__ == "__main__": - args = dcargs.cli(Args) - print(args) + :linenos: + + + import dataclasses + from typing import Dict, Tuple + + import dcargs + + + class Color: + pass + + + @dataclasses.dataclass + class RGB(Color): + r: int + g: int + b: int + + + @dataclasses.dataclass + class HSL(Color): + h: int + s: int + l: int + + + @dataclasses.dataclass + class Args: + # Example of specifying nested structures via a fixed-length tuple. + color_tuple: Tuple[RGB, HSL] + + # Examples of nested structures in variable-length containers. These need a default + # provided for length inference; we don't currently support specifying dynamic + # container lengths directly from the commandline. + color_tuple_alt: Tuple[Color, ...] = ( + RGB(255, 0, 0), + HSL(0, 255, 0), + ) + color_map: Dict[str, RGB] = dataclasses.field( + # We can't use mutable values as defaults directly. + default_factory={ + "red": RGB(255, 0, 0), + "green": RGB(0, 255, 0), + "blue": RGB(0, 0, 255), + }.copy + ) + + + if __name__ == "__main__": + args = dcargs.cli(Args) + print(args) ------------ diff --git a/docs/source/examples/16_advanced_configuration.rst b/docs/source/examples/16_advanced_configuration.rst index 12c27e8c8..55fde3935 100644 --- a/docs/source/examples/16_advanced_configuration.rst +++ b/docs/source/examples/16_advanced_configuration.rst @@ -13,60 +13,61 @@ Features here are supported, but generally unnecessary and should be used sparin .. code-block:: python - :linenos: - - import dataclasses - from typing import Union - - from typing_extensions import Annotated - - import dcargs - - - @dataclasses.dataclass(frozen=True) - class CheckoutArgs: - """Checkout a branch.""" - - branch: str - - - @dataclasses.dataclass(frozen=True) - class CommitArgs: - """Commit changes.""" - - message: str - all: bool = False - - - @dataclasses.dataclass - class Args: - # A boolean field with flag conversion turned off. - boolean: dcargs.conf.FlagConversionOff[bool] = False - - # A numeric field parsed as a positional argument. - positional: dcargs.conf.Positional[int] = 3 - - # A numeric field that can't be changed via the CLI. - fixed: dcargs.conf.Fixed[int] = 5 - - # A union over nested structures, but without subcommand generation. When a default - # is provided, the type is simply fixed to that default. - union_without_subcommand: dcargs.conf.AvoidSubcommands[ - Union[CheckoutArgs, CommitArgs] - ] = CheckoutArgs("main") - - # `dcargs.conf.subcommand()` can be used to configure subcommands in a Union. Here, - # we make the subcommand names more succinct. - renamed_subcommand: Union[ - Annotated[ - CheckoutArgs, dcargs.conf.subcommand(name="checkout", prefix_name=False) - ], - Annotated[CommitArgs, dcargs.conf.subcommand(name="commit", prefix_name=False)], - ] = CheckoutArgs("main") - - - if __name__ == "__main__": - print(dcargs.cli(Args)) + :linenos: + + + import dataclasses + from typing import Union + + from typing_extensions import Annotated + + import dcargs + + + @dataclasses.dataclass(frozen=True) + class CheckoutArgs: + """Checkout a branch.""" + + branch: str + + + @dataclasses.dataclass(frozen=True) + class CommitArgs: + """Commit changes.""" + + message: str + all: bool = False + + + @dataclasses.dataclass + class Args: + # A boolean field with flag conversion turned off. + boolean: dcargs.conf.FlagConversionOff[bool] = False + + # A numeric field parsed as a positional argument. + positional: dcargs.conf.Positional[int] = 3 + + # A numeric field that can't be changed via the CLI. + fixed: dcargs.conf.Fixed[int] = 5 + + # A union over nested structures, but without subcommand generation. When a default + # is provided, the type is simply fixed to that default. + union_without_subcommand: dcargs.conf.AvoidSubcommands[ + Union[CheckoutArgs, CommitArgs] + ] = CheckoutArgs("main") + + # `dcargs.conf.subcommand()` can be used to configure subcommands in a Union. Here, + # we make the subcommand names more succinct. + renamed_subcommand: Union[ + Annotated[ + CheckoutArgs, dcargs.conf.subcommand(name="checkout", prefix_name=False) + ], + Annotated[CommitArgs, dcargs.conf.subcommand(name="commit", prefix_name=False)], + ] = CheckoutArgs("main") + + + if __name__ == "__main__": + print(dcargs.cli(Args)) ------------ diff --git a/docs/update_example_docs.py b/docs/update_example_docs.py index 6cc4a19f1..7e0ffa334 100644 --- a/docs/update_example_docs.py +++ b/docs/update_example_docs.py @@ -121,9 +121,12 @@ def main( "", "", ".. code-block:: python", - " :linenos:", + " :linenos:", "", - " " + "\n ".join(ex.source.split("\n")), + "", + "\n".join( + f" {line}".rstrip() for line in ex.source.split("\n") + ), "", ] + usage_lines From 65c60006d7e3cc94d7f56fec0a3fbb25665098b6 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 15 Sep 2022 12:40:20 -0700 Subject: [PATCH 139/491] Fix tab complete for subcommands with colons (closes #10) --- dcargs/_shtab/README.md | 1 + dcargs/_shtab/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dcargs/_shtab/README.md b/dcargs/_shtab/README.md index f637e63d6..2a7c38f7b 100644 --- a/dcargs/_shtab/README.md +++ b/dcargs/_shtab/README.md @@ -2,5 +2,6 @@ Temporary copy of `shtab` that incorporates two PRs: - https://github.com/iterative/shtab/pull/106 - https://github.com/iterative/shtab/pull/108 +- https://github.com/iterative/shtab/pull/110 Can be removed when these are merged. diff --git a/dcargs/_shtab/__init__.py b/dcargs/_shtab/__init__.py index 86d8b8e58..482162e59 100644 --- a/dcargs/_shtab/__init__.py +++ b/dcargs/_shtab/__init__.py @@ -138,7 +138,7 @@ def complete2pattern(opt_complete, shell, choice_type2fn) -> bool: def wordify(string: str) -> str: """Replace non-word chars [-. ] with underscores [_]""" - return string.replace("-", "_").replace(".", " ").replace(" ", "_") + return re.compile(r"[-.\s:]").sub("_", string) def get_public_subcommands(sub): @@ -706,7 +706,7 @@ def command_option(prefix, options): def command_list(prefix, options): name = " ".join([prog, *options["paths"]]) commands = "\n ".join( - '"{}:{}"'.format(cmd, escape_zsh(opt["help"])) + '"{}:{}"'.format(escape_zsh(cmd), escape_zsh(opt["help"])) for cmd, opt in sorted(options["commands"].items()) ) return """ From 79543dcb9e997bb77c7f71f70b0062f92f8111bb Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 16 Sep 2022 09:47:13 -0700 Subject: [PATCH 140/491] formatter --- dcargs/_argparse_formatter.py | 205 +++++++++++++++++++++++++++++++++- dcargs/_parsers.py | 8 +- 2 files changed, 205 insertions(+), 8 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index a6dac3593..c3dfb743e 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -6,6 +6,14 @@ from typing import Any, ContextManager, Generator import termcolor +from rich.columns import Columns +from rich.console import Console, Group +from rich.padding import Padding +from rich.panel import Panel +from rich.rule import Rule +from rich.style import Style +from rich.table import Table +from rich.text import Text from . import _strings @@ -84,11 +92,11 @@ class _ArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): def __init__(self, prog, *, field_count: int): indent_increment = 2 width = shutil.get_terminal_size().columns - 2 - max_help_position = min(36, width // 3) # Usual is 24. + max_help_position = min(24, width // 3) # Usual is 24. # Try to make helptext more concise when we have a lot of fields! - if field_count > 16 and width >= 100: # pragma: no cover - max_help_position = min(96, width // 2) # Usual is 24. + # if field_count > 16 and width >= 100: # pragma: no cover + # max_help_position = min(96, width // 2) # Usual is 24. super().__init__(prog, indent_increment, max_help_position, width) @@ -113,6 +121,7 @@ def _format_action(self, action): # determine the required width and the entry label help_position = min(self._action_max_length + 2, self._max_help_position) help_width = max(self._width - help_position, 11) + help_width = min(60, help_width) action_width = help_position - self._current_indent - 2 action_header = self._format_action_invocation(action) @@ -173,6 +182,7 @@ def _format_action(self, action): def _split_lines(self, text, width): text = self._whitespace_matcher.sub(" ", text).strip() + return [text] # The textwrap module is used only for formatting help. # Delay its import for speeding up the common usage of argparse. import textwrap as textwrap @@ -185,3 +195,192 @@ def _split_lines(self, text, width): def _fill_text(self, text, width, indent): return "".join(indent + line for line in text.splitlines(keepends=True)) + + class _Section(object): + def __init__(self, formatter, parent, heading=None): + self.formatter = formatter + self.parent = parent + self.heading = heading + self.items = [] + + def format_help(self): + if self.parent is None: + return self._dcargs_format_root() + else: + return self._dcargs_format_nonroot() + + def _dcargs_format_root(self): + console = Console(width=self.formatter._width) + with console.capture() as capture: + top_parts = [] + column_parts = [] + panel_lines_cumsum = [0] + usage = None + for func, args in self.items: + item_content = func(*args) + if item_content is None: + pass + elif isinstance(item_content, str): + top_parts.append(Text.from_ansi(item_content)) + else: + # assert isinstance(item_content, Panel) + panel_lines_cumsum.append( + panel_lines_cumsum[-1] + + str(item_content).strip().count("\n") + # + 3 + ) + column_parts.append(item_content) + + if panel_lines_cumsum[-1] < 20: + print("!") + columns = Group(*column_parts) + else: + half_lines = panel_lines_cumsum[-1] // 2 + deltas = tuple( + map( + lambda l: l > half_lines, + panel_lines_cumsum, + ) + ) + (True,) + split_index = deltas.index(True) + columns = Columns( + [ + Group(*column_parts[:split_index]), + Group(*column_parts[split_index:]), + ], + column_first=True, + width=self.formatter._width // 2 - 2, + ) + + console.print(Padding(Group(*top_parts), pad=(0, 2))) + console.print(columns) + return capture.get() + + def _dcargs_format_nonroot(self): + # Add each child item as a rich renderable. + item_parts = [] + has_description = False + for func, args in self.items: + item_content = func(*args) + if ( + getattr(func, "__func__", None) + is _ArgparseHelpFormatter._format_action + ): + (action,) = args + assert isinstance(action, argparse.Action) + invocation = self.formatter._format_action_invocation(action) + + help_position = self.formatter._max_help_position + + # Put invocation and help side-by-side. + if ( + action.help + and len(_strings.strip_ansi_sequences(invocation)) + < help_position + 2 + ): + table = Table(show_header=False, box=None, padding=0) + table.add_column(width=help_position) + table.add_column() + table.add_row( + Text.from_ansi(invocation), Text.from_ansi(action.help) + ) + item_parts.append(table) + + # Put invocation and help on separate lines. + else: + item_parts.append( + Text.from_ansi( + invocation + "\n", + style=Style(encircle=True), + ) + ) + if action.help: + item_parts.append( + Padding( + Text.from_ansi(action.help), + pad=(0, 0, 0, help_position), + ) + ) + else: + assert isinstance(item_content, str) + if item_content.strip() != "": + has_description = True + item_parts.append( + Text.from_ansi( + item_content.strip() + "\n", + style=Style(dim=True), + ) + ) + item_parts.append( + Rule( + # title="item", + style=Style(dim=True), + align="left", + ) + ) + + if len(item_parts) == 0: + return None + + # Get heading. + if self.heading is not argparse.SUPPRESS and self.heading is not None: + current_indent = self.formatter._current_indent + heading = "%*s%s:\n" % (current_indent, "", self.heading) + else: + heading = "" + + # If no description: use the heading as one. + # if not has_description: + # item_parts.append( + # Text.from_ansi( + # heading.strip()[:-1], + # style=Style(dim=True), + # ) + # ) + # item_parts.append( + # Rule( + # # title="test", + # style=Style(dim=True), + # align="left", + # ) + # ) + # item_parts = item_parts[-2:] + item_parts[:-2] + + # return Group(*item_parts) + if not has_description: + return Panel( + Group(*item_parts), + title=heading.strip()[:-1], + title_align="left", + border_style=Style(dim=True), + padding=(1, 1, 0, 1), + ) + else: + return Panel( + Group(*item_parts), + # title=heading, + # title_align="left", + border_style=Style(dim=True), + ) + + # # format the indented section + # if self.parent is not None: + # self.formatter._indent() + # join = self.formatter._join_parts + # item_help = join([func(*args) for func, args in self.items]) + # if self.parent is not None: + # self.formatter._dedent() + # + # # return nothing if the section was empty + # if not item_help: + # return '' + # + # # add the heading if the section was non-empty + # if self.heading is not SUPPRESS and self.heading is not None: + # current_indent = self.formatter._current_indent + # heading = '%*s%s:\n' % (current_indent, '', self.heading) + # else: + # heading = '' + # + # # join the section-initial newline, the heading and the help + # return join(['\n', heading, item_help, '\n']) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index c2b7585e0..48eb51be5 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -192,9 +192,7 @@ def apply(self, parser: argparse.ArgumentParser) -> None: # Make argument groups. def format_group_name(nested_field_name: str) -> str: - return termcolor.colored( - (nested_field_name + " arguments").strip(), attrs=["bold"] - ) + return (nested_field_name + " arguments").strip() group_from_prefix: Dict[str, argparse._ArgumentGroup] = { "": parser._action_groups[1], @@ -203,7 +201,7 @@ def format_group_name(nested_field_name: str) -> str: # Break some API boundaries to rename the optional group. parser._action_groups[1].title = format_group_name("") positional_group = parser.add_argument_group( - termcolor.colored("positional arguments", attrs=["bold"]) + "positional arguments" ) parser._action_groups = parser._action_groups[::-1] @@ -433,7 +431,7 @@ def apply( dest=_strings.make_subparser_dest(self.prefix), description=self.description, required=self.required, - title=termcolor.colored(title, attrs=["bold"]), + title=title, metavar=metavar, ) From a7847812cf847f0664bd4edbf117fe24a4461fb0 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 16 Sep 2022 13:53:53 -0700 Subject: [PATCH 141/491] Help formatting with rich --- dcargs/_argparse_formatter.py | 444 +++++++++++++++++++--------------- dcargs/_instantiators.py | 8 +- dcargs/_parsers.py | 10 +- dcargs/_strings.py | 6 - poetry.lock | 71 +++++- pyproject.toml | 3 +- tests/test_dict_namedtuple.py | 14 +- tests/test_helptext.py | 55 ++--- 8 files changed, 355 insertions(+), 256 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index c3dfb743e..67828c474 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -1,13 +1,13 @@ import argparse import contextlib import functools -import itertools +import re as _re import shutil -from typing import Any, ContextManager, Generator +from typing import Any, ContextManager, Generator, List import termcolor from rich.columns import Columns -from rich.console import Console, Group +from rich.console import Console, Group, RenderableType from rich.padding import Padding from rich.panel import Panel from rich.rule import Rule @@ -17,6 +17,11 @@ from . import _strings +BORDER_STYLE = Style(color="bright_blue", dim=True) +DESCRIPTION_STYLE = Style(color="bright_blue", bold=True) +INVOCATION_STYLE = Style(color="bright_white", bold=True) +METAVAR_STYLE = Style(color="bright_blue") + def monkeypatch_len(obj: Any) -> int: if isinstance(obj, str): @@ -84,6 +89,13 @@ def inner() -> Generator[None, None, None]: return inner() +def str_from_rich(renderable: RenderableType) -> str: + console = Console() + with console.capture() as out: + console.print(renderable, soft_wrap=True) + return out.get().rstrip("\n") + + def make_formatter_class(field_count: int) -> Any: return functools.partial(_ArgparseHelpFormatter, field_count=field_count) @@ -92,11 +104,12 @@ class _ArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): def __init__(self, prog, *, field_count: int): indent_increment = 2 width = shutil.get_terminal_size().columns - 2 - max_help_position = min(24, width // 3) # Usual is 24. # Try to make helptext more concise when we have a lot of fields! - # if field_count > 16 and width >= 100: # pragma: no cover - # max_help_position = min(96, width // 2) # Usual is 24. + if field_count > 16 and width >= 100: # pragma: no cover + max_help_position = min(12, width // 2) # Usual is 24. + else: + max_help_position = min(24, width // 3) # Usual is 24. super().__init__(prog, indent_increment, max_help_position, width) @@ -104,85 +117,22 @@ def _format_args(self, action, default_metavar): """Override _format_args() to ignore nargs and always expect single string metavars.""" get_metavar = self._metavar_formatter(action, default_metavar) - return get_metavar(1)[0] + + out = get_metavar(1)[0] + if isinstance(out, str): + return str_from_rich(Text(out, style=METAVAR_STYLE)) + return out def add_argument(self, action): # pragma: no cover # Patch to avoid super long arguments from shifting the helptext of all of the # fields. prev_max_length = self._action_max_length super().add_argument(action) - if ( - self._action_max_length >= 40 - and self._action_max_length > self._max_help_position + 2 - ): + if self._action_max_length > self._max_help_position + 2: self._action_max_length = prev_max_length - def _format_action(self, action): - # determine the required width and the entry label - help_position = min(self._action_max_length + 2, self._max_help_position) - help_width = max(self._width - help_position, 11) - help_width = min(60, help_width) - action_width = help_position - self._current_indent - 2 - action_header = self._format_action_invocation(action) - - # no help; start on same line and add a final newline - if not action.help: - tup = self._current_indent, "", action_header - action_header = "%*s%s\n" % tup - - # short action name; start on the same line and pad two spaces - elif monkeypatch_len(action_header) <= action_width: - # Original: - # tup = self._current_indent, "", action_width, action_header - # action_header = "%*s%-*s " % tup - # - action_header = ( - " " * self._current_indent - + action_header - + " " * (action_width - monkeypatch_len(action_header) + 2) - ) - # - indent_first = 0 - - # long action name; start on the next line - else: - tup = self._current_indent, "", action_header - action_header = "%*s%s\n" % tup - indent_first = help_position - - # collect the pieces of the action help - parts = [action_header] - - # if there was help for the action, add lines of help text - if action.help: - help_text = self._expand_help(action) - # - # Respect existing line breaks. - help_lines = tuple( - itertools.chain( - *(self._split_lines(h, help_width) for h in help_text.split("\n")) - ) - ) - # - - parts.append("%*s%s\n" % (indent_first, "", help_lines[0])) # type: ignore - for line in help_lines[1:]: - parts.append("%*s%s\n" % (help_position, "", line)) - - # or add a newline if the description doesn't end with one - elif not action_header.endswith("\n"): - parts.append("\n") - - # if there are any sub-actions, add their help as well - for subaction in self._iter_indented_subactions(action): - parts.append(self._format_action(subaction)) - - # return a single string - return self._join_parts(parts) - def _split_lines(self, text, width): text = self._whitespace_matcher.sub(" ", text).strip() - return [text] # The textwrap module is used only for formatting help. # Delay its import for speeding up the common usage of argparse. import textwrap as textwrap @@ -212,54 +162,142 @@ def format_help(self): def _dcargs_format_root(self): console = Console(width=self.formatter._width) with console.capture() as capture: + # Get rich renderables from items. top_parts = [] column_parts = [] panel_lines_cumsum = [0] - usage = None for func, args in self.items: item_content = func(*args) if item_content is None: pass + + # Add strings. (usage, description, etc) elif isinstance(item_content, str): + if item_content.strip() == "": + continue top_parts.append(Text.from_ansi(item_content)) + + # Add panels. (argument groups, subcommands, etc) else: - # assert isinstance(item_content, Panel) + assert isinstance(item_content, Panel) + with console.capture() as length_capture: + console.print(item_content) panel_lines_cumsum.append( panel_lines_cumsum[-1] - + str(item_content).strip().count("\n") - # + 3 + + length_capture.get().strip().count("\n") + + 2 ) column_parts.append(item_content) - if panel_lines_cumsum[-1] < 20: - print("!") - columns = Group(*column_parts) - else: - half_lines = panel_lines_cumsum[-1] // 2 + def _index_closest_to(line_count): + """Find the index of the first panel where the line count is closest + to a target length.""" deltas = tuple( - map( - lambda l: l > half_lines, - panel_lines_cumsum, - ) - ) + (True,) - split_index = deltas.index(True) + map(lambda l: abs(l - line_count), panel_lines_cumsum) + ) + return deltas.index(min(deltas)) + + # Single column. + if panel_lines_cumsum[-1] < 40 or self.formatter._width < 160: + # Wrapping in columns here prevents everything from going + # full-width. + columns = Columns([Group(*column_parts)], column_first=True) + + # Two column mode. + elif panel_lines_cumsum[-1] < 120 or self.formatter._width < 205: + split_index = _index_closest_to(panel_lines_cumsum[-1] // 2) + column_width = self.formatter._width // 2 - 1 columns = Columns( [ Group(*column_parts[:split_index]), Group(*column_parts[split_index:]), ], column_first=True, - width=self.formatter._width // 2 - 2, + width=column_width, + ) + + # Three column mode. + else: + split_index1 = _index_closest_to(panel_lines_cumsum[-1] // 3) + split_index2 = _index_closest_to( + panel_lines_cumsum[split_index1] + panel_lines_cumsum[-1] // 3 + ) + column_width = self.formatter._width // 3 - 1 + columns = Columns( + [ + Group(*column_parts[:split_index1]), + Group(*column_parts[split_index1:split_index2]), + Group(*column_parts[split_index2:]), + ], + column_first=True, + width=column_width, ) - console.print(Padding(Group(*top_parts), pad=(0, 2))) + # Three column mode. + + console.print(Group(*top_parts)) console.print(columns) return capture.get() + def _format_action(self, action: argparse.Action): + invocation = self.formatter._format_action_invocation(action) + help_position = self.formatter._action_max_length + + item_parts: List[RenderableType] = [] + + # Put invocation and help side-by-side. + if ( + action.help + and len(_strings.strip_ansi_sequences(invocation)) < help_position - 1 + ): + table = Table(show_header=False, box=None, padding=0) + table.add_column(width=help_position) + table.add_column() + table.add_row( + Text.from_ansi( + invocation, + style=INVOCATION_STYLE, + ), + # Unescape % signs, which need special handling in argparse. + Text.from_ansi(action.help.replace("%%", "%")), + ) + item_parts.append(table) + + # Put invocation and help on separate lines. + else: + item_parts.append( + Text.from_ansi( + invocation + "\n", + style=INVOCATION_STYLE, + ) + ) + if action.help: + item_parts.append( + Padding( + # Unescape % signs, which need special handling in argparse. + Text.from_ansi(action.help.replace("%%", "%")), + pad=(0, 0, 0, help_position), + ) + ) + + # Add subactions, indented. + try: + subaction: argparse.Action + for subaction in action._get_subactions(): # type: ignore + item_parts.append( + Padding( + Group(*self._format_action(subaction)), pad=(0, 0, 0, 4) + ) + ) + except AttributeError: + pass + + return item_parts + def _dcargs_format_nonroot(self): # Add each child item as a rich renderable. + description_part = None item_parts = [] - has_description = False for func, args in self.items: item_content = func(*args) if ( @@ -268,55 +306,17 @@ def _dcargs_format_nonroot(self): ): (action,) = args assert isinstance(action, argparse.Action) - invocation = self.formatter._format_action_invocation(action) - - help_position = self.formatter._max_help_position - - # Put invocation and help side-by-side. - if ( - action.help - and len(_strings.strip_ansi_sequences(invocation)) - < help_position + 2 - ): - table = Table(show_header=False, box=None, padding=0) - table.add_column(width=help_position) - table.add_column() - table.add_row( - Text.from_ansi(invocation), Text.from_ansi(action.help) - ) - item_parts.append(table) + item_parts.extend(self._format_action(action)) - # Put invocation and help on separate lines. - else: - item_parts.append( - Text.from_ansi( - invocation + "\n", - style=Style(encircle=True), - ) - ) - if action.help: - item_parts.append( - Padding( - Text.from_ansi(action.help), - pad=(0, 0, 0, help_position), - ) - ) else: assert isinstance(item_content, str) if item_content.strip() != "": - has_description = True - item_parts.append( - Text.from_ansi( - item_content.strip() + "\n", - style=Style(dim=True), - ) - ) - item_parts.append( - Rule( - # title="item", - style=Style(dim=True), - align="left", - ) + assert ( + description_part is None + ) # Should only have one description part. + description_part = Text.from_ansi( + " " + item_content.strip() + "\n", + style=DESCRIPTION_STYLE, ) if len(item_parts) == 0: @@ -326,61 +326,123 @@ def _dcargs_format_nonroot(self): if self.heading is not argparse.SUPPRESS and self.heading is not None: current_indent = self.formatter._current_indent heading = "%*s%s:\n" % (current_indent, "", self.heading) + # Remove colon from heading. + heading = heading.strip()[:-1] else: heading = "" - # If no description: use the heading as one. - # if not has_description: - # item_parts.append( - # Text.from_ansi( - # heading.strip()[:-1], - # style=Style(dim=True), - # ) - # ) - # item_parts.append( - # Rule( - # # title="test", - # style=Style(dim=True), - # align="left", - # ) - # ) - # item_parts = item_parts[-2:] + item_parts[:-2] - - # return Group(*item_parts) - if not has_description: - return Panel( - Group(*item_parts), - title=heading.strip()[:-1], - title_align="left", - border_style=Style(dim=True), - padding=(1, 1, 0, 1), - ) + if description_part is not None: + item_parts = [description_part, Rule(style=BORDER_STYLE)] + item_parts + + return Panel( + Group(*item_parts), + title=heading, + title_align="left", + border_style=BORDER_STYLE, + # padding=(1, 1, 0, 1), + ) + + def _format_actions_usage(self, actions, groups): + # find group indices and identify actions in groups + group_actions = set() + inserts = {} + for group in groups: + try: + start = actions.index(group._group_actions[0]) # type: ignore + except ValueError: + continue else: - return Panel( - Group(*item_parts), - # title=heading, - # title_align="left", - border_style=Style(dim=True), - ) + end = start + len(group._group_actions) + if actions[start:end] == group._group_actions: # type: ignore + for action in group._group_actions: + group_actions.add(action) + if not group.required: # type: ignore + if start in inserts: + inserts[start] += " [" + else: + inserts[start] = "[" + if end in inserts: + inserts[end] += "]" + else: + inserts[end] = "]" + else: + if start in inserts: + inserts[start] += " (" + else: + inserts[start] = "(" + if end in inserts: + inserts[end] += ")" + else: + inserts[end] = ")" + for i in range(start + 1, end): + inserts[i] = "|" + + # collect all actions format strings + parts = [] + for i, action in enumerate(actions): + # suppressed arguments are marked with None + # remove | separators for suppressed arguments + if action.help is argparse.SUPPRESS: + parts.append(None) + if inserts.get(i) == "|": + inserts.pop(i) + elif inserts.get(i + 1) == "|": + inserts.pop(i + 1) + + # produce all arg strings + elif not action.option_strings: + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) + + # if it's in a group, strip the outer [] + if action in group_actions: + if part[0] == "[" and part[-1] == "]": + part = part[1:-1] + + # add the action string to the list + parts.append(part) + + # produce the first way to invoke the option in brackets + else: + option_string = action.option_strings[0] + + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = "%s" % option_string - # # format the indented section - # if self.parent is not None: - # self.formatter._indent() - # join = self.formatter._join_parts - # item_help = join([func(*args) for func, args in self.items]) - # if self.parent is not None: - # self.formatter._dedent() - # - # # return nothing if the section was empty - # if not item_help: - # return '' - # - # # add the heading if the section was non-empty - # if self.heading is not SUPPRESS and self.heading is not None: - # current_indent = self.formatter._current_indent - # heading = '%*s%s:\n' % (current_indent, '', self.heading) - # else: - # heading = '' - # - # # join the section-initial newline, the heading and the help - # return join(['\n', heading, item_help, '\n']) + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + part = "%s %s" % (option_string, args_string) + + # make it look optional if it's not required or in a group + if not action.required and action not in group_actions: + part = "[%s]" % part + + # Apply invocation style. + part = str_from_rich(Text(part, style=INVOCATION_STYLE)) + + # add the action string to the list + parts.append(part) + + # insert things at the necessary indices + for i in sorted(inserts, reverse=True): + parts[i:i] = [inserts[i]] + + # join all the action items with spaces + text = " ".join([item for item in parts if item is not None]) + + # clean up separators for mutually exclusive groups + open = r"[\[(]" + close = r"[\])]" + text = _re.sub(r"(%s) " % open, r"\1", text) + text = _re.sub(r" (%s)" % close, r"\1", text) + text = _re.sub(r"%s *%s" % (open, close), r"", text) + text = _re.sub(r"\(([^|]*)\)", r"\1", text) + text = text.strip() + + # return the text + return text diff --git a/dcargs/_instantiators.py b/dcargs/_instantiators.py index d7ff78c4f..62c3e4b1d 100644 --- a/dcargs/_instantiators.py +++ b/dcargs/_instantiators.py @@ -126,7 +126,7 @@ def instantiator(strings: List[str]) -> None: return instantiator, InstantiatorMetadata( nargs=1, - metavar="{" + _strings.format_metavar("None") + "}", + metavar="{None}", choices=("None",), ) @@ -213,9 +213,9 @@ def instantiator_base_case(strings: List[str]) -> Any: return instantiator_base_case, InstantiatorMetadata( nargs=1, - metavar=_strings.format_metavar(typ.__name__.upper()) + metavar=typ.__name__.upper() if auto_choices is None - else "{" + ",".join(map(_strings.format_metavar, map(str, auto_choices))) + "}", + else "{" + ",".join(map(str, auto_choices)) + "}", choices=auto_choices, ) @@ -574,7 +574,7 @@ def _instantiator_from_literal( lambda strings: choices[str_choices.index(strings[0])], InstantiatorMetadata( nargs=1, - metavar="{" + ",".join(map(_strings.format_metavar, str_choices)) + "}", + metavar="{" + ",".join(str_choices) + "}", choices=str_choices, ), ) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 48eb51be5..204d1cec3 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -6,7 +6,6 @@ import itertools from typing import Any, Callable, Dict, List, Optional, Set, Type, TypeVar, Union, cast -import termcolor from typing_extensions import get_args, get_origin from . import ( @@ -200,9 +199,7 @@ def format_group_name(nested_field_name: str) -> str: # Break some API boundaries to rename the optional group. parser._action_groups[1].title = format_group_name("") - positional_group = parser.add_argument_group( - "positional arguments" - ) + positional_group = parser.add_argument_group("positional arguments") parser._action_groups = parser._action_groups[::-1] # Add each argument group. Note that groups with only suppressed arguments won't @@ -212,11 +209,10 @@ def format_group_name(nested_field_name: str) -> str: arg.lowered.help is not argparse.SUPPRESS and arg.prefix not in group_from_prefix ): + description = self.helptext_from_nested_class_field_name.get(arg.prefix) group_from_prefix[arg.prefix] = parser.add_argument_group( format_group_name(arg.prefix), - description=self.helptext_from_nested_class_field_name.get( - arg.prefix - ), + description=description, ) # Add each argument. diff --git a/dcargs/_strings.py b/dcargs/_strings.py index 8df3ed826..af87d1b4e 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -5,8 +5,6 @@ import textwrap from typing import Iterable, List, Sequence, Tuple, Type, Union -import termcolor - from . import _resolver dummy_field_name = "__dcargs_dummy_field__" @@ -114,10 +112,6 @@ def strip_ansi_sequences(x: str): return _get_ansi_pattern().sub("", x) -def format_metavar(x: str) -> str: - return termcolor.colored(x, attrs=["bold"]) - - def multi_metavar_from_single(single: str) -> str: if len(strip_ansi_sequences(single)) >= 32: # Shorten long metavars diff --git a/poetry.lock b/poetry.lock index 14a891815..2a1e2f3ab 100644 --- a/poetry.lock +++ b/poetry.lock @@ -44,6 +44,17 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "commonmark" +version = "0.9.1" +description = "Python parser for the CommonMark Markdown spec" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] + [[package]] name = "coverage" version = "6.4.4" @@ -191,6 +202,17 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "Pygments" +version = "2.13.0" +description = "Pygments is a syntax highlighting package written in Python." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +plugins = ["importlib-metadata"] + [[package]] name = "pyparsing" version = "3.0.9" @@ -263,6 +285,22 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "rich" +version = "12.5.1" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" +optional = false +python-versions = ">=3.6.3,<4.0.0" + +[package.dependencies] +commonmark = ">=0.9.0,<0.10.0" +pygments = ">=2.6.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] + [[package]] name = "setuptools" version = "65.3.0" @@ -276,14 +314,6 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-g testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] -[[package]] -name = "shtab" -version = "1.5.5" -description = "Automagic shell tab completion for Python CLI applications" -category = "main" -optional = false -python-versions = ">=3.2" - [[package]] name = "termcolor" version = "1.1.0" @@ -342,7 +372,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "880ee7ca4570c69fe1fadfac1b13d81c6719eaf7b45420bbb587985f9e9ef5ed" +content-hash = "d1bd3ffbe8589f4941e07f3216d6d1ae26436163899658c6056a9996038d4d12" [metadata.files] antlr4-python3-runtime = [ @@ -363,6 +393,10 @@ colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] +commonmark = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] coverage = [ {file = "coverage-6.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e7b4da9bafad21ea45a714d3ea6f3e1679099e420c8741c74905b92ee9bfa7cc"}, {file = "coverage-6.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fde17bc42e0716c94bf19d92e4c9f5a00c5feb401f5bc01101fdf2a8b7cacf60"}, @@ -525,6 +559,10 @@ py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] +Pygments = [ + {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, + {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, +] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, @@ -549,6 +587,13 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -576,14 +621,14 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +rich = [ + {file = "rich-12.5.1-py3-none-any.whl", hash = "sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb"}, + {file = "rich-12.5.1.tar.gz", hash = "sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca"}, +] setuptools = [ {file = "setuptools-65.3.0-py3-none-any.whl", hash = "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82"}, {file = "setuptools-65.3.0.tar.gz", hash = "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57"}, ] -shtab = [ - {file = "shtab-1.5.5-py2.py3-none-any.whl", hash = "sha256:f4bf7cc122fb434cb65b96db7e3adcf3b5258846e906e60c48b3725d2965c6b5"}, - {file = "shtab-1.5.5.tar.gz", hash = "sha256:f90a6ce64b821002d5881b6212992a27ab40c3bab36aabca8de118b0b78f61f6"}, -] termcolor = [ {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, ] diff --git a/pyproject.toml b/pyproject.toml index e0aade281..6fd837fa1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.2" +version = "0.3.3" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] @@ -19,6 +19,7 @@ termcolor = "^1.1.0" colorama = {version = "^0.4.0", platform = "win32"} frozendict = "^2.3.4" # shtab = "^1.5.5" +rich = "^12.5.1" [tool.poetry.dev-dependencies] pytest = "^7.1.2" diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index c129a4241..0156ea766 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -188,7 +188,7 @@ class HelptextTypedDict(TypedDict): assert "--z INT" in helptext assert "Documentation 1 (required)" in helptext assert "Documentation 2 (required)" in helptext - assert "Documentation 3 (default: 3)\n" in helptext + assert "Documentation 3 (default: 3)" in helptext def test_basic_namedtuple(): @@ -249,9 +249,9 @@ class HelptextNamedTupleDefault(NamedTuple): assert "--x INT" in helptext assert "--y INT" in helptext assert "--z INT" in helptext - assert "Documentation 1 (required)\n" in helptext - assert "Documentation 2 (required)\n" in helptext - assert "Documentation 3 (default: 3)\n" in helptext + assert "Documentation 1 (required)" in helptext + assert "Documentation 2 (required)" in helptext + assert "Documentation 3 (default: 3)" in helptext def test_helptext_and_default_namedtuple_alternate(): @@ -287,7 +287,7 @@ class HelptextNamedTuple(NamedTuple): ) helptext = dcargs._strings.strip_ansi_sequences(f.getvalue()) assert cast(str, HelptextNamedTuple.__doc__) in helptext - assert "Documentation 1 (required)\n" in helptext - assert "Documentation 2 (required)\n" in helptext + assert "Documentation 1 (required)" in helptext + assert "Documentation 2 (required)" in helptext assert "Documentation 3" in helptext - assert "(default: 3)\n" in helptext + assert "(default: 3)" in helptext diff --git a/tests/test_helptext.py b/tests/test_helptext.py index a3b1a907c..11d394078 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -49,9 +49,9 @@ class Helptext: assert "x INT" in helptext assert "y INT" in helptext assert "z INT" in helptext - assert "Documentation 1 (required)\n" in helptext - assert "Documentation 2 (required)\n" in helptext - assert "Documentation 3 (default: 3)\n" in helptext + assert "Documentation 1 (required)" in helptext + assert "Documentation 2 (required)" in helptext + assert "Documentation 3 (default: 3)" in helptext def test_helptext_from_class_docstring(): @@ -75,9 +75,9 @@ class Helptext2: assert "x INT" in helptext assert "y INT" in helptext assert "z INT" in helptext - assert "Documentation 1 (required)\n" in helptext - assert "Documentation 2 (required)\n" in helptext - assert "Documentation 3 (default: 3)\n" in helptext + assert "Documentation 1 (required)" in helptext + assert "Documentation 2 (required)" in helptext + assert "Documentation 3 (default: 3)" in helptext def test_helptext_from_class_docstring_args(): @@ -101,9 +101,9 @@ class Helptext3: assert "x INT" in helptext assert "y INT" in helptext assert "z INT" in helptext - assert "Documentation 1 (required)\n" in helptext - assert "Documentation 2 (required)\n" in helptext - assert "Documentation 3 (default: 3)\n" in helptext + assert "Documentation 1 (required)" in helptext + assert "Documentation 2 (required)" in helptext + assert "Documentation 3 (default: 3)" in helptext def test_helptext_inherited(): @@ -211,8 +211,9 @@ class HelptextWithVariousDefaults: y: Color = Color.RED helptext = _get_helptext(HelptextWithVariousDefaults) - assert "show this help message and exit\n --x PATH" in helptext - assert "(default: /some/path/to/a/file)\n" in helptext + assert "show this help message and exit" in helptext + assert "--x PATH" in helptext + assert "(default: /some/path/to/a/file)" in helptext assert "--y {RED,GREEN,BLUE}" in helptext assert "(default: RED)" in helptext @@ -233,7 +234,7 @@ class HelptextMultiline: Next line of documentation 3""" helptext = _get_helptext(HelptextMultiline) - assert "Documentation 1 (required)\n" in helptext + assert "Documentation 1 (required)" in helptext assert "Documentation 2" in helptext assert "documentation 2" in helptext assert "Documentation 3" in helptext @@ -249,9 +250,9 @@ class HelptextGrouped: z: int = 3 helptext = _get_helptext(HelptextGrouped) - assert "Documentation 1 (required)\n" in helptext - assert "Description of both y and z. (required)\n" in helptext - assert "Description of both y and z. (default: 3)\n" in helptext + assert "Documentation 1 (required)" in helptext + assert "Description of both y and z. (required)" in helptext + assert "Description of both y and z. (default: 3)" in helptext def test_none_default_value_helptext(): @@ -261,8 +262,8 @@ class Config: """An optional variable.""" helptext = _get_helptext(Config) - assert " --x {None}|INT" in helptext - assert "An optional variable. (default: None)\n" in helptext + assert "--x {None}|INT" in helptext + assert "An optional variable. (default: None)" in helptext def test_helptext_hard_bool(): @@ -280,7 +281,7 @@ class HelptextHardString: helptext = _get_helptext(HelptextHardString) assert "--x" in helptext - assert "Helptext. 2% milk." in helptext + assert "2% milk." in helptext def test_helptext_with_inheritance(): @@ -333,7 +334,7 @@ class TupleHelptext: x: Tuple[int, str, float] helptext = _get_helptext(TupleHelptext) - assert "--x INT STR FLOAT\n" in helptext + assert "--x INT STR FLOAT" in helptext def test_tuple_helptext_defaults(): @@ -343,7 +344,7 @@ class TupleHelptextDefaults: helptext = _get_helptext(TupleHelptextDefaults) assert "--x INT STR STR" in helptext - assert "(default: 5 'hello world' hello)\n" in helptext + assert "(default: 5 'hello world' hello)" in helptext def test_generic_helptext(): @@ -354,7 +355,7 @@ class GenericTupleHelptext(Generic[T]): x: T helptext = _get_helptext(GenericTupleHelptext[int]) - assert "--x INT\n" in helptext + assert "--x INT" in helptext def test_generic_tuple_helptext(): @@ -365,7 +366,7 @@ class GenericTupleHelptext(Generic[T]): x: Tuple[T, T, T] helptext = _get_helptext(GenericTupleHelptext[int]) - assert "--x INT INT INT\n" in helptext + assert "--x INT INT INT" in helptext def test_generic_list_helptext(): @@ -376,7 +377,7 @@ class GenericTupleHelptext(Generic[T]): x: List[T] helptext = _get_helptext(GenericTupleHelptext[int]) - assert "--x INT [INT ...]\n" in helptext + assert "--x INT [INT ...]" in helptext def test_literal_helptext(): @@ -387,7 +388,7 @@ class LiteralHelptext: helptext = _get_helptext(LiteralHelptext) assert "--x {1,2,3}" in helptext - assert "A number. (required)\n" in helptext + assert "A number. (required)" in helptext def test_optional_literal_helptext(): @@ -398,7 +399,7 @@ class OptionalLiteralHelptext: helptext = _get_helptext(OptionalLiteralHelptext) assert "--x {None,1,2,3}" in helptext - assert "A number. (default: None)\n" in helptext + assert "A number. (default: None)" in helptext def test_multiple_subparsers_helptext(): @@ -457,8 +458,8 @@ class OptionalHelptext: helptext = _get_helptext(OptionalHelptext) assert cast(str, OptionalHelptext.__doc__) in helptext assert "--x {None}|INT" in helptext - assert "--y {None}|INT [{None}|INT ...]\n" in helptext - assert "[--z {None}|INT]\n" in helptext + assert "--y {None}|INT [{None}|INT ...]" in helptext + assert "[--z {None}|INT]" in helptext def test_metavar_0(): From 9b95d7fd8d8bddd41abb97dd95278d8fcef9b04b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 17 Sep 2022 00:55:22 -0700 Subject: [PATCH 142/491] Clean up rich formatter --- dcargs/_argparse_formatter.py | 205 +++++++++------------------------- dcargs/extras/__init__.py | 13 ++- 2 files changed, 62 insertions(+), 156 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index 67828c474..2a4a43376 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -1,7 +1,6 @@ import argparse import contextlib import functools -import re as _re import shutil from typing import Any, ContextManager, Generator, List @@ -17,10 +16,26 @@ from . import _strings -BORDER_STYLE = Style(color="bright_blue", dim=True) -DESCRIPTION_STYLE = Style(color="bright_blue", bold=True) -INVOCATION_STYLE = Style(color="bright_white", bold=True) -METAVAR_STYLE = Style(color="bright_blue") +BORDER_STYLE = Style() +DESCRIPTION_STYLE = Style() +INVOCATION_STYLE = Style() +METAVAR_STYLE = Style() + + +def set_accent_color(accent_color: str) -> None: + """Set an accent color to use in help messages. Takes any color supported by `rich`, + see `python -m rich.color`. Experimental.""" + global BORDER_STYLE + BORDER_STYLE = Style(color=accent_color, dim=True) + global DESCRIPTION_STYLE + DESCRIPTION_STYLE = Style(color=accent_color, bold=True) + global INVOCATION_STYLE + INVOCATION_STYLE = Style(bold=True) + global METAVAR_STYLE + METAVAR_STYLE = Style(color=accent_color) + + +set_accent_color("color(30)") def monkeypatch_len(obj: Any) -> int: @@ -165,7 +180,7 @@ def _dcargs_format_root(self): # Get rich renderables from items. top_parts = [] column_parts = [] - panel_lines_cumsum = [0] + column_parts_lines_cumsum = [0] for func, args in self.items: item_content = func(*args) if item_content is None: @@ -180,11 +195,9 @@ def _dcargs_format_root(self): # Add panels. (argument groups, subcommands, etc) else: assert isinstance(item_content, Panel) - with console.capture() as length_capture: - console.print(item_content) - panel_lines_cumsum.append( - panel_lines_cumsum[-1] - + length_capture.get().strip().count("\n") + column_parts_lines_cumsum.append( + column_parts_lines_cumsum[-1] + + str_from_rich(item_content).strip().count("\n") + 2 ) column_parts.append(item_content) @@ -193,47 +206,38 @@ def _index_closest_to(line_count): """Find the index of the first panel where the line count is closest to a target length.""" deltas = tuple( - map(lambda l: abs(l - line_count), panel_lines_cumsum) + map(lambda l: abs(l - line_count), column_parts_lines_cumsum) ) return deltas.index(min(deltas)) - # Single column. - if panel_lines_cumsum[-1] < 40 or self.formatter._width < 160: - # Wrapping in columns here prevents everything from going - # full-width. - columns = Columns([Group(*column_parts)], column_first=True) - - # Two column mode. - elif panel_lines_cumsum[-1] < 120 or self.formatter._width < 205: - split_index = _index_closest_to(panel_lines_cumsum[-1] // 2) - column_width = self.formatter._width // 2 - 1 - columns = Columns( - [ - Group(*column_parts[:split_index]), - Group(*column_parts[split_index:]), - ], - column_first=True, - width=column_width, - ) - - # Three column mode. - else: - split_index1 = _index_closest_to(panel_lines_cumsum[-1] // 3) - split_index2 = _index_closest_to( - panel_lines_cumsum[split_index1] + panel_lines_cumsum[-1] // 3 - ) - column_width = self.formatter._width // 3 - 1 - columns = Columns( - [ - Group(*column_parts[:split_index1]), - Group(*column_parts[split_index1:split_index2]), - Group(*column_parts[split_index2:]), - ], - column_first=True, - width=column_width, + # Split into columns. + min_column_width = 65 + height_breakpoint = 50 + column_count = max( + 1, + min( + column_parts_lines_cumsum[-1] // height_breakpoint + 1, + self.formatter._width // min_column_width, + ), + ) + split_indices = [0] + for i in range(1, column_count): + split_indices.append( + _index_closest_to( + column_parts_lines_cumsum[-1] // column_count * i + ) ) - - # Three column mode. + split_indices.append(len(column_parts)) + columns = Columns( + [ + Group(*column_parts[split_indices[i] : split_indices[i + 1]]) + for i in range(column_count) + ], + column_first=True, + width=self.formatter._width // column_count - 1 + if column_count > 1 + else None, + ) console.print(Group(*top_parts)) console.print(columns) @@ -341,108 +345,3 @@ def _dcargs_format_nonroot(self): border_style=BORDER_STYLE, # padding=(1, 1, 0, 1), ) - - def _format_actions_usage(self, actions, groups): - # find group indices and identify actions in groups - group_actions = set() - inserts = {} - for group in groups: - try: - start = actions.index(group._group_actions[0]) # type: ignore - except ValueError: - continue - else: - end = start + len(group._group_actions) - if actions[start:end] == group._group_actions: # type: ignore - for action in group._group_actions: - group_actions.add(action) - if not group.required: # type: ignore - if start in inserts: - inserts[start] += " [" - else: - inserts[start] = "[" - if end in inserts: - inserts[end] += "]" - else: - inserts[end] = "]" - else: - if start in inserts: - inserts[start] += " (" - else: - inserts[start] = "(" - if end in inserts: - inserts[end] += ")" - else: - inserts[end] = ")" - for i in range(start + 1, end): - inserts[i] = "|" - - # collect all actions format strings - parts = [] - for i, action in enumerate(actions): - # suppressed arguments are marked with None - # remove | separators for suppressed arguments - if action.help is argparse.SUPPRESS: - parts.append(None) - if inserts.get(i) == "|": - inserts.pop(i) - elif inserts.get(i + 1) == "|": - inserts.pop(i + 1) - - # produce all arg strings - elif not action.option_strings: - default = self._get_default_metavar_for_positional(action) - part = self._format_args(action, default) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == "[" and part[-1] == "]": - part = part[1:-1] - - # add the action string to the list - parts.append(part) - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = "%s" % option_string - - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - part = "%s %s" % (option_string, args_string) - - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = "[%s]" % part - - # Apply invocation style. - part = str_from_rich(Text(part, style=INVOCATION_STYLE)) - - # add the action string to the list - parts.append(part) - - # insert things at the necessary indices - for i in sorted(inserts, reverse=True): - parts[i:i] = [inserts[i]] - - # join all the action items with spaces - text = " ".join([item for item in parts if item is not None]) - - # clean up separators for mutually exclusive groups - open = r"[\[(]" - close = r"[\])]" - text = _re.sub(r"(%s) " % open, r"\1", text) - text = _re.sub(r" (%s)" % close, r"\1", text) - text = _re.sub(r"%s *%s" % (open, close), r"", text) - text = _re.sub(r"\(([^|]*)\)", r"\1", text) - text = text.strip() - - # return the text - return text diff --git a/dcargs/extras/__init__.py b/dcargs/extras/__init__.py index d51a0b898..9a332d059 100644 --- a/dcargs/extras/__init__.py +++ b/dcargs/extras/__init__.py @@ -1,7 +1,14 @@ -"""The :mod:`dcargs.extras` submodule contains helpers that complement :func:`dcargs.cli()`, but -aren't considered part of the core interface.""" +"""The :mod:`dcargs.extras` submodule contains helpers that complement :func:`dcargs.cli()`. +Compared to the core interface, APIs here are more likely to be changed or deprecated. """ + +from .._argparse_formatter import set_accent_color from ._base_configs import subcommand_type_from_defaults from ._serialization import from_yaml, to_yaml -__all__ = ["subcommand_type_from_defaults", "to_yaml", "from_yaml"] +__all__ = [ + "set_accent_color", + "subcommand_type_from_defaults", + "to_yaml", + "from_yaml", +] From c6aa30de0402856d3718f5f8d91d5ea2f584c106 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 17 Sep 2022 18:25:55 -0700 Subject: [PATCH 143/491] Minor layout/visual improvements --- dcargs/_argparse_formatter.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index 2a4a43376..aeacee8c1 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -2,7 +2,7 @@ import contextlib import functools import shutil -from typing import Any, ContextManager, Generator, List +from typing import Any, ContextManager, Generator, List, Optional import termcolor from rich.columns import Columns @@ -22,7 +22,7 @@ METAVAR_STYLE = Style() -def set_accent_color(accent_color: str) -> None: +def set_accent_color(accent_color: Optional[str]) -> None: """Set an accent color to use in help messages. Takes any color supported by `rich`, see `python -m rich.color`. Experimental.""" global BORDER_STYLE @@ -32,10 +32,10 @@ def set_accent_color(accent_color: str) -> None: global INVOCATION_STYLE INVOCATION_STYLE = Style(bold=True) global METAVAR_STYLE - METAVAR_STYLE = Style(color=accent_color) + METAVAR_STYLE = Style(color=accent_color, bold=True) -set_accent_color("color(30)") +set_accent_color(None) def monkeypatch_len(obj: Any) -> int: @@ -121,10 +121,11 @@ def __init__(self, prog, *, field_count: int): width = shutil.get_terminal_size().columns - 2 # Try to make helptext more concise when we have a lot of fields! - if field_count > 16 and width >= 100: # pragma: no cover + if field_count > 32: # pragma: no cover + # When there are more fields, make helptext more compact. max_help_position = min(12, width // 2) # Usual is 24. else: - max_help_position = min(24, width // 3) # Usual is 24. + max_help_position = min(36, width // 3) # Usual is 24. super().__init__(prog, indent_increment, max_help_position, width) From 070390a7df9d916fc46086ea5e7a20acbdd57d07 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 17 Sep 2022 18:40:28 -0700 Subject: [PATCH 144/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6fd837fa1..7b6b91b4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.3" +version = "0.3.4" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From 6b46e1fce526c82fc4cd8904072267c0786ff895 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 17 Sep 2022 18:53:48 -0700 Subject: [PATCH 145/491] Layout tweaks --- dcargs/_argparse_formatter.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index aeacee8c1..b6145a776 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -320,7 +320,7 @@ def _dcargs_format_nonroot(self): description_part is None ) # Should only have one description part. description_part = Text.from_ansi( - " " + item_content.strip() + "\n", + item_content.strip() + "\n", style=DESCRIPTION_STYLE, ) diff --git a/pyproject.toml b/pyproject.toml index 7b6b91b4b..59a323f93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.4" +version = "0.3.5" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From 2d0cce7d8bc3fcb5773e4580b4d584c6f7e83280 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 19 Sep 2022 00:07:05 -0700 Subject: [PATCH 146/491] More layout improvements, bump version --- dcargs/_argparse_formatter.py | 89 +++++++++++++++++++---------------- pyproject.toml | 2 +- 2 files changed, 50 insertions(+), 41 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index b6145a776..c2959be77 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -104,10 +104,12 @@ def inner() -> Generator[None, None, None]: return inner() -def str_from_rich(renderable: RenderableType) -> str: - console = Console() +def str_from_rich( + renderable: RenderableType, width: Optional[int] = None, soft_wrap: bool = False +) -> str: + console = Console(width=width) with console.capture() as out: - console.print(renderable, soft_wrap=True) + console.print(renderable, soft_wrap=soft_wrap) return out.get().rstrip("\n") @@ -117,15 +119,15 @@ def make_formatter_class(field_count: int) -> Any: class _ArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): def __init__(self, prog, *, field_count: int): - indent_increment = 2 + indent_increment = 4 width = shutil.get_terminal_size().columns - 2 # Try to make helptext more concise when we have a lot of fields! - if field_count > 32: # pragma: no cover + if field_count > 64: # pragma: no cover # When there are more fields, make helptext more compact. - max_help_position = min(12, width // 2) # Usual is 24. + max_help_position = 8 else: - max_help_position = min(36, width // 3) # Usual is 24. + max_help_position = 36 # Usual is 24. super().__init__(prog, indent_increment, max_help_position, width) @@ -136,7 +138,8 @@ def _format_args(self, action, default_metavar): out = get_metavar(1)[0] if isinstance(out, str): - return str_from_rich(Text(out, style=METAVAR_STYLE)) + # Can result in an failed argparse assertion if we turn off soft wrapping. + return str_from_rich(Text(out, style=METAVAR_STYLE), soft_wrap=True) return out def add_argument(self, action): # pragma: no cover @@ -181,7 +184,7 @@ def _dcargs_format_root(self): # Get rich renderables from items. top_parts = [] column_parts = [] - column_parts_lines_cumsum = [0] + column_parts_lines = [] for func, args in self.items: item_content = func(*args) if item_content is None: @@ -196,20 +199,13 @@ def _dcargs_format_root(self): # Add panels. (argument groups, subcommands, etc) else: assert isinstance(item_content, Panel) - column_parts_lines_cumsum.append( - column_parts_lines_cumsum[-1] - + str_from_rich(item_content).strip().count("\n") - + 2 - ) column_parts.append(item_content) - - def _index_closest_to(line_count): - """Find the index of the first panel where the line count is closest - to a target length.""" - deltas = tuple( - map(lambda l: abs(l - line_count), column_parts_lines_cumsum) - ) - return deltas.index(min(deltas)) + # Estimate line count. This won't correctly account for + # wrapping, as we don't know the column layout yet. + column_parts_lines.append( + str_from_rich(item_content, width=65).strip().count("\n") + + 1 + ) # Split into columns. min_column_width = 65 @@ -217,27 +213,34 @@ def _index_closest_to(line_count): column_count = max( 1, min( - column_parts_lines_cumsum[-1] // height_breakpoint + 1, + sum(column_parts_lines) // height_breakpoint + 1, self.formatter._width // min_column_width, ), ) - split_indices = [0] - for i in range(1, column_count): - split_indices.append( - _index_closest_to( - column_parts_lines_cumsum[-1] // column_count * i - ) + if column_count > 1: + column_width = self.formatter._width // column_count - 1 + # Correct the line count for each panel using the known column + # width. This will account for word wrap. + column_parts_lines = map( + lambda p: str_from_rich(p, width=column_width) + .strip() + .count("\n") + + 1, + column_parts, ) - split_indices.append(len(column_parts)) + else: + column_width = None + + column_lines = [0 for i in range(column_count)] + column_parts_grouped = [[] for i in range(column_count)] + for p, l in zip(column_parts, column_parts_lines): + chosen_column = column_lines.index(min(column_lines)) + column_parts_grouped[chosen_column].append(p) + column_lines[chosen_column] += l columns = Columns( - [ - Group(*column_parts[split_indices[i] : split_indices[i + 1]]) - for i in range(column_count) - ], + [Group(*g) for g in column_parts_grouped], column_first=True, - width=self.formatter._width // column_count - 1 - if column_count > 1 - else None, + width=column_width, ) console.print(Group(*top_parts)) @@ -246,7 +249,10 @@ def _index_closest_to(line_count): def _format_action(self, action: argparse.Action): invocation = self.formatter._format_action_invocation(action) - help_position = self.formatter._action_max_length + help_position = min( + self.formatter._action_max_length + 4, self.formatter._max_help_position + ) + indent = self.formatter._current_indent item_parts: List[RenderableType] = [] @@ -256,7 +262,7 @@ def _format_action(self, action: argparse.Action): and len(_strings.strip_ansi_sequences(invocation)) < help_position - 1 ): table = Table(show_header=False, box=None, padding=0) - table.add_column(width=help_position) + table.add_column(width=help_position - indent) table.add_column() table.add_row( Text.from_ansi( @@ -289,11 +295,14 @@ def _format_action(self, action: argparse.Action): try: subaction: argparse.Action for subaction in action._get_subactions(): # type: ignore + self.formatter._indent() item_parts.append( Padding( - Group(*self._format_action(subaction)), pad=(0, 0, 0, 4) + Group(*self._format_action(subaction)), + pad=(0, 0, 0, self.formatter._indent_increment), ) ) + self.formatter._dedent() except AttributeError: pass diff --git a/pyproject.toml b/pyproject.toml index 59a323f93..54fc14350 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.5" +version = "0.3.6" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From e0b5ec5d6151b414fa519a9d287fc2b22a037261 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 19 Sep 2022 11:13:52 -0700 Subject: [PATCH 147/491] Formatting refactor, remove termcolor dependency --- dcargs/_argparse_formatter.py | 96 ++++++++++++++++++++--------------- dcargs/_arguments.py | 21 +++++--- dcargs/_cli.py | 14 ++--- poetry.lock | 20 +------- pyproject.toml | 3 +- tests/test_helptext.py | 27 +++++++--- 6 files changed, 98 insertions(+), 83 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index c2959be77..b3eff1c8b 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -1,10 +1,10 @@ import argparse import contextlib +import dataclasses import functools import shutil -from typing import Any, ContextManager, Generator, List, Optional +from typing import Any, ContextManager, Dict, Generator, List, Optional -import termcolor from rich.columns import Columns from rich.console import Console, Group, RenderableType from rich.padding import Padding @@ -13,28 +13,45 @@ from rich.style import Style from rich.table import Table from rich.text import Text +from rich.theme import Theme from . import _strings -BORDER_STYLE = Style() -DESCRIPTION_STYLE = Style() -INVOCATION_STYLE = Style() -METAVAR_STYLE = Style() + +@dataclasses.dataclass +class DcargsTheme: + border: Style = Style() + description: Style = Style() + invocation: Style = Style() + metavar: Style = Style() + metavar_fixed: Style = Style() + helptext: Style = Style() + helptext_required: Style = Style() + helptext_default: Style = Style() + + def as_rich_theme(self) -> Theme: + return Theme(vars(self)) def set_accent_color(accent_color: Optional[str]) -> None: """Set an accent color to use in help messages. Takes any color supported by `rich`, see `python -m rich.color`. Experimental.""" - global BORDER_STYLE - BORDER_STYLE = Style(color=accent_color, dim=True) - global DESCRIPTION_STYLE - DESCRIPTION_STYLE = Style(color=accent_color, bold=True) - global INVOCATION_STYLE - INVOCATION_STYLE = Style(bold=True) - global METAVAR_STYLE - METAVAR_STYLE = Style(color=accent_color, bold=True) - - + THEME.border = Style(color=accent_color, dim=True) + THEME.description = Style(color=accent_color, bold=True) + THEME.invocation = Style() + THEME.metavar = Style(color=accent_color, bold=True) + THEME.metavar_fixed = Style(color="red", bold=True) + THEME.helptext = Style(dim=True) + THEME.helptext_required = Style(color="bright_red", bold=True) + THEME.helptext_default = Style( + color=accent_color if accent_color is not None else "cyan", + dim=True, + ) + + +# TODO: this is a prototype; for a v1.0.0 release we should revisit whether the global +# state here is acceptable or not. +THEME = DcargsTheme() set_accent_color(None) @@ -45,22 +62,6 @@ def monkeypatch_len(obj: Any) -> int: return len(obj) -def dummy_termcolor_context() -> ContextManager[None]: - """Context for turning termcolor off.""" - - def dummy_colored(*args, **kwargs) -> str: - return args[0] - - @contextlib.contextmanager - def inner() -> Generator[None, None, None]: - orig_colored = termcolor.colored - termcolor.colored = dummy_colored - yield - termcolor.colored = orig_colored - - return inner() - - def ansi_context() -> ContextManager[None]: """Context for working with ANSI codes + argparse: - Applies a temporary monkey patch for making argparse ignore ANSI codes when @@ -107,7 +108,7 @@ def inner() -> Generator[None, None, None]: def str_from_rich( renderable: RenderableType, width: Optional[int] = None, soft_wrap: bool = False ) -> str: - console = Console(width=width) + console = Console(width=width, theme=THEME.as_rich_theme()) with console.capture() as out: console.print(renderable, soft_wrap=soft_wrap) return out.get().rstrip("\n") @@ -139,7 +140,13 @@ def _format_args(self, action, default_metavar): out = get_metavar(1)[0] if isinstance(out, str): # Can result in an failed argparse assertion if we turn off soft wrapping. - return str_from_rich(Text(out, style=METAVAR_STYLE), soft_wrap=True) + return str_from_rich( + Text( + out, + style=THEME.metavar_fixed if out == "{fixed}" else THEME.metavar, + ), + soft_wrap=True, + ) return out def add_argument(self, action): # pragma: no cover @@ -179,7 +186,7 @@ def format_help(self): return self._dcargs_format_nonroot() def _dcargs_format_root(self): - console = Console(width=self.formatter._width) + console = Console(width=self.formatter._width, theme=THEME.as_rich_theme()) with console.capture() as capture: # Get rich renderables from items. top_parts = [] @@ -257,6 +264,11 @@ def _format_action(self, action: argparse.Action): item_parts: List[RenderableType] = [] # Put invocation and help side-by-side. + if action.option_strings == ["-h", "--help"]: + # Darken helptext for --help flag. This makes it visually consistent + # with the helptext strings defined via docstrings and set by + # _arguments.py. + action.help = "[helptext]" + action.help + "[/helptext]" if ( action.help and len(_strings.strip_ansi_sequences(invocation)) < help_position - 1 @@ -267,10 +279,10 @@ def _format_action(self, action: argparse.Action): table.add_row( Text.from_ansi( invocation, - style=INVOCATION_STYLE, + style=THEME.invocation, ), # Unescape % signs, which need special handling in argparse. - Text.from_ansi(action.help.replace("%%", "%")), + Text.from_markup(action.help.replace("%%", "%")), ) item_parts.append(table) @@ -279,14 +291,14 @@ def _format_action(self, action: argparse.Action): item_parts.append( Text.from_ansi( invocation + "\n", - style=INVOCATION_STYLE, + style=THEME.invocation, ) ) if action.help: item_parts.append( Padding( # Unescape % signs, which need special handling in argparse. - Text.from_ansi(action.help.replace("%%", "%")), + Text.from_markup(action.help.replace("%%", "%")), pad=(0, 0, 0, help_position), ) ) @@ -330,7 +342,7 @@ def _dcargs_format_nonroot(self): ) # Should only have one description part. description_part = Text.from_ansi( item_content.strip() + "\n", - style=DESCRIPTION_STYLE, + style=THEME.description, ) if len(item_parts) == 0: @@ -346,12 +358,12 @@ def _dcargs_format_nonroot(self): heading = "" if description_part is not None: - item_parts = [description_part, Rule(style=BORDER_STYLE)] + item_parts + item_parts = [description_part, Rule(style=THEME.border)] + item_parts return Panel( Group(*item_parts), title=heading, title_align="left", - border_style=BORDER_STYLE, + border_style=THEME.border, # padding=(1, 1, 0, 1), ) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 8c0cc9e3c..1842c304f 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -21,8 +21,6 @@ Union, ) -import termcolor - from . import _fields, _instantiators, _resolver, _strings from .conf import _markers @@ -197,7 +195,7 @@ def _rule_recursive_instantiator_from_type( return dataclasses.replace( lowered, instantiator=None, - metavar=termcolor.colored("{fixed}", color="red"), + metavar="{fixed}", required=False, default=_fields.MISSING_PROP, ) @@ -220,7 +218,7 @@ def _rule_recursive_instantiator_from_type( # available. return dataclasses.replace( lowered, - metavar=termcolor.colored("{fixed}", color="red"), + metavar="{fixed}", required=False, default=_fields.MISSING_PROP, ) @@ -263,6 +261,15 @@ def as_str(x: Any) -> Tuple[str, ...]: return dataclasses.replace(lowered, default=as_str(lowered.default)) +# This can be turned off when we don't want rich-based formatting. (notably for +# completion scripts) +USE_RICH = True + + +def _rich_tag_if_enabled(x: str, tag: str): + return x if not USE_RICH else f"[{tag}]{x}[/{tag}]" + + def _rule_generate_helptext( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, @@ -281,7 +288,7 @@ def _rule_generate_helptext( # Note that the percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string docstring_help = docstring_help.replace("%", "%%") - help_parts.append(docstring_help) + help_parts.append(_rich_tag_if_enabled(docstring_help, "helptext")) default = lowered.default if lowered.is_fixed(): @@ -316,9 +323,9 @@ def _rule_generate_helptext( default_text = f"(default: {' '.join(default_parts)})" else: default_text = f"(default: {shlex.quote(str(default))})" - help_parts.append(termcolor.colored(default_text, attrs=["dark"])) + help_parts.append(_rich_tag_if_enabled(default_text, "helptext_default")) else: - help_parts.append(termcolor.colored("(required)", color="red", attrs=["bold"])) + help_parts.append(_rich_tag_if_enabled("(required)", "helptext_required")) return dataclasses.replace(lowered, help=" ".join(help_parts)) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index aa9a345cf..c0a66d6aa 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -5,7 +5,7 @@ import warnings from typing import Callable, Optional, Sequence, Type, TypeVar, Union, cast, overload -from . import _argparse_formatter, _calling, _fields, _parsers +from . import _argparse_formatter, _arguments, _calling, _fields, _parsers from . import _shtab as shtab from . import _strings, conf @@ -173,21 +173,20 @@ def fix_arg(arg: str) -> str: args = list(map(fix_arg, args)) - # If we pass in the --dcargs-print-completion flag: turn termcolor off, and get the - # shell we want to generate a completion script for (bash/zsh/tcsh). + # If we pass in the --dcargs-print-completion flag: turn formatting tags, and get + # the shell we want to generate a completion script for (bash/zsh/tcsh). # # Note that shtab also offers an add_argument_to() functions that fulfills a similar - # goal, but manual parsing of argv is convenient for turning off colors. + # goal, but manual parsing of argv is convenient for turning off formatting. print_completion = len(args) >= 2 and args[0] == "--dcargs-print-completion" - formatting_context = _argparse_formatter.ansi_context() completion_shell = None if print_completion: - formatting_context = _argparse_formatter.dummy_termcolor_context() + _arguments.USE_RICH = False completion_shell = args[1] # Generate parser! - with formatting_context: + with _argparse_formatter.ansi_context(): parser = argparse.ArgumentParser( prog=prog, formatter_class=_argparse_formatter.make_formatter_class( @@ -197,6 +196,7 @@ def fix_arg(arg: str) -> str: parser_definition.apply(parser) if print_completion: + _arguments.USE_RICH = True assert completion_shell in ("bash", "zsh", "tcsh",), ( "Shell should be one `bash`, `zsh`, or `tcsh`, but got" f" {completion_shell}" diff --git a/poetry.lock b/poetry.lock index 2a1e2f3ab..6c2086046 100644 --- a/poetry.lock +++ b/poetry.lock @@ -314,14 +314,6 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-g testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] -[[package]] -name = "termcolor" -version = "1.1.0" -description = "ANSII Color formatting for output in terminal." -category = "main" -optional = false -python-versions = "*" - [[package]] name = "tomli" version = "2.0.1" @@ -372,7 +364,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "d1bd3ffbe8589f4941e07f3216d6d1ae26436163899658c6056a9996038d4d12" +content-hash = "d1f0b8af20560d4fbb7e3feaa34db00e90872d0cff42756047c44cac67773c7e" [metadata.files] antlr4-python3-runtime = [ @@ -587,13 +579,6 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -629,9 +614,6 @@ setuptools = [ {file = "setuptools-65.3.0-py3-none-any.whl", hash = "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82"}, {file = "setuptools-65.3.0.tar.gz", hash = "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57"}, ] -termcolor = [ - {file = "termcolor-1.1.0.tar.gz", hash = "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b"}, -] tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, diff --git a/pyproject.toml b/pyproject.toml index 54fc14350..80a484af9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.6" +version = "0.3.7" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] @@ -14,7 +14,6 @@ python = "^3.7" docstring-parser = "^0.14.1" typing-extensions = "^4.3.0" PyYAML = "^6.0" -termcolor = "^1.1.0" "backports.cached-property" = { version = "^1.0.2", python = "~3.7" } colorama = {version = "^0.4.0", platform = "win32"} frozendict = "^2.3.4" diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 11d394078..db03188dd 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -2,6 +2,7 @@ import dataclasses import enum import io +import os import pathlib from collections.abc import Callable from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar, Union, cast @@ -11,7 +12,7 @@ from typing_extensions import Annotated, Literal import dcargs -import dcargs._argparse_formatter +import dcargs._arguments import dcargs._strings @@ -20,14 +21,28 @@ def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: with pytest.raises(SystemExit), contextlib.redirect_stdout(target): dcargs.cli(f, args=args) - # Check helptext with vs without termcolor. This can help catch text wrapping bugs + # Completion scripts; just smoke test for now. + with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): + dcargs.cli(f, args=["--dcargs-print-completion", "bash"]) + with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): + dcargs.cli(f, args=["--dcargs-print-completion", "zsh"]) + + # Check helptext with vs without formatting. This can help catch text wrapping bugs # caused by ANSI sequences. target2 = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): - with dcargs._argparse_formatter.dummy_termcolor_context(): - dcargs.cli(f, args=args) + dcargs._arguments.USE_RICH = False + dcargs.cli(f, args=args) + dcargs._arguments.USE_RICH = True + + if target2.getvalue() != dcargs._strings.strip_ansi_sequences(target.getvalue()): + raise AssertionError( + "Potential wrapping bug! These two strings should match:\n" + + target2.getvalue() + + "\n\n" + + dcargs._strings.strip_ansi_sequences(target.getvalue()) + ) - assert target2.getvalue() == dcargs._strings.strip_ansi_sequences(target.getvalue()) return target2.getvalue() @@ -45,7 +60,7 @@ class Helptext: """Documentation 3""" helptext = _get_helptext(Helptext) - assert cast(str, Helptext.__doc__) in helptext + assert cast(str, helptext) in helptext assert "x INT" in helptext assert "y INT" in helptext assert "z INT" in helptext From a233a60f1d93e99121f5f1c1defb70ae304d8e68 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 19 Sep 2022 11:17:37 -0700 Subject: [PATCH 148/491] Mypy fix --- dcargs/_argparse_formatter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index b3eff1c8b..fa3f64878 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -268,6 +268,7 @@ def _format_action(self, action: argparse.Action): # Darken helptext for --help flag. This makes it visually consistent # with the helptext strings defined via docstrings and set by # _arguments.py. + assert action.help is not None action.help = "[helptext]" + action.help + "[/helptext]" if ( action.help From f7e4844fcb19c25ad267a14a1fd83d7625077b2a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 20 Sep 2022 18:40:24 -0700 Subject: [PATCH 149/491] Add example, tests for initializing Flax modules --- docs/source/examples/17_flax_modules.rst | 76 +++ examples/17_flax_modules.py | 52 ++ poetry.lock | 690 ++++++++++++++++++++++- pyproject.toml | 6 +- tests/test_flax.py | 60 ++ 5 files changed, 853 insertions(+), 31 deletions(-) create mode 100644 docs/source/examples/17_flax_modules.rst create mode 100644 examples/17_flax_modules.py create mode 100644 tests/test_flax.py diff --git a/docs/source/examples/17_flax_modules.rst b/docs/source/examples/17_flax_modules.rst new file mode 100644 index 000000000..64b6ef71d --- /dev/null +++ b/docs/source/examples/17_flax_modules.rst @@ -0,0 +1,76 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +17. Flax Modules +========================================== + + +If you use `Flax `_\ , modules can be instantiated +directly from ``dcargs.cli``. + + + +.. code-block:: python + :linenos: + + + from flax import linen as nn + from jax import numpy as jnp + + import dcargs + + + class Classifier(nn.Module): + layers: int + """Layers in our network.""" + units: int = 32 + """Hidden unit count.""" + output_dim: int = 10 + """Number of classes.""" + + @nn.compact + def __call__(self, x: jnp.ndarray) -> jnp.ndarray: # type: ignore + for i in range(self.layers - 1): + x = nn.Dense( + self.units, + kernel_init=nn.initializers.kaiming_normal(), + )(x) + x = nn.relu(x) + + x = nn.Dense( + self.output_dim, + kernel_init=nn.initializers.xavier_normal(), + )(x) + x = nn.sigmoid(x) + return x + + + def train(model: Classifier, num_iterations: int = 1000) -> None: + """Train a model. + + Args: + model: Model to train. + num_iterations: Number of training iterations. + """ + print(f"{model=}") + print(f"{num_iterations=}") + + + if __name__ == "__main__": + dcargs.cli(train) + +------------ + +.. raw:: html + + python 17_flax_modules.py --help + +.. program-output:: python ../../examples/17_flax_modules.py --help + +------------ + +.. raw:: html + + python 17_flax_modules.py --model.layers 4 + +.. program-output:: python ../../examples/17_flax_modules.py --model.layers 4 diff --git a/examples/17_flax_modules.py b/examples/17_flax_modules.py new file mode 100644 index 000000000..37ebb3d17 --- /dev/null +++ b/examples/17_flax_modules.py @@ -0,0 +1,52 @@ +"""If you use [Flax](https://github.com/google/flax), modules can be instantiated +directly from `dcargs.cli`. + +Usage: +`python ./17_flax_modules.py --help` +`python ./17_flax_modules.py --model.layers 4` +""" + +from flax import linen as nn +from jax import numpy as jnp + +import dcargs + + +class Classifier(nn.Module): + layers: int + """Layers in our network.""" + units: int = 32 + """Hidden unit count.""" + output_dim: int = 10 + """Number of classes.""" + + @nn.compact + def __call__(self, x: jnp.ndarray) -> jnp.ndarray: # type: ignore + for i in range(self.layers - 1): + x = nn.Dense( + self.units, + kernel_init=nn.initializers.kaiming_normal(), + )(x) + x = nn.relu(x) + + x = nn.Dense( + self.output_dim, + kernel_init=nn.initializers.xavier_normal(), + )(x) + x = nn.sigmoid(x) + return x + + +def train(model: Classifier, num_iterations: int = 1000) -> None: + """Train a model. + + Args: + model: Model to train. + num_iterations: Number of training iterations. + """ + print(f"{model=}") + print(f"{num_iterations=}") + + +if __name__ == "__main__": + dcargs.cli(train) diff --git a/poetry.lock b/poetry.lock index 6c2086046..2d40220f0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,18 +1,18 @@ [[package]] -name = "antlr4-python3-runtime" -version = "4.9.3" -description = "ANTLR 4.9.3 runtime for Python 3.7" +name = "absl-py" +version = "1.2.0" +description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.6" [[package]] -name = "atomicwrites" -version = "1.4.1" -description = "Atomic file writes." +name = "antlr4-python3-runtime" +version = "4.9.3" +description = "ANTLR 4.9.3 runtime for Python 3.7" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = "*" [[package]] name = "attrs" @@ -36,6 +36,22 @@ category = "main" optional = false python-versions = ">=3.6.0" +[[package]] +name = "chex" +version = "0.1.5" +description = "Chex: Testing made fun, in JAX!" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +absl-py = ">=0.9.0" +dm-tree = ">=0.1.5" +jax = ">=0.1.55" +jaxlib = ">=0.1.37" +numpy = ">=1.18.0" +toolz = ">=0.9.0" + [[package]] name = "colorama" version = "0.4.5" @@ -69,6 +85,22 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 [package.extras] toml = ["tomli"] +[[package]] +name = "cycler" +version = "0.11.0" +description = "Composable style cycles" +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "dm-tree" +version = "0.1.7" +description = "Tree is a library for working with nested data structures." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "docstring-parser" version = "0.14.1" @@ -77,6 +109,78 @@ category = "main" optional = false python-versions = ">=3.6,<4.0" +[[package]] +name = "etils" +version = "0.8.0" +description = "Collection of common python utils" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +importlib_resources = {version = "*", optional = true, markers = "extra == \"epath\""} +typing_extensions = {version = "*", optional = true, markers = "extra == \"epath\""} +zipp = {version = "*", optional = true, markers = "extra == \"epath\""} + +[package.extras] +all = ["etils[array-types]", "etils[ecolab]", "etils[edc]", "etils[enp]", "etils[epath]", "etils[epy]", "etils[etqdm]", "etils[etree-dm]", "etils[etree-jax]", "etils[etree-tf]", "etils[etree]"] +array-types = ["etils[enp]"] +dev = ["chex", "pylint (>=2.6.0)", "pytest", "pytest-subtests", "pytest-xdist", "yapf"] +ecolab = ["etils[enp]", "etils[epy]", "jupyter", "mediapy", "numpy"] +edc = ["etils[epy]"] +enp = ["etils[epy]", "numpy"] +epath = ["etils[epy]", "importlib_resources", "typing_extensions", "zipp"] +epy = ["typing_extensions"] +etqdm = ["absl-py", "etils[epy]", "tqdm"] +etree = ["etils[array_types]", "etils[enp]", "etils[epy]", "etils[etqdm]"] +etree-dm = ["dm-tree", "etils[etree]"] +etree-jax = ["etils[etree]", "jax[cpu]"] +etree-tf = ["etils[etree]", "tf-nightly"] +lazy-imports = ["etils[ecolab]"] + +[[package]] +name = "flax" +version = "0.6.0" +description = "Flax: A neural network library for JAX designed for flexibility" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +jax = ">=0.3.16" +matplotlib = "*" +msgpack = "*" +numpy = ">=1.12" +optax = "*" +PyYAML = ">=5.4.1" +rich = ">=11.1,<12.0" +typing-extensions = ">=4.1.1" + +[package.extras] +testing = ["atari-py (==0.2.5)", "clu", "gym (==0.18.3)", "jaxlib", "jraph (>=0.0.6dev0)", "ml-collections", "opencv-python", "pytest", "pytest-cov", "pytest-custom-exit-code", "pytest-xdist (==1.34.0)", "pytype", "sentencepiece", "svn", "tensorflow", "tensorflow-datasets", "tensorflow-text (>=2.4.0)", "torch"] + +[[package]] +name = "fonttools" +version = "4.37.3" +description = "Tools to manipulate font files" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +all = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "fs (>=2.2.0,<3)", "lxml (>=4.0,<5)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres", "scipy", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=14.0.0)", "xattr", "zopfli (>=0.1.4)"] +graphite = ["lz4 (>=1.7.4.2)"] +interpolatable = ["munkres", "scipy"] +lxml = ["lxml (>=4.0,<5)"] +pathops = ["skia-pathops (>=0.5.0)"] +plot = ["matplotlib"] +repacker = ["uharfbuzz (>=0.23.0)"] +symfont = ["sympy"] +type1 = ["xattr"] +ufo = ["fs (>=2.2.0,<3)"] +unicode = ["unicodedata2 (>=14.0.0)"] +woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] + [[package]] name = "frozendict" version = "2.3.4" @@ -102,6 +206,21 @@ docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] perf = ["ipython"] testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] +[[package]] +name = "importlib-resources" +version = "5.9.0" +description = "Read resources from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] +testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + [[package]] name = "iniconfig" version = "1.1.1" @@ -110,6 +229,83 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "jax" +version = "0.3.17" +description = "Differentiate, compile, and transform Numpy code." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +absl-py = "*" +etils = {version = "*", extras = ["epath"]} +numpy = ">=1.20" +opt_einsum = "*" +scipy = ">=1.5" +typing_extensions = "*" + +[package.extras] +australis = ["protobuf (>=3.13,<4)"] +ci = ["jaxlib (==0.3.15)"] +cpu = ["jaxlib (==0.3.15)"] +cuda = ["jaxlib (==0.3.15+cuda11.cudnn82)"] +cuda11_cudnn805 = ["jaxlib (==0.3.15+cuda11.cudnn805)"] +cuda11_cudnn82 = ["jaxlib (==0.3.15+cuda11.cudnn82)"] +minimum-jaxlib = ["jaxlib (==0.3.14)"] +tpu = ["jaxlib (==0.3.15)", "libtpu-nightly (==0.1.dev20220723)", "requests"] + +[[package]] +name = "jaxlib" +version = "0.3.15" +description = "XLA library for JAX" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +absl-py = "*" +numpy = ">=1.19" +scipy = ">=1.5" + +[[package]] +name = "kiwisolver" +version = "1.4.4" +description = "A fast implementation of the Cassowary constraint solver" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +typing-extensions = {version = "*", markers = "python_version < \"3.8\""} + +[[package]] +name = "matplotlib" +version = "3.5.3" +description = "Python plotting package" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +cycler = ">=0.10" +fonttools = ">=4.22.0" +kiwisolver = ">=1.0.1" +numpy = ">=1.17" +packaging = ">=20.0" +pillow = ">=6.2.0" +pyparsing = ">=2.2.1" +python-dateutil = ">=2.7" +setuptools_scm = ">=4,<7" + +[[package]] +name = "msgpack" +version = "1.0.4" +description = "MessagePack serializer" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "mypy" version = "0.971" @@ -158,7 +354,7 @@ python-versions = ">=3.7" [[package]] name = "omegaconf" -version = "2.2.2" +version = "2.2.3" description = "A flexible configuration library" category = "dev" optional = false @@ -168,6 +364,37 @@ python-versions = ">=3.6" antlr4-python3-runtime = ">=4.9.0,<4.10.0" PyYAML = ">=5.1.0" +[[package]] +name = "opt-einsum" +version = "3.3.0" +description = "Optimizing numpys einsum function" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +numpy = ">=1.7" + +[package.extras] +docs = ["numpydoc", "sphinx (==1.2.3)", "sphinx-rtd-theme", "sphinxcontrib-napoleon"] +tests = ["pytest", "pytest-cov", "pytest-pep8"] + +[[package]] +name = "optax" +version = "0.1.3" +description = "A gradient processing and optimisation library in JAX." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +absl-py = ">=0.7.1" +chex = ">=0.0.4" +jax = ">=0.1.55" +jaxlib = ">=0.1.37" +numpy = ">=1.18.0" +typing-extensions = ">=3.10.0" + [[package]] name = "packaging" version = "21.3" @@ -179,6 +406,18 @@ python-versions = ">=3.6" [package.dependencies] pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" +[[package]] +name = "Pillow" +version = "9.2.0" +description = "Python Imaging Library (Fork)" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-issues (>=3.0.1)", "sphinx-removed-in", "sphinxext-opengraph"] +tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] + [[package]] name = "pluggy" version = "1.0.0" @@ -226,7 +465,7 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pyright" -version = "1.1.266" +version = "1.1.271" description = "Command line wrapper for pyright" category = "dev" optional = false @@ -242,14 +481,13 @@ dev = ["twine (>=3.4.1)"] [[package]] name = "pytest" -version = "7.1.2" +version = "7.1.3" description = "pytest: simple powerful testing with Python" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} @@ -278,7 +516,18 @@ pytest = ">=4.6" testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] [[package]] -name = "pyyaml" +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "PyYAML" version = "6.0" description = "YAML parser and emitter for Python" category = "main" @@ -287,20 +536,32 @@ python-versions = ">=3.6" [[package]] name = "rich" -version = "12.5.1" +version = "11.2.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false -python-versions = ">=3.6.3,<4.0.0" +python-versions = ">=3.6.2,<4.0.0" [package.dependencies] +colorama = ">=0.4.0,<0.5.0" commonmark = ">=0.9.0,<0.10.0" pygments = ">=2.6.0,<3.0.0" -typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} +typing-extensions = {version = ">=3.7.4,<5.0", markers = "python_version < \"3.8\""} [package.extras] jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] +[[package]] +name = "scipy" +version = "1.6.1" +description = "SciPy: Scientific Library for Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +numpy = ">=1.16.5" + [[package]] name = "setuptools" version = "65.3.0" @@ -314,6 +575,31 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-g testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +[[package]] +name = "setuptools-scm" +version = "6.4.2" +description = "the blessed package to manage your versions by scm tags" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +packaging = ">=20.0" +setuptools = "*" +tomli = ">=1.0.0" + +[package.extras] +test = ["pytest (>=6.2)", "virtualenv (>20)"] +toml = ["setuptools (>=42)"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + [[package]] name = "tomli" version = "2.0.1" @@ -322,6 +608,14 @@ category = "dev" optional = false python-versions = ">=3.7" +[[package]] +name = "toolz" +version = "0.12.0" +description = "List processing tools and functional utilities" +category = "dev" +optional = false +python-versions = ">=3.5" + [[package]] name = "torch" version = "1.12.1" @@ -364,15 +658,16 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "d1f0b8af20560d4fbb7e3feaa34db00e90872d0cff42756047c44cac67773c7e" +content-hash = "efb24b1ca37536bb4af0e0c2d80937cc1880a0cd1cb5af5dab562f6205db9696" [metadata.files] +absl-py = [ + {file = "absl-py-1.2.0.tar.gz", hash = "sha256:f568809938c49abbda89826223c992b630afd23c638160ad7840cfe347710d97"}, + {file = "absl_py-1.2.0-py3-none-any.whl", hash = "sha256:5d15f85b8cc859c6245bc9886ba664460ed96a6fee895416caa37d669ee74a9a"}, +] antlr4-python3-runtime = [ {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, ] -atomicwrites = [ - {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, -] attrs = [ {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, @@ -381,6 +676,10 @@ attrs = [ {file = "backports.cached-property-1.0.2.tar.gz", hash = "sha256:9306f9eed6ec55fd156ace6bc1094e2c86fae5fb2bf07b6a9c00745c656e75dd"}, {file = "backports.cached_property-1.0.2-py3-none-any.whl", hash = "sha256:baeb28e1cd619a3c9ab8941431fe34e8490861fb998c6c4590693d50171db0cc"}, ] +chex = [ + {file = "chex-0.1.5-py3-none-any.whl", hash = "sha256:b3321184850d5fc29b2eca63087cdbdd83a1b3e4f33c1314ff8b3b8bd67abbca"}, + {file = "chex-0.1.5.tar.gz", hash = "sha256:686858320f8f220c82a6c7eeb54dcdcaa4f3d7f66690dacd13a24baa1ee8299e"}, +] colorama = [ {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, @@ -441,10 +740,50 @@ coverage = [ {file = "coverage-6.4.4-pp36.pp37.pp38-none-any.whl", hash = "sha256:f67cf9f406cf0d2f08a3515ce2db5b82625a7257f88aad87904674def6ddaec1"}, {file = "coverage-6.4.4.tar.gz", hash = "sha256:e16c45b726acb780e1e6f88b286d3c10b3914ab03438f32117c4aa52d7f30d58"}, ] +cycler = [ + {file = "cycler-0.11.0-py3-none-any.whl", hash = "sha256:3a27e95f763a428a739d2add979fa7494c912a32c17c4c38c4d5f082cad165a3"}, + {file = "cycler-0.11.0.tar.gz", hash = "sha256:9c87405839a19696e837b3b818fed3f5f69f16f1eec1a1ad77e043dcea9c772f"}, +] +dm-tree = [ + {file = "dm-tree-0.1.7.tar.gz", hash = "sha256:30fec8aca5b92823c0e796a2f33b875b4dccd470b57e91e6c542405c5f77fd2a"}, + {file = "dm_tree-0.1.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3fae437135b6cbbdd51e96488a35e78c3617defa0b65265e7e8752d506f933fd"}, + {file = "dm_tree-0.1.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d377bd621b485db42c4aeea0eabbd8f6274b89a9c338c2c1bf69a40c3b86a1fd"}, + {file = "dm_tree-0.1.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1410fa2f2cc8dc7c01386f4e93ddeeb56765574ffafb632a9b6bd96496195b10"}, + {file = "dm_tree-0.1.7-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57edb6fbd88fcdd9908547cbf21045a9d663c0d9e5983dca7e6f9cf8b6584bb5"}, + {file = "dm_tree-0.1.7-cp310-cp310-win_amd64.whl", hash = "sha256:9edc1783a08d87c4e130781f55cbd904d6a564f7cce7dfb63f9ef3bee8e38209"}, + {file = "dm_tree-0.1.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:98fce150ceebb0a818f0eace1616004031cfa5e3375f50599ad790ff52414ba9"}, + {file = "dm_tree-0.1.7-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b4364fc9a5721a2b840ac8ea75b8f58b430bec9fdc8b99304d2aecb3cfe46b1b"}, + {file = "dm_tree-0.1.7-cp36-cp36m-win_amd64.whl", hash = "sha256:a085f500b295a6bf439c538e9058c7798ecb8c7d0dc916291f3d8d79d6124d17"}, + {file = "dm_tree-0.1.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f3e2bd9b9c05d1a0039f7c128d8b055c8a05708ef569cdbbeec0a2946e425bd4"}, + {file = "dm_tree-0.1.7-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:91c6240e47c9d80dbd7de5a29a2ca663143717a72c613130ba8ac4354fa741a9"}, + {file = "dm_tree-0.1.7-cp37-cp37m-win_amd64.whl", hash = "sha256:0f01743cc2247170e64798c6b4b31853717054bf9ceec47a1b1b8c2a4baf5792"}, + {file = "dm_tree-0.1.7-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:4992ac5c42af1d73042cd2d3af4e7892d3750e6c1bb8e5a4f81534aa6515f350"}, + {file = "dm_tree-0.1.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:20f24cad4decbf4c1f176a959d16e877c73df33b07d7d1f078a5b8abe72f79f8"}, + {file = "dm_tree-0.1.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3166304411d14c50a5da1c583e24d6069b44de0c9e06479cb36cdf048a466945"}, + {file = "dm_tree-0.1.7-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3b00885c21267934a3d3c68660811d3f891c9539fd53712f5b2423c6d74bf1e6"}, + {file = "dm_tree-0.1.7-cp38-cp38-win_amd64.whl", hash = "sha256:7f1f3dca9d669f3c09654ff6d69cfafd86a7f967c3095405b2692ee8d8ef3cfd"}, + {file = "dm_tree-0.1.7-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:51b9bdf1109b47cc22884b1919e6fe38edf28b5aa02e7c661bb760a0e7cf0157"}, + {file = "dm_tree-0.1.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2a843608e078d1622ebb5e50962a8c718d3fa1ab9461b95a12395a803545b2f5"}, + {file = "dm_tree-0.1.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7fa0740b7fbae2c3a43a3114a514891b5d6c383050828f36aa1816cf40f73a6a"}, + {file = "dm_tree-0.1.7-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1379a02df36e2bbff9819ceafa55ccd436b15af398803f781f372f8ead7ed871"}, + {file = "dm_tree-0.1.7-cp39-cp39-win_amd64.whl", hash = "sha256:3ca0a58e219b7b0bc201fea4679971188d0a9028a2543c16803a84e8f8c7eb2c"}, +] docstring-parser = [ {file = "docstring_parser-0.14.1-py3-none-any.whl", hash = "sha256:14ac6ec1f1ba6905c4d8cb90fd0bc55394f5678183752c90e44812bf28d7a515"}, {file = "docstring_parser-0.14.1.tar.gz", hash = "sha256:2c77522e31b7c88b1ab457a1f3c9ae38947ad719732260ba77ee8a3deb58622a"}, ] +etils = [ + {file = "etils-0.8.0-py3-none-any.whl", hash = "sha256:b6299d2a835206b063498e443bd8036edb974058899169677dd6ddb92430bb84"}, + {file = "etils-0.8.0.tar.gz", hash = "sha256:d1d5af7bd9c784a273c4e1eccfaa8feaca5e0481a08717b5313fa231da22a903"}, +] +flax = [ + {file = "flax-0.6.0-py3-none-any.whl", hash = "sha256:98197d17a82bb239c13d9317508cc4cee8daf8728e8db8fcfbe80277a7dfd9e6"}, + {file = "flax-0.6.0.tar.gz", hash = "sha256:6d9bc9d5e86291610e646c945a314912a8769508aa95ed751790f983f6e29c3d"}, +] +fonttools = [ + {file = "fonttools-4.37.3-py3-none-any.whl", hash = "sha256:a5bc5f5d48faa4085310b8ebd4c5d33bf27c6636c5f10a7de792510af2745a81"}, + {file = "fonttools-4.37.3.zip", hash = "sha256:f32ef6ec966cf0e7d2aa88601fed2e3a8f2851c26b5db2c80ccc8f82bee4eedc"}, +] frozendict = [ {file = "frozendict-2.3.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4a3b32d47282ae0098b9239a6d53ec539da720258bd762d62191b46f2f87c5fc"}, {file = "frozendict-2.3.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84c9887179a245a66a50f52afa08d4d92ae0f269839fab82285c70a0fa0dd782"}, @@ -468,10 +807,191 @@ importlib-metadata = [ {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, ] +importlib-resources = [ + {file = "importlib_resources-5.9.0-py3-none-any.whl", hash = "sha256:f78a8df21a79bcc30cfd400bdc38f314333de7c0fb619763f6b9dabab8268bb7"}, + {file = "importlib_resources-5.9.0.tar.gz", hash = "sha256:5481e97fb45af8dcf2f798952625591c58fe599d0735d86b10f54de086a61681"}, +] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] +jax = [ + {file = "jax-0.3.17.tar.gz", hash = "sha256:2a2794e4e0c93595a1b1d625026580c0686be93bd60d4f6906b090446692cadc"}, +] +jaxlib = [ + {file = "jaxlib-0.3.15-cp310-none-macosx_10_14_x86_64.whl", hash = "sha256:7555be95d567572579a2010ed03506e31ff87d55c21c1bc80eddcccd87b3f723"}, + {file = "jaxlib-0.3.15-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:89f5e27c59f8f5c2e5f3ad377ace2bc03e68728ff27aaf9e82e59d2a1a4c3c6d"}, + {file = "jaxlib-0.3.15-cp310-none-manylinux2014_x86_64.whl", hash = "sha256:3af8343024ec44488bb542e62c7a61488df558e6da4bf072ebbdac319362a03f"}, + {file = "jaxlib-0.3.15-cp37-none-macosx_10_14_x86_64.whl", hash = "sha256:8f999a69fda69976bdcfa45abcc99614160035ee878a544e713affd45cd5b4d1"}, + {file = "jaxlib-0.3.15-cp37-none-manylinux2014_x86_64.whl", hash = "sha256:36570c1699b5bb4038f1f708d976afefff9fb745cc131c0c8c9340a072b38244"}, + {file = "jaxlib-0.3.15-cp38-none-macosx_10_14_x86_64.whl", hash = "sha256:19038f91f49f44df1a9d2c00e11f7f61e922e4651c10cd928ac3dcb745234dbf"}, + {file = "jaxlib-0.3.15-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:30385dafe8299c60da1e1f6f3e0d095cfd2aeefc398c67501a772c8238fc57cd"}, + {file = "jaxlib-0.3.15-cp38-none-manylinux2014_x86_64.whl", hash = "sha256:a97e80d533d1c66fe0c963e6a25cf85e00732710b6e17af568b1f4f026b52919"}, + {file = "jaxlib-0.3.15-cp39-none-macosx_10_14_x86_64.whl", hash = "sha256:e8d63b156cbac46a15c2024f63761b0bc1f73df94d6a97fb641326c74aeab6ef"}, + {file = "jaxlib-0.3.15-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:c290ca60b1c08d0ac991be922146775a80943e7a970cde3fc42d6667ca0c115d"}, + {file = "jaxlib-0.3.15-cp39-none-manylinux2014_x86_64.whl", hash = "sha256:ef58c67341c87789417891f7bd832fc84e23044ef6b085fb9c8306c60ec43bda"}, +] +kiwisolver = [ + {file = "kiwisolver-1.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2f5e60fabb7343a836360c4f0919b8cd0d6dbf08ad2ca6b9cf90bf0c76a3c4f6"}, + {file = "kiwisolver-1.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:10ee06759482c78bdb864f4109886dff7b8a56529bc1609d4f1112b93fe6423c"}, + {file = "kiwisolver-1.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c79ebe8f3676a4c6630fd3f777f3cfecf9289666c84e775a67d1d358578dc2e3"}, + {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:abbe9fa13da955feb8202e215c4018f4bb57469b1b78c7a4c5c7b93001699938"}, + {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7577c1987baa3adc4b3c62c33bd1118c3ef5c8ddef36f0f2c950ae0b199e100d"}, + {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8ad8285b01b0d4695102546b342b493b3ccc6781fc28c8c6a1bb63e95d22f09"}, + {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ed58b8acf29798b036d347791141767ccf65eee7f26bde03a71c944449e53de"}, + {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a68b62a02953b9841730db7797422f983935aeefceb1679f0fc85cbfbd311c32"}, + {file = "kiwisolver-1.4.4-cp310-cp310-win32.whl", hash = "sha256:e92a513161077b53447160b9bd8f522edfbed4bd9759e4c18ab05d7ef7e49408"}, + {file = "kiwisolver-1.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:3fe20f63c9ecee44560d0e7f116b3a747a5d7203376abeea292ab3152334d004"}, + {file = "kiwisolver-1.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e0ea21f66820452a3f5d1655f8704a60d66ba1191359b96541eaf457710a5fc6"}, + {file = "kiwisolver-1.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bc9db8a3efb3e403e4ecc6cd9489ea2bac94244f80c78e27c31dcc00d2790ac2"}, + {file = "kiwisolver-1.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d5b61785a9ce44e5a4b880272baa7cf6c8f48a5180c3e81c59553ba0cb0821ca"}, + {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2dbb44c3f7e6c4d3487b31037b1bdbf424d97687c1747ce4ff2895795c9bf69"}, + {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6295ecd49304dcf3bfbfa45d9a081c96509e95f4b9d0eb7ee4ec0530c4a96514"}, + {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4bd472dbe5e136f96a4b18f295d159d7f26fd399136f5b17b08c4e5f498cd494"}, + {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bf7d9fce9bcc4752ca4a1b80aabd38f6d19009ea5cbda0e0856983cf6d0023f5"}, + {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d6601aed50c74e0ef02f4204da1816147a6d3fbdc8b3872d263338a9052c51"}, + {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:877272cf6b4b7e94c9614f9b10140e198d2186363728ed0f701c6eee1baec1da"}, + {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:db608a6757adabb32f1cfe6066e39b3706d8c3aa69bbc353a5b61edad36a5cb4"}, + {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5853eb494c71e267912275e5586fe281444eb5e722de4e131cddf9d442615626"}, + {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:f0a1dbdb5ecbef0d34eb77e56fcb3e95bbd7e50835d9782a45df81cc46949750"}, + {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:283dffbf061a4ec60391d51e6155e372a1f7a4f5b15d59c8505339454f8989e4"}, + {file = "kiwisolver-1.4.4-cp311-cp311-win32.whl", hash = "sha256:d06adcfa62a4431d404c31216f0f8ac97397d799cd53800e9d3efc2fbb3cf14e"}, + {file = "kiwisolver-1.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:e7da3fec7408813a7cebc9e4ec55afed2d0fd65c4754bc376bf03498d4e92686"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:62ac9cc684da4cf1778d07a89bf5f81b35834cb96ca523d3a7fb32509380cbf6"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41dae968a94b1ef1897cb322b39360a0812661dba7c682aa45098eb8e193dbdf"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02f79693ec433cb4b5f51694e8477ae83b3205768a6fb48ffba60549080e295b"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0611a0a2a518464c05ddd5a3a1a0e856ccc10e67079bb17f265ad19ab3c7597"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:db5283d90da4174865d520e7366801a93777201e91e79bacbac6e6927cbceede"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1041feb4cda8708ce73bb4dcb9ce1ccf49d553bf87c3954bdfa46f0c3f77252c"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-win32.whl", hash = "sha256:a553dadda40fef6bfa1456dc4be49b113aa92c2a9a9e8711e955618cd69622e3"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:03baab2d6b4a54ddbb43bba1a3a2d1627e82d205c5cf8f4c924dc49284b87166"}, + {file = "kiwisolver-1.4.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:841293b17ad704d70c578f1f0013c890e219952169ce8a24ebc063eecf775454"}, + {file = "kiwisolver-1.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f4f270de01dd3e129a72efad823da90cc4d6aafb64c410c9033aba70db9f1ff0"}, + {file = "kiwisolver-1.4.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f9f39e2f049db33a908319cf46624a569b36983c7c78318e9726a4cb8923b26c"}, + {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c97528e64cb9ebeff9701e7938653a9951922f2a38bd847787d4a8e498cc83ae"}, + {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d1573129aa0fd901076e2bfb4275a35f5b7aa60fbfb984499d661ec950320b0"}, + {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad881edc7ccb9d65b0224f4e4d05a1e85cf62d73aab798943df6d48ab0cd79a1"}, + {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b428ef021242344340460fa4c9185d0b1f66fbdbfecc6c63eff4b7c29fad429d"}, + {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2e407cb4bd5a13984a6c2c0fe1845e4e41e96f183e5e5cd4d77a857d9693494c"}, + {file = "kiwisolver-1.4.4-cp38-cp38-win32.whl", hash = "sha256:75facbe9606748f43428fc91a43edb46c7ff68889b91fa31f53b58894503a191"}, + {file = "kiwisolver-1.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:5bce61af018b0cb2055e0e72e7d65290d822d3feee430b7b8203d8a855e78766"}, + {file = "kiwisolver-1.4.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8c808594c88a025d4e322d5bb549282c93c8e1ba71b790f539567932722d7bd8"}, + {file = "kiwisolver-1.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0a71d85ecdd570ded8ac3d1c0f480842f49a40beb423bb8014539a9f32a5897"}, + {file = "kiwisolver-1.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b533558eae785e33e8c148a8d9921692a9fe5aa516efbdff8606e7d87b9d5824"}, + {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:efda5fc8cc1c61e4f639b8067d118e742b812c930f708e6667a5ce0d13499e29"}, + {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7c43e1e1206cd421cd92e6b3280d4385d41d7166b3ed577ac20444b6995a445f"}, + {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc8d3bd6c72b2dd9decf16ce70e20abcb3274ba01b4e1c96031e0c4067d1e7cd"}, + {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ea39b0ccc4f5d803e3337dd46bcce60b702be4d86fd0b3d7531ef10fd99a1ac"}, + {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:968f44fdbf6dd757d12920d63b566eeb4d5b395fd2d00d29d7ef00a00582aac9"}, + {file = "kiwisolver-1.4.4-cp39-cp39-win32.whl", hash = "sha256:da7e547706e69e45d95e116e6939488d62174e033b763ab1496b4c29b76fabea"}, + {file = "kiwisolver-1.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:ba59c92039ec0a66103b1d5fe588fa546373587a7d68f5c96f743c3396afc04b"}, + {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:91672bacaa030f92fc2f43b620d7b337fd9a5af28b0d6ed3f77afc43c4a64b5a"}, + {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:787518a6789009c159453da4d6b683f468ef7a65bbde796bcea803ccf191058d"}, + {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da152d8cdcab0e56e4f45eb08b9aea6455845ec83172092f09b0e077ece2cf7a"}, + {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ecb1fa0db7bf4cff9dac752abb19505a233c7f16684c5826d1f11ebd9472b871"}, + {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:28bc5b299f48150b5f822ce68624e445040595a4ac3d59251703779836eceff9"}, + {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:81e38381b782cc7e1e46c4e14cd997ee6040768101aefc8fa3c24a4cc58e98f8"}, + {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2a66fdfb34e05b705620dd567f5a03f239a088d5a3f321e7b6ac3239d22aa286"}, + {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872b8ca05c40d309ed13eb2e582cab0c5a05e81e987ab9c521bf05ad1d5cf5cb"}, + {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:70e7c2e7b750585569564e2e5ca9845acfaa5da56ac46df68414f29fea97be9f"}, + {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9f85003f5dfa867e86d53fac6f7e6f30c045673fa27b603c397753bebadc3008"}, + {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e307eb9bd99801f82789b44bb45e9f541961831c7311521b13a6c85afc09767"}, + {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1792d939ec70abe76f5054d3f36ed5656021dcad1322d1cc996d4e54165cef9"}, + {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6cb459eea32a4e2cf18ba5fcece2dbdf496384413bc1bae15583f19e567f3b2"}, + {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36dafec3d6d6088d34e2de6b85f9d8e2324eb734162fba59d2ba9ed7a2043d5b"}, + {file = "kiwisolver-1.4.4.tar.gz", hash = "sha256:d41997519fcba4a1e46eb4a2fe31bc12f0ff957b2b81bac28db24744f333e955"}, +] +matplotlib = [ + {file = "matplotlib-3.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a206a1b762b39398efea838f528b3a6d60cdb26fe9d58b48265787e29cd1d693"}, + {file = "matplotlib-3.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cd45a6f3e93a780185f70f05cf2a383daed13c3489233faad83e81720f7ede24"}, + {file = "matplotlib-3.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d62880e1f60e5a30a2a8484432bcb3a5056969dc97258d7326ad465feb7ae069"}, + {file = "matplotlib-3.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ab29589cef03bc88acfa3a1490359000c18186fc30374d8aa77d33cc4a51a4a"}, + {file = "matplotlib-3.5.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2886cc009f40e2984c083687251821f305d811d38e3df8ded414265e4583f0c5"}, + {file = "matplotlib-3.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c995f7d9568f18b5db131ab124c64e51b6820a92d10246d4f2b3f3a66698a15b"}, + {file = "matplotlib-3.5.3-cp310-cp310-win32.whl", hash = "sha256:6bb93a0492d68461bd458eba878f52fdc8ac7bdb6c4acdfe43dba684787838c2"}, + {file = "matplotlib-3.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:2e6d184ebe291b9e8f7e78bbab7987d269c38ea3e062eace1fe7d898042ef804"}, + {file = "matplotlib-3.5.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6ea6aef5c4338e58d8d376068e28f80a24f54e69f09479d1c90b7172bad9f25b"}, + {file = "matplotlib-3.5.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:839d47b8ead7ad9669aaacdbc03f29656dc21f0d41a6fea2d473d856c39c8b1c"}, + {file = "matplotlib-3.5.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3b4fa56159dc3c7f9250df88f653f085068bcd32dcd38e479bba58909254af7f"}, + {file = "matplotlib-3.5.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:94ff86af56a3869a4ae26a9637a849effd7643858a1a04dd5ee50e9ab75069a7"}, + {file = "matplotlib-3.5.3-cp37-cp37m-win32.whl", hash = "sha256:35a8ad4dddebd51f94c5d24bec689ec0ec66173bf614374a1244c6241c1595e0"}, + {file = "matplotlib-3.5.3-cp37-cp37m-win_amd64.whl", hash = "sha256:43e9d3fa077bf0cc95ded13d331d2156f9973dce17c6f0c8b49ccd57af94dbd9"}, + {file = "matplotlib-3.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:22227c976ad4dc8c5a5057540421f0d8708c6560744ad2ad638d48e2984e1dbc"}, + {file = "matplotlib-3.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bf618a825deb6205f015df6dfe6167a5d9b351203b03fab82043ae1d30f16511"}, + {file = "matplotlib-3.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9befa5954cdbc085e37d974ff6053da269474177921dd61facdad8023c4aeb51"}, + {file = "matplotlib-3.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3840c280ebc87a48488a46f760ea1c0c0c83fcf7abbe2e6baf99d033fd35fd8"}, + {file = "matplotlib-3.5.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dacddf5bfcec60e3f26ec5c0ae3d0274853a258b6c3fc5ef2f06a8eb23e042be"}, + {file = "matplotlib-3.5.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b428076a55fb1c084c76cb93e68006f27d247169f056412607c5c88828d08f88"}, + {file = "matplotlib-3.5.3-cp38-cp38-win32.whl", hash = "sha256:874df7505ba820e0400e7091199decf3ff1fde0583652120c50cd60d5820ca9a"}, + {file = "matplotlib-3.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:b28de401d928890187c589036857a270a032961411934bdac4cf12dde3d43094"}, + {file = "matplotlib-3.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3211ba82b9f1518d346f6309df137b50c3dc4421b4ed4815d1d7eadc617f45a1"}, + {file = "matplotlib-3.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6fe807e8a22620b4cd95cfbc795ba310dc80151d43b037257250faf0bfcd82bc"}, + {file = "matplotlib-3.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5c096363b206a3caf43773abebdbb5a23ea13faef71d701b21a9c27fdcef72f4"}, + {file = "matplotlib-3.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bcdfcb0f976e1bac6721d7d457c17be23cf7501f977b6a38f9d38a3762841f7"}, + {file = "matplotlib-3.5.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1e64ac9be9da6bfff0a732e62116484b93b02a0b4d4b19934fb4f8e7ad26ad6a"}, + {file = "matplotlib-3.5.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:73dd93dc35c85dece610cca8358003bf0760d7986f70b223e2306b4ea6d1406b"}, + {file = "matplotlib-3.5.3-cp39-cp39-win32.whl", hash = "sha256:879c7e5fce4939c6aa04581dfe08d57eb6102a71f2e202e3314d5fbc072fd5a0"}, + {file = "matplotlib-3.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:ab8d26f07fe64f6f6736d635cce7bfd7f625320490ed5bfc347f2cdb4fae0e56"}, + {file = "matplotlib-3.5.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:99482b83ebf4eb6d5fc6813d7aacdefdd480f0d9c0b52dcf9f1cc3b2c4b3361a"}, + {file = "matplotlib-3.5.3-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f814504e459c68118bf2246a530ed953ebd18213dc20e3da524174d84ed010b2"}, + {file = "matplotlib-3.5.3-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57f1b4e69f438a99bb64d7f2c340db1b096b41ebaa515cf61ea72624279220ce"}, + {file = "matplotlib-3.5.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:d2484b350bf3d32cae43f85dcfc89b3ed7bd2bcd781ef351f93eb6fb2cc483f9"}, + {file = "matplotlib-3.5.3.tar.gz", hash = "sha256:339cac48b80ddbc8bfd05daae0a3a73414651a8596904c2a881cfd1edb65f26c"}, +] +msgpack = [ + {file = "msgpack-1.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4ab251d229d10498e9a2f3b1e68ef64cb393394ec477e3370c457f9430ce9250"}, + {file = "msgpack-1.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:112b0f93202d7c0fef0b7810d465fde23c746a2d482e1e2de2aafd2ce1492c88"}, + {file = "msgpack-1.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:002b5c72b6cd9b4bafd790f364b8480e859b4712e91f43014fe01e4f957b8467"}, + {file = "msgpack-1.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35bc0faa494b0f1d851fd29129b2575b2e26d41d177caacd4206d81502d4c6a6"}, + {file = "msgpack-1.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4733359808c56d5d7756628736061c432ded018e7a1dff2d35a02439043321aa"}, + {file = "msgpack-1.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb514ad14edf07a1dbe63761fd30f89ae79b42625731e1ccf5e1f1092950eaa6"}, + {file = "msgpack-1.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c23080fdeec4716aede32b4e0ef7e213c7b1093eede9ee010949f2a418ced6ba"}, + {file = "msgpack-1.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:49565b0e3d7896d9ea71d9095df15b7f75a035c49be733051c34762ca95bbf7e"}, + {file = "msgpack-1.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:aca0f1644d6b5a73eb3e74d4d64d5d8c6c3d577e753a04c9e9c87d07692c58db"}, + {file = "msgpack-1.0.4-cp310-cp310-win32.whl", hash = "sha256:0dfe3947db5fb9ce52aaea6ca28112a170db9eae75adf9339a1aec434dc954ef"}, + {file = "msgpack-1.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dea20515f660aa6b7e964433b1808d098dcfcabbebeaaad240d11f909298075"}, + {file = "msgpack-1.0.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e83f80a7fec1a62cf4e6c9a660e39c7f878f603737a0cdac8c13131d11d97f52"}, + {file = "msgpack-1.0.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c11a48cf5e59026ad7cb0dc29e29a01b5a66a3e333dc11c04f7e991fc5510a9"}, + {file = "msgpack-1.0.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1276e8f34e139aeff1c77a3cefb295598b504ac5314d32c8c3d54d24fadb94c9"}, + {file = "msgpack-1.0.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c9566f2c39ccced0a38d37c26cc3570983b97833c365a6044edef3574a00c08"}, + {file = "msgpack-1.0.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:fcb8a47f43acc113e24e910399376f7277cf8508b27e5b88499f053de6b115a8"}, + {file = "msgpack-1.0.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:76ee788122de3a68a02ed6f3a16bbcd97bc7c2e39bd4d94be2f1821e7c4a64e6"}, + {file = "msgpack-1.0.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:0a68d3ac0104e2d3510de90a1091720157c319ceeb90d74f7b5295a6bee51bae"}, + {file = "msgpack-1.0.4-cp36-cp36m-win32.whl", hash = "sha256:85f279d88d8e833ec015650fd15ae5eddce0791e1e8a59165318f371158efec6"}, + {file = "msgpack-1.0.4-cp36-cp36m-win_amd64.whl", hash = "sha256:c1683841cd4fa45ac427c18854c3ec3cd9b681694caf5bff04edb9387602d661"}, + {file = "msgpack-1.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a75dfb03f8b06f4ab093dafe3ddcc2d633259e6c3f74bb1b01996f5d8aa5868c"}, + {file = "msgpack-1.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9667bdfdf523c40d2511f0e98a6c9d3603be6b371ae9a238b7ef2dc4e7a427b0"}, + {file = "msgpack-1.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11184bc7e56fd74c00ead4f9cc9a3091d62ecb96e97653add7a879a14b003227"}, + {file = "msgpack-1.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac5bd7901487c4a1dd51a8c58f2632b15d838d07ceedaa5e4c080f7190925bff"}, + {file = "msgpack-1.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1e91d641d2bfe91ba4c52039adc5bccf27c335356055825c7f88742c8bb900dd"}, + {file = "msgpack-1.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2a2df1b55a78eb5f5b7d2a4bb221cd8363913830145fad05374a80bf0877cb1e"}, + {file = "msgpack-1.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:545e3cf0cf74f3e48b470f68ed19551ae6f9722814ea969305794645da091236"}, + {file = "msgpack-1.0.4-cp37-cp37m-win32.whl", hash = "sha256:2cc5ca2712ac0003bcb625c96368fd08a0f86bbc1a5578802512d87bc592fe44"}, + {file = "msgpack-1.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:eba96145051ccec0ec86611fe9cf693ce55f2a3ce89c06ed307de0e085730ec1"}, + {file = "msgpack-1.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7760f85956c415578c17edb39eed99f9181a48375b0d4a94076d84148cf67b2d"}, + {file = "msgpack-1.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:449e57cc1ff18d3b444eb554e44613cffcccb32805d16726a5494038c3b93dab"}, + {file = "msgpack-1.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d603de2b8d2ea3f3bcb2efe286849aa7a81531abc52d8454da12f46235092bcb"}, + {file = "msgpack-1.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f5d88c99f64c456413d74a975bd605a9b0526293218a3b77220a2c15458ba9"}, + {file = "msgpack-1.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6916c78f33602ecf0509cc40379271ba0f9ab572b066bd4bdafd7434dee4bc6e"}, + {file = "msgpack-1.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:81fc7ba725464651190b196f3cd848e8553d4d510114a954681fd0b9c479d7e1"}, + {file = "msgpack-1.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5b5b962221fa2c5d3a7f8133f9abffc114fe218eb4365e40f17732ade576c8e"}, + {file = "msgpack-1.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:77ccd2af37f3db0ea59fb280fa2165bf1b096510ba9fe0cc2bf8fa92a22fdb43"}, + {file = "msgpack-1.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b17be2478b622939e39b816e0aa8242611cc8d3583d1cd8ec31b249f04623243"}, + {file = "msgpack-1.0.4-cp38-cp38-win32.whl", hash = "sha256:2bb8cdf50dd623392fa75525cce44a65a12a00c98e1e37bf0fb08ddce2ff60d2"}, + {file = "msgpack-1.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:26b8feaca40a90cbe031b03d82b2898bf560027160d3eae1423f4a67654ec5d6"}, + {file = "msgpack-1.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:462497af5fd4e0edbb1559c352ad84f6c577ffbbb708566a0abaaa84acd9f3ae"}, + {file = "msgpack-1.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2999623886c5c02deefe156e8f869c3b0aaeba14bfc50aa2486a0415178fce55"}, + {file = "msgpack-1.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f0029245c51fd9473dc1aede1160b0a29f4a912e6b1dd353fa6d317085b219da"}, + {file = "msgpack-1.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed6f7b854a823ea44cf94919ba3f727e230da29feb4a99711433f25800cf747f"}, + {file = "msgpack-1.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0df96d6eaf45ceca04b3f3b4b111b86b33785683d682c655063ef8057d61fd92"}, + {file = "msgpack-1.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a4192b1ab40f8dca3f2877b70e63799d95c62c068c84dc028b40a6cb03ccd0f"}, + {file = "msgpack-1.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0e3590f9fb9f7fbc36df366267870e77269c03172d086fa76bb4eba8b2b46624"}, + {file = "msgpack-1.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1576bd97527a93c44fa856770197dec00d223b0b9f36ef03f65bac60197cedf8"}, + {file = "msgpack-1.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:63e29d6e8c9ca22b21846234913c3466b7e4ee6e422f205a2988083de3b08cae"}, + {file = "msgpack-1.0.4-cp39-cp39-win32.whl", hash = "sha256:fb62ea4b62bfcb0b380d5680f9a4b3f9a2d166d9394e9bbd9666c0ee09a3645c"}, + {file = "msgpack-1.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:4d5834a2a48965a349da1c5a79760d94a1a0172fbb5ab6b5b33cbf8447e109ce"}, + {file = "msgpack-1.0.4.tar.gz", hash = "sha256:f5d869c18f030202eb412f08b28d2afeea553d6613aee89e200d7aca7ef01f5f"}, +] mypy = [ {file = "mypy-0.971-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f2899a3cbd394da157194f913a931edfd4be5f274a88041c9dc2d9cdcb1c315c"}, {file = "mypy-0.971-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:98e02d56ebe93981c41211c05adb630d1d26c14195d04d95e49cd97dbc046dc5"}, @@ -536,13 +1056,81 @@ numpy = [ {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, ] omegaconf = [ - {file = "omegaconf-2.2.2-py3-none-any.whl", hash = "sha256:556917181487fb66fe832d3c7b324f51b2f4c8adc373dd5091be921501b7d420"}, - {file = "omegaconf-2.2.2.tar.gz", hash = "sha256:65c85b2a84669a570c70f2df00de3cebcd9b47a8587d3c53b1aa5766bb096f77"}, + {file = "omegaconf-2.2.3-py3-none-any.whl", hash = "sha256:d6f2cbf79a992899eb76c6cb1aedfcf0fe7456a8654382edd5ee0c1b199c0657"}, + {file = "omegaconf-2.2.3.tar.gz", hash = "sha256:59ff9fba864ffbb5fb710b64e8a9ba37c68fa339a2e2bb4f1b648d6901552523"}, +] +opt-einsum = [ + {file = "opt_einsum-3.3.0-py3-none-any.whl", hash = "sha256:2455e59e3947d3c275477df7f5205b30635e266fe6dc300e3d9f9646bfcea147"}, + {file = "opt_einsum-3.3.0.tar.gz", hash = "sha256:59f6475f77bbc37dcf7cd748519c0ec60722e91e63ca114e68821c0c54a46549"}, +] +optax = [ + {file = "optax-0.1.3-py3-none-any.whl", hash = "sha256:33ac3b36bc8f6e087e112fd3a14ab054b99d1c26867aae552db80e234916e522"}, + {file = "optax-0.1.3.tar.gz", hash = "sha256:159e954405c3ba2072c2add7cec5532be7399bcafab3039acbf608b11844a879"}, ] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] +Pillow = [ + {file = "Pillow-9.2.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:a9c9bc489f8ab30906d7a85afac4b4944a572a7432e00698a7239f44a44e6efb"}, + {file = "Pillow-9.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:510cef4a3f401c246cfd8227b300828715dd055463cdca6176c2e4036df8bd4f"}, + {file = "Pillow-9.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7888310f6214f19ab2b6df90f3f06afa3df7ef7355fc025e78a3044737fab1f5"}, + {file = "Pillow-9.2.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:831e648102c82f152e14c1a0938689dbb22480c548c8d4b8b248b3e50967b88c"}, + {file = "Pillow-9.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1cc1d2451e8a3b4bfdb9caf745b58e6c7a77d2e469159b0d527a4554d73694d1"}, + {file = "Pillow-9.2.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:136659638f61a251e8ed3b331fc6ccd124590eeff539de57c5f80ef3a9594e58"}, + {file = "Pillow-9.2.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:6e8c66f70fb539301e064f6478d7453e820d8a2c631da948a23384865cd95544"}, + {file = "Pillow-9.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:37ff6b522a26d0538b753f0b4e8e164fdada12db6c6f00f62145d732d8a3152e"}, + {file = "Pillow-9.2.0-cp310-cp310-win32.whl", hash = "sha256:c79698d4cd9318d9481d89a77e2d3fcaeff5486be641e60a4b49f3d2ecca4e28"}, + {file = "Pillow-9.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:254164c57bab4b459f14c64e93df11eff5ded575192c294a0c49270f22c5d93d"}, + {file = "Pillow-9.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:adabc0bce035467fb537ef3e5e74f2847c8af217ee0be0455d4fec8adc0462fc"}, + {file = "Pillow-9.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:336b9036127eab855beec9662ac3ea13a4544a523ae273cbf108b228ecac8437"}, + {file = "Pillow-9.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50dff9cc21826d2977ef2d2a205504034e3a4563ca6f5db739b0d1026658e004"}, + {file = "Pillow-9.2.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb6259196a589123d755380b65127ddc60f4c64b21fc3bb46ce3a6ea663659b0"}, + {file = "Pillow-9.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b0554af24df2bf96618dac71ddada02420f946be943b181108cac55a7a2dcd4"}, + {file = "Pillow-9.2.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:15928f824870535c85dbf949c09d6ae7d3d6ac2d6efec80f3227f73eefba741c"}, + {file = "Pillow-9.2.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:bdd0de2d64688ecae88dd8935012c4a72681e5df632af903a1dca8c5e7aa871a"}, + {file = "Pillow-9.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5b87da55a08acb586bad5c3aa3b86505f559b84f39035b233d5bf844b0834b1"}, + {file = "Pillow-9.2.0-cp311-cp311-win32.whl", hash = "sha256:b6d5e92df2b77665e07ddb2e4dbd6d644b78e4c0d2e9272a852627cdba0d75cf"}, + {file = "Pillow-9.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6bf088c1ce160f50ea40764f825ec9b72ed9da25346216b91361eef8ad1b8f8c"}, + {file = "Pillow-9.2.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:2c58b24e3a63efd22554c676d81b0e57f80e0a7d3a5874a7e14ce90ec40d3069"}, + {file = "Pillow-9.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eef7592281f7c174d3d6cbfbb7ee5984a671fcd77e3fc78e973d492e9bf0eb3f"}, + {file = "Pillow-9.2.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dcd7b9c7139dc8258d164b55696ecd16c04607f1cc33ba7af86613881ffe4ac8"}, + {file = "Pillow-9.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a138441e95562b3c078746a22f8fca8ff1c22c014f856278bdbdd89ca36cff1b"}, + {file = "Pillow-9.2.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:93689632949aff41199090eff5474f3990b6823404e45d66a5d44304e9cdc467"}, + {file = "Pillow-9.2.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:f3fac744f9b540148fa7715a435d2283b71f68bfb6d4aae24482a890aed18b59"}, + {file = "Pillow-9.2.0-cp37-cp37m-win32.whl", hash = "sha256:fa768eff5f9f958270b081bb33581b4b569faabf8774726b283edb06617101dc"}, + {file = "Pillow-9.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:69bd1a15d7ba3694631e00df8de65a8cb031911ca11f44929c97fe05eb9b6c1d"}, + {file = "Pillow-9.2.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:030e3460861488e249731c3e7ab59b07c7853838ff3b8e16aac9561bb345da14"}, + {file = "Pillow-9.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:74a04183e6e64930b667d321524e3c5361094bb4af9083db5c301db64cd341f3"}, + {file = "Pillow-9.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d33a11f601213dcd5718109c09a52c2a1c893e7461f0be2d6febc2879ec2402"}, + {file = "Pillow-9.2.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fd6f5e3c0e4697fa7eb45b6e93996299f3feee73a3175fa451f49a74d092b9f"}, + {file = "Pillow-9.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a647c0d4478b995c5e54615a2e5360ccedd2f85e70ab57fbe817ca613d5e63b8"}, + {file = "Pillow-9.2.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:4134d3f1ba5f15027ff5c04296f13328fecd46921424084516bdb1b2548e66ff"}, + {file = "Pillow-9.2.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:bc431b065722a5ad1dfb4df354fb9333b7a582a5ee39a90e6ffff688d72f27a1"}, + {file = "Pillow-9.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1536ad017a9f789430fb6b8be8bf99d2f214c76502becc196c6f2d9a75b01b76"}, + {file = "Pillow-9.2.0-cp38-cp38-win32.whl", hash = "sha256:2ad0d4df0f5ef2247e27fc790d5c9b5a0af8ade9ba340db4a73bb1a4a3e5fb4f"}, + {file = "Pillow-9.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:ec52c351b35ca269cb1f8069d610fc45c5bd38c3e91f9ab4cbbf0aebc136d9c8"}, + {file = "Pillow-9.2.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:0ed2c4ef2451de908c90436d6e8092e13a43992f1860275b4d8082667fbb2ffc"}, + {file = "Pillow-9.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ad2f835e0ad81d1689f1b7e3fbac7b01bb8777d5a985c8962bedee0cc6d43da"}, + {file = "Pillow-9.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea98f633d45f7e815db648fd7ff0f19e328302ac36427343e4432c84432e7ff4"}, + {file = "Pillow-9.2.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7761afe0126d046974a01e030ae7529ed0ca6a196de3ec6937c11df0df1bc91c"}, + {file = "Pillow-9.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a54614049a18a2d6fe156e68e188da02a046a4a93cf24f373bffd977e943421"}, + {file = "Pillow-9.2.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:5aed7dde98403cd91d86a1115c78d8145c83078e864c1de1064f52e6feb61b20"}, + {file = "Pillow-9.2.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:13b725463f32df1bfeacbf3dd197fb358ae8ebcd8c5548faa75126ea425ccb60"}, + {file = "Pillow-9.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:808add66ea764ed97d44dda1ac4f2cfec4c1867d9efb16a33d158be79f32b8a4"}, + {file = "Pillow-9.2.0-cp39-cp39-win32.whl", hash = "sha256:337a74fd2f291c607d220c793a8135273c4c2ab001b03e601c36766005f36885"}, + {file = "Pillow-9.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:fac2d65901fb0fdf20363fbd345c01958a742f2dc62a8dd4495af66e3ff502a4"}, + {file = "Pillow-9.2.0-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:ad2277b185ebce47a63f4dc6302e30f05762b688f8dc3de55dbae4651872cdf3"}, + {file = "Pillow-9.2.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c7b502bc34f6e32ba022b4a209638f9e097d7a9098104ae420eb8186217ebbb"}, + {file = "Pillow-9.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d1f14f5f691f55e1b47f824ca4fdcb4b19b4323fe43cc7bb105988cad7496be"}, + {file = "Pillow-9.2.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:dfe4c1fedfde4e2fbc009d5ad420647f7730d719786388b7de0999bf32c0d9fd"}, + {file = "Pillow-9.2.0-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:f07f1f00e22b231dd3d9b9208692042e29792d6bd4f6639415d2f23158a80013"}, + {file = "Pillow-9.2.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1802f34298f5ba11d55e5bb09c31997dc0c6aed919658dfdf0198a2fe75d5490"}, + {file = "Pillow-9.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17d4cafe22f050b46d983b71c707162d63d796a1235cdf8b9d7a112e97b15bac"}, + {file = "Pillow-9.2.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:96b5e6874431df16aee0c1ba237574cb6dff1dcb173798faa6a9d8b399a05d0e"}, + {file = "Pillow-9.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:0030fdbd926fb85844b8b92e2f9449ba89607231d3dd597a21ae72dc7fe26927"}, + {file = "Pillow-9.2.0.tar.gz", hash = "sha256:75e636fd3e0fb872693f23ccb8a5ff2cd578801251f3a4f6854c6a5d437d3c04"}, +] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, @@ -560,18 +1148,22 @@ pyparsing = [ {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] pyright = [ - {file = "pyright-1.1.266-py3-none-any.whl", hash = "sha256:246b2fb33c9483ac6e684e8e7619e3ecf0fee20180ec22549a4f3cdf47dbd0de"}, - {file = "pyright-1.1.266.tar.gz", hash = "sha256:1154aeaaa0b62f5161af09342bf0a586468d6789377dfd8710ba8d85c22eee94"}, + {file = "pyright-1.1.271-py3-none-any.whl", hash = "sha256:1d8da0eca3eade0500cf19da90aaf029263b6e339c6210c158c863758fe21de1"}, + {file = "pyright-1.1.271.tar.gz", hash = "sha256:67b18ae5e6a5e291120bda19880d0e78c9c05e3cb10bd4439921bacc0bfd780f"}, ] pytest = [ - {file = "pytest-7.1.2-py3-none-any.whl", hash = "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c"}, - {file = "pytest-7.1.2.tar.gz", hash = "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"}, + {file = "pytest-7.1.3-py3-none-any.whl", hash = "sha256:1377bda3466d70b55e3f5cecfa55bb7cfcf219c7964629b967c37cf0bda818b7"}, + {file = "pytest-7.1.3.tar.gz", hash = "sha256:4f365fec2dff9c1162f834d9f18af1ba13062db0c708bf7b946f8a5c76180c39"}, ] pytest-cov = [ {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, ] -pyyaml = [ +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] +PyYAML = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, @@ -579,6 +1171,13 @@ pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, @@ -607,17 +1206,50 @@ pyyaml = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] rich = [ - {file = "rich-12.5.1-py3-none-any.whl", hash = "sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb"}, - {file = "rich-12.5.1.tar.gz", hash = "sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca"}, + {file = "rich-11.2.0-py3-none-any.whl", hash = "sha256:d5f49ad91fb343efcae45a2b2df04a9755e863e50413623ab8c9e74f05aee52b"}, + {file = "rich-11.2.0.tar.gz", hash = "sha256:1a6266a5738115017bb64a66c59c717e7aa047b3ae49a011ede4abdeffc6536e"}, +] +scipy = [ + {file = "scipy-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a15a1f3fc0abff33e792d6049161b7795909b40b97c6cc2934ed54384017ab76"}, + {file = "scipy-1.6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:e79570979ccdc3d165456dd62041d9556fb9733b86b4b6d818af7a0afc15f092"}, + {file = "scipy-1.6.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a423533c55fec61456dedee7b6ee7dce0bb6bfa395424ea374d25afa262be261"}, + {file = "scipy-1.6.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:33d6b7df40d197bdd3049d64e8e680227151673465e5d85723b3b8f6b15a6ced"}, + {file = "scipy-1.6.1-cp37-cp37m-win32.whl", hash = "sha256:6725e3fbb47da428794f243864f2297462e9ee448297c93ed1dcbc44335feb78"}, + {file = "scipy-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:5fa9c6530b1661f1370bcd332a1e62ca7881785cc0f80c0d559b636567fab63c"}, + {file = "scipy-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bd50daf727f7c195e26f27467c85ce653d41df4358a25b32434a50d8870fc519"}, + {file = "scipy-1.6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f46dd15335e8a320b0fb4685f58b7471702234cba8bb3442b69a3e1dc329c345"}, + {file = "scipy-1.6.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0e5b0ccf63155d90da576edd2768b66fb276446c371b73841e3503be1d63fb5d"}, + {file = "scipy-1.6.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2481efbb3740977e3c831edfd0bd9867be26387cacf24eb5e366a6a374d3d00d"}, + {file = "scipy-1.6.1-cp38-cp38-win32.whl", hash = "sha256:68cb4c424112cd4be886b4d979c5497fba190714085f46b8ae67a5e4416c32b4"}, + {file = "scipy-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:5f331eeed0297232d2e6eea51b54e8278ed8bb10b099f69c44e2558c090d06bf"}, + {file = "scipy-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0c8a51d33556bf70367452d4d601d1742c0e806cd0194785914daf19775f0e67"}, + {file = "scipy-1.6.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:83bf7c16245c15bc58ee76c5418e46ea1811edcc2e2b03041b804e46084ab627"}, + {file = "scipy-1.6.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:794e768cc5f779736593046c9714e0f3a5940bc6dcc1dba885ad64cbfb28e9f0"}, + {file = "scipy-1.6.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5da5471aed911fe7e52b86bf9ea32fb55ae93e2f0fac66c32e58897cfb02fa07"}, + {file = "scipy-1.6.1-cp39-cp39-win32.whl", hash = "sha256:8e403a337749ed40af60e537cc4d4c03febddcc56cd26e774c9b1b600a70d3e4"}, + {file = "scipy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a5193a098ae9f29af283dcf0041f762601faf2e595c0db1da929875b7570353f"}, + {file = "scipy-1.6.1.tar.gz", hash = "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11"}, ] setuptools = [ {file = "setuptools-65.3.0-py3-none-any.whl", hash = "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82"}, {file = "setuptools-65.3.0.tar.gz", hash = "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57"}, ] +setuptools-scm = [ + {file = "setuptools_scm-6.4.2-py3-none-any.whl", hash = "sha256:acea13255093849de7ccb11af9e1fb8bde7067783450cee9ef7a93139bddf6d4"}, + {file = "setuptools_scm-6.4.2.tar.gz", hash = "sha256:6833ac65c6ed9711a4d5d2266f8024cfa07c533a0e55f4c12f6eff280a5a9e30"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +toolz = [ + {file = "toolz-0.12.0-py3-none-any.whl", hash = "sha256:2059bd4148deb1884bb0eb770a3cde70e7f954cfbbdc2285f1f2de01fd21eb6f"}, + {file = "toolz-0.12.0.tar.gz", hash = "sha256:88c570861c440ee3f2f6037c4654613228ff40c93a6c25e0eba70d17282c6194"}, +] torch = [ {file = "torch-1.12.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:9c038662db894a23e49e385df13d47b2a777ffd56d9bcd5b832593fab0a7e286"}, {file = "torch-1.12.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:4e1b9c14cf13fd2ab8d769529050629a0e68a6fc5cb8e84b4a3cc1dd8c4fe541"}, diff --git a/pyproject.toml b/pyproject.toml index 80a484af9..65f5aa061 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,8 +17,7 @@ PyYAML = "^6.0" "backports.cached-property" = { version = "^1.0.2", python = "~3.7" } colorama = {version = "^0.4.0", platform = "win32"} frozendict = "^2.3.4" -# shtab = "^1.5.5" -rich = "^12.5.1" +rich = ">=11.1.0" [tool.poetry.dev-dependencies] pytest = "^7.1.2" @@ -31,6 +30,9 @@ pyright = "^1.1.264" coverage = {extras = ["toml"], version = "^6.4.2"} numpy = ">=1.20.0" +[tool.poetry.group.dev.dependencies] +flax = "^0.6.0" + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/tests/test_flax.py b/tests/test_flax.py new file mode 100644 index 000000000..fa2cb027f --- /dev/null +++ b/tests/test_flax.py @@ -0,0 +1,60 @@ +"""Tests initializing flax modules directly via dcargs.""" +import jax +import pytest +from flax import linen as nn +from jax import numpy as jnp + +import dcargs + + +class Classifier(nn.Module): + layers: int + units: int + output_dim: int + + @nn.compact + def __call__(self, x: jnp.ndarray) -> jnp.ndarray: # type: ignore + for i in range(self.layers - 1): + x = nn.Dense( + self.units, + kernel_init=nn.initializers.kaiming_normal(), + )(x) + x = nn.relu(x) + + x = nn.Dense( + self.output_dim, + kernel_init=nn.initializers.xavier_normal(), + )(x) + x = nn.sigmoid(x) + return x + + +def test_ok(): + network = dcargs.cli( + Classifier, + args=[ + "--layers", + "3", + "--units", + "8", + "--output-dim", + "3", + ], + ) + + x = jnp.zeros((10, 4)) + params = network.init(jax.random.PRNGKey(0), x) + assert network.apply(params, x).shape == (10, 3) + + +def test_missing(): + with pytest.raises(SystemExit): + dcargs.cli( + Classifier, + args=[ + "--layers", + "3", + "--units", + "8", + ], + ) From 9c133bf7ad0f784f815b07dac17a3b65763de203 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 20 Sep 2022 22:37:51 -0700 Subject: [PATCH 150/491] Ignore flax tests for Python 3.10 --- tests/conftest.py | 3 +++ tests/{test_flax.py => test_flax_ignore_py310.py} | 0 2 files changed, 3 insertions(+) rename tests/{test_flax.py => test_flax_ignore_py310.py} (100%) diff --git a/tests/conftest.py b/tests/conftest.py index 0e7694219..58c8975e4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,5 +4,8 @@ if sys.version_info.major == 3 and sys.version_info.minor == 7: collect_ignore_glob.append("*_ignore_py37.py") +if sys.version_info.major == 3 and sys.version_info.minor == 10: + collect_ignore_glob.append("*_ignore_py310.py") + if not (sys.version_info.major == 3 and sys.version_info.minor == 10): collect_ignore_glob.append("*_only_py310.py") diff --git a/tests/test_flax.py b/tests/test_flax_ignore_py310.py similarity index 100% rename from tests/test_flax.py rename to tests/test_flax_ignore_py310.py From 3df0e543c1d05b45d9a28675a4c676c507b5c2bd Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 20 Sep 2022 23:06:02 -0700 Subject: [PATCH 151/491] Smarter minimum column count, help positioning --- dcargs/_argparse_formatter.py | 39 ++++++++++++++++++++--------------- dcargs/_cli.py | 4 +--- dcargs/_parsers.py | 6 ++---- pyproject.toml | 2 +- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index fa3f64878..5073c58b6 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -1,9 +1,8 @@ import argparse import contextlib import dataclasses -import functools import shutil -from typing import Any, ContextManager, Dict, Generator, List, Optional +from typing import Any, ContextManager, Generator, List, Optional from rich.columns import Columns from rich.console import Console, Group, RenderableType @@ -114,22 +113,12 @@ def str_from_rich( return out.get().rstrip("\n") -def make_formatter_class(field_count: int) -> Any: - return functools.partial(_ArgparseHelpFormatter, field_count=field_count) - - -class _ArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): - def __init__(self, prog, *, field_count: int): +class DcargsArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): + def __init__(self, prog: str): indent_increment = 4 width = shutil.get_terminal_size().columns - 2 - - # Try to make helptext more concise when we have a lot of fields! - if field_count > 64: # pragma: no cover - # When there are more fields, make helptext more compact. - max_help_position = 8 - else: - max_help_position = 36 # Usual is 24. - + max_help_position = 24 + self._fixed_help_position = False super().__init__(prog, indent_increment, max_help_position, width) def _format_args(self, action, default_metavar): @@ -172,6 +161,18 @@ def _split_lines(self, text, width): def _fill_text(self, text, width, indent): return "".join(indent + line for line in text.splitlines(keepends=True)) + def format_help(self): + # Try with and without a fixed help position, then return the shorter help + # message. + # For dense multi-column layouts, the fixed help position is often shorter. + # For wider layouts, using the default help position settings can be more + # efficient. + self._fixed_help_position = False + help1 = super().format_help() + self._fixed_help_position = True + help2 = super().format_help() + return help1 if help1.count("\n") < help2.count("\n") else help2 + class _Section(object): def __init__(self, formatter, parent, heading=None): self.formatter = formatter @@ -222,6 +223,7 @@ def _dcargs_format_root(self): min( sum(column_parts_lines) // height_breakpoint + 1, self.formatter._width // min_column_width, + len(column_parts), ), ) if column_count > 1: @@ -259,6 +261,8 @@ def _format_action(self, action: argparse.Action): help_position = min( self.formatter._action_max_length + 4, self.formatter._max_help_position ) + if self.formatter._fixed_help_position: + help_position = 4 indent = self.formatter._current_indent item_parts: List[RenderableType] = [] @@ -273,6 +277,7 @@ def _format_action(self, action: argparse.Action): if ( action.help and len(_strings.strip_ansi_sequences(invocation)) < help_position - 1 + and not self.formatter._fixed_help_position ): table = Table(show_header=False, box=None, padding=0) table.add_column(width=help_position - indent) @@ -329,7 +334,7 @@ def _dcargs_format_nonroot(self): item_content = func(*args) if ( getattr(func, "__func__", None) - is _ArgparseHelpFormatter._format_action + is DcargsArgparseHelpFormatter._format_action ): (action,) = args assert isinstance(action, argparse.Action) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index c0a66d6aa..69a31e187 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -189,9 +189,7 @@ def fix_arg(arg: str) -> str: with _argparse_formatter.ansi_context(): parser = argparse.ArgumentParser( prog=prog, - formatter_class=_argparse_formatter.make_formatter_class( - len(parser_definition.args) - ), + formatter_class=_argparse_formatter.DcargsArgparseHelpFormatter, ) parser_definition.apply(parser) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 204d1cec3..303031f37 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -434,7 +434,7 @@ def apply( if self.can_be_none: subparser = argparse_subparsers.add_parser( name=_strings.subparser_name_from_type(self.prefix, None), - formatter_class=_argparse_formatter.make_formatter_class(0), + formatter_class=_argparse_formatter.DcargsArgparseHelpFormatter, help="", ) subparser_tree_nodes.append(subparser) @@ -442,9 +442,7 @@ def apply( for name, subparser_def in self.parser_from_name.items(): subparser = argparse_subparsers.add_parser( name, - formatter_class=_argparse_formatter.make_formatter_class( - len(subparser_def.args) - ), + formatter_class=_argparse_formatter.DcargsArgparseHelpFormatter, help=subparser_def.description, ) subparser_def.apply(subparser) diff --git a/pyproject.toml b/pyproject.toml index 65f5aa061..1489a6378 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.7" +version = "0.3.8" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From 11930b5f65bec5bbc9ee0ceb23fcee1d3cdf1f56 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 21 Sep 2022 01:31:06 -0700 Subject: [PATCH 152/491] Minor docs fixes --- docs/source/examples/04_flags.rst | 2 +- docs/source/examples/05_hierarchical_configs.rst | 2 +- examples/04_flags.py | 2 +- examples/05_hierarchical_configs.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/examples/04_flags.rst b/docs/source/examples/04_flags.rst index a77411878..30c217882 100644 --- a/docs/source/examples/04_flags.rst +++ b/docs/source/examples/04_flags.rst @@ -8,7 +8,7 @@ Booleans can either be expected to be explicitly passed in, or, if given a default value, automatically converted to flags. -To turn off conversion, see :func:`dcargs.conf.FlagConversionOff`. +To turn off conversion, see :class:`dcargs.conf.FlagConversionOff`. diff --git a/docs/source/examples/05_hierarchical_configs.rst b/docs/source/examples/05_hierarchical_configs.rst index 253ac14bd..f05df0c19 100644 --- a/docs/source/examples/05_hierarchical_configs.rst +++ b/docs/source/examples/05_hierarchical_configs.rst @@ -41,7 +41,7 @@ objects. This helps with modularity and grouping in larger projects. @dataclasses.dataclass class ExperimentConfig: # Various configurable options for our optimizer. - optimizer_config: OptimizerConfig + optimizer: OptimizerConfig # Batch size. batch_size: int = 32 diff --git a/examples/04_flags.py b/examples/04_flags.py index 635acea2e..3df1d0c7e 100644 --- a/examples/04_flags.py +++ b/examples/04_flags.py @@ -1,7 +1,7 @@ """Booleans can either be expected to be explicitly passed in, or, if given a default value, automatically converted to flags. -To turn off conversion, see :func:`dcargs.conf.FlagConversionOff`. +To turn off conversion, see :class:`dcargs.conf.FlagConversionOff`. Usage: `python ./04_flags.py --help` diff --git a/examples/05_hierarchical_configs.py b/examples/05_hierarchical_configs.py index 4f696c366..c0a1be98c 100644 --- a/examples/05_hierarchical_configs.py +++ b/examples/05_hierarchical_configs.py @@ -34,7 +34,7 @@ class OptimizerConfig: @dataclasses.dataclass class ExperimentConfig: # Various configurable options for our optimizer. - optimizer_config: OptimizerConfig + optimizer: OptimizerConfig # Batch size. batch_size: int = 32 From 0f54c13a1b310d65acddefb0ce4d0fa51260d449 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 22 Sep 2022 17:44:19 -0700 Subject: [PATCH 153/491] Tab complete for paths in bash --- dcargs/_arguments.py | 21 +++++++++++++++++++-- pyproject.toml | 2 +- tests/test_dcargs.py | 19 +++++++++++++++---- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 1842c304f..b69d93e08 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -8,6 +8,7 @@ import functools import itertools import shlex +from pathlib import Path from typing import ( Any, Dict, @@ -21,7 +22,9 @@ Union, ) -from . import _fields, _instantiators, _resolver, _strings +from . import _fields, _instantiators, _resolver +from . import _shtab as shtab +from . import _strings from .conf import _markers try: @@ -77,7 +80,21 @@ def add_argument( kwargs["choices"] = _PatchedList(kwargs["choices"]) # Note that the name must be passed in as a position argument. - parser.add_argument(name_or_flag, **kwargs) + arg = parser.add_argument(name_or_flag, **kwargs) + + # Do our best to tab complete paths. + # There will be false positives here, but if choices is unset they should be + # harmless. + if "choices" not in kwargs: + if "dir" in self.field.name: + arg.complete = shtab.DIRECTORY # type: ignore + elif ( + # Catch types like Path, List[Path], Tuple[Path, ...] etc. + "Path" in str(self.field.typ) + or "path" in self.field.name + or "file" in self.field.name + ): + arg.complete = shtab.FILE # type: ignore @cached_property def lowered(self) -> LoweredArgumentDefinition: diff --git a/pyproject.toml b/pyproject.toml index 1489a6378..55c3c5329 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.8" +version = "0.3.9" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index ede75c57b..5cdf0adc8 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -99,7 +99,7 @@ class InitFalseDataclass: i: int s: str f: float - p: pathlib.Path + dir: pathlib.Path ignored: str = dataclasses.field(default="hello", init=False) assert dcargs.cli( @@ -111,15 +111,26 @@ class InitFalseDataclass: "5", "--f", "5", - "--p", + "--dir", "~", ], - ) == InitFalseDataclass(i=5, s="5", f=5.0, p=pathlib.Path("~")) + ) == InitFalseDataclass(i=5, s="5", f=5.0, dir=pathlib.Path("~")) with pytest.raises(SystemExit): dcargs.cli( InitFalseDataclass, - args=["--i", "5", "--s", "5", "--f", "5", "--p", "~", "--ignored", "blah"], + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--dir", + "~", + "--ignored", + "blah", + ], ) From 7d26ecf8fc98e1a91a7eda57f9fa7c047f20f9bc Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 23 Sep 2022 01:56:08 -0700 Subject: [PATCH 154/491] Improve path/directory heuristics for tab complete --- dcargs/_arguments.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index b69d93e08..f332d7868 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -8,7 +8,6 @@ import functools import itertools import shlex -from pathlib import Path from typing import ( Any, Dict, @@ -86,15 +85,28 @@ def add_argument( # There will be false positives here, but if choices is unset they should be # harmless. if "choices" not in kwargs: - if "dir" in self.field.name: - arg.complete = shtab.DIRECTORY # type: ignore - elif ( + name_suggests_dir = ( + # The conditions are intended to be conservative; if a directory path is + # registered as a normal file one that's OK, the reverse on the other + # hand will be overly restrictive. + self.field.name.endswith("_dir") + or self.field.name.endswith("_directory") + or self.field.name.endswith("_folder") + ) + name_suggests_path = ( + self.field.name.endswith("_file") + or self.field.name.endswith("_path") + or self.field.name.endswith("_filename") + or name_suggests_dir + ) + complete_as_path = ( # Catch types like Path, List[Path], Tuple[Path, ...] etc. "Path" in str(self.field.typ) - or "path" in self.field.name - or "file" in self.field.name - ): - arg.complete = shtab.FILE # type: ignore + # For string types, we require more evidence. + or ("str" in str(self.field.typ) and name_suggests_path) + ) + if complete_as_path: + arg.complete = shtab.DIRECTORY if name_suggests_dir else shtab.FILE # type: ignore @cached_property def lowered(self) -> LoweredArgumentDefinition: From ea1aa1c6cae423e41cc2f3b67443eabd8743b5e0 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 24 Sep 2022 17:41:46 -0700 Subject: [PATCH 155/491] Dynamic width divider for description text --- dcargs/_argparse_formatter.py | 55 +++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index 5073c58b6..60f6237f9 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -1,6 +1,19 @@ +"""Utilities and functions for helptext formatting. We replace argparse's simple help +messages with ones that: + - Are more nicely formatted! + - Support multiple columns when many fields are defined. + - Use `rich` for formatting. + - Can be themed with an accent color. + +This is largely built by fussing around in argparse implementation details, and is by +far the hackiest part of `dcargs`. +""" + + import argparse import contextlib import dataclasses +import itertools import shutil from typing import Any, ContextManager, Generator, List, Optional @@ -8,7 +21,6 @@ from rich.console import Console, Group, RenderableType from rich.padding import Padding from rich.panel import Panel -from rich.rule import Rule from rich.style import Style from rich.table import Table from rich.text import Text @@ -167,10 +179,14 @@ def format_help(self): # For dense multi-column layouts, the fixed help position is often shorter. # For wider layouts, using the default help position settings can be more # efficient. + self._dcargs_rule = None self._fixed_help_position = False help1 = super().format_help() + + self._dcargs_rule = None self._fixed_help_position = True help2 = super().format_help() + return help1 if help1.count("\n") < help2.count("\n") else help2 class _Section(object): @@ -179,6 +195,7 @@ def __init__(self, formatter, parent, heading=None): self.parent = parent self.heading = heading self.items = [] + self.formatter._dcargs_rule = None def format_help(self): if self.parent is None: @@ -363,8 +380,42 @@ def _dcargs_format_nonroot(self): else: heading = "" + # Determine width for divider below description text. This is shared across + # all sections in a particular formatter. + lines = list( + itertools.chain( + *map( + lambda p: _strings.strip_ansi_sequences( + str_from_rich( + p, width=self.formatter._width, soft_wrap=True + ) + ) + .rstrip() + .split("\n"), + item_parts + [description_part] + if description_part is not None + else item_parts, + ) + ) + ) + max_width = max(map(len, lines)) + + if self.formatter._dcargs_rule is None: + # Note: we don't use rich.rule.Rule() because this will make all of + # the panels expand to fill the full width of the console. (this only + # impacts single-column layouts) + self.formatter._dcargs_rule = Text( + "─" * max_width, style=THEME.border, overflow="crop" + ) + elif len(self.formatter._dcargs_rule._text[0]) < max_width: + self.formatter._dcargs_rule._text = ["─" * max_width] + + # Add description text if needed. if description_part is not None: - item_parts = [description_part, Rule(style=THEME.border)] + item_parts + item_parts = [ + description_part, + self.formatter._dcargs_rule, + ] + item_parts return Panel( Group(*item_parts), From 2b35624dd49b69d391172b9b103a73c1bdb4ff8a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 25 Sep 2022 05:35:09 -0700 Subject: [PATCH 156/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 55c3c5329..ca6b53ea9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.9" +version = "0.3.10" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From 9820cccc0231e30279688ff71c206645d72fe748 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 27 Sep 2022 00:57:11 -0700 Subject: [PATCH 157/491] Helptext wrapping, coloring --- dcargs/_argparse_formatter.py | 9 +++++++-- dcargs/_docstrings.py | 14 +++++++++---- dcargs/_strings.py | 25 +++++++++++++++++++++++ pyproject.toml | 2 +- tests/test_strings.py | 37 +++++++++++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 7 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index 60f6237f9..c9a035cc8 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -55,8 +55,13 @@ def set_accent_color(accent_color: Optional[str]) -> None: THEME.helptext = Style(dim=True) THEME.helptext_required = Style(color="bright_red", bold=True) THEME.helptext_default = Style( - color=accent_color if accent_color is not None else "cyan", - dim=True, + color="cyan" + if accent_color != "cyan" + else "magenta" + # Another option: make default color match accent color. This is maybe more + # visually consistent, but harder to read. + # color=accent_color if accent_color is not None else "cyan", + # dim=accent_color is not None, ) diff --git a/dcargs/_docstrings.py b/dcargs/_docstrings.py index b236b583d..85d310c32 100644 --- a/dcargs/_docstrings.py +++ b/dcargs/_docstrings.py @@ -154,7 +154,11 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: if docstring is not None: for param_doc in docstring_parser.parse(docstring).params: if param_doc.arg_name == field_name: - return param_doc.description + return ( + _strings.postprocess_helptext(param_doc.description) + if param_doc.description is not None + else None + ) tokenization = get_class_tokenization_with_field(cls, field_name) if tokenization is None: # Currently only happens for dynamic dataclasses. @@ -177,7 +181,9 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: and first_token_content.startswith('"""') and first_token_content.endswith('"""') ): - return _strings.dedent(first_token_content[3:-3]) + return _strings.postprocess_helptext( + _strings.dedent(first_token_content[3:-3]) + ) # Check for comment on the same line as the field. final_token_on_line = tokenization.tokens_from_logical_line[ @@ -186,7 +192,7 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: if final_token_on_line.token_type == tokenize.COMMENT: comment: str = final_token_on_line.content assert comment.startswith("#") - return comment[1:].strip() + return _strings.postprocess_helptext(comment[1:].strip()) # Check for comments that come before the field. This is intentionally written to # support comments covering multiple (grouped) fields, for example: @@ -239,7 +245,7 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: current_actual_line -= 1 if len(comments) > 0: - return "\n".join(reversed(comments)) + return _strings.postprocess_helptext("\n".join(reversed(comments))) return None diff --git a/dcargs/_strings.py b/dcargs/_strings.py index af87d1b4e..6897bfd8d 100644 --- a/dcargs/_strings.py +++ b/dcargs/_strings.py @@ -118,3 +118,28 @@ def multi_metavar_from_single(single: str) -> str: return f"{single} [...]" else: return f"{single} [{single} ...]" + + +def postprocess_helptext(helptext: str) -> str: + lines = helptext.split("\n") + output_parts: List[str] = [] + for line in lines: + # Remove trailing whitespace. + line = line.rstrip() + + # Empty line. + if len(line) == 0: + prev_is_break = len(output_parts) >= 1 and output_parts[-1] == "\n" + if not prev_is_break: + output_parts.append("\n") + output_parts.append("\n") + # Empty line. + else: + if not line[0].isalpha(): + output_parts.append("\n") + prev_is_break = len(output_parts) >= 1 and output_parts[-1] == "\n" + if len(output_parts) >= 1 and not prev_is_break: + output_parts.append(" ") + output_parts.append(line) + + return "".join(output_parts).rstrip() # type: ignore diff --git a/pyproject.toml b/pyproject.toml index ca6b53ea9..992b3937d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.10" +version = "0.3.11" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] diff --git a/tests/test_strings.py b/tests/test_strings.py index 7f7fab974..398b84cd4 100644 --- a/tests/test_strings.py +++ b/tests/test_strings.py @@ -19,3 +19,40 @@ def test_make_field_name(): _strings.make_field_name(["hello_world", "---hello_world"]) == "hello-world.---hello-world" ) + + +def test_postprocess_helptext(): + assert _strings.postprocess_helptext("hello world") == "hello world" + assert _strings.postprocess_helptext("hello\nworld") == "hello world" + assert _strings.postprocess_helptext("hello \nworld") == "hello world" + assert _strings.postprocess_helptext("hello\n\nworld") == "hello\n\nworld" + assert ( + _strings.postprocess_helptext( + "a paragraph:\nSentence one.\nSentence two.\nSentence three.\n" + ) + == "a paragraph: Sentence one. Sentence two. Sentence three." + ) + assert ( + _strings.postprocess_helptext( + "a bulleted list:\n" + "- The first problem.\n" + "- The second problem.\n" + "- The third problem.\n" + ) + == "a bulleted list:\n" + "- The first problem.\n" + "- The second problem.\n" + "- The third problem." + ) + assert ( + _strings.postprocess_helptext( + "a numbered list:\n" + "1. The first problem.\n" + "2. The second problem.\n" + "3. The third problem.\n" + ) + == "a numbered list:\n" + "1. The first problem.\n" + "2. The second problem.\n" + "3. The third problem." + ) From 9999b392d7e48164aff5007c578b2ca7328d9770 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 27 Sep 2022 01:12:04 -0700 Subject: [PATCH 158/491] Escape docstrings with brackets --- dcargs/_arguments.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index f332d7868..8d06d00e5 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -21,6 +21,8 @@ Union, ) +import rich.markup + from . import _fields, _instantiators, _resolver from . import _shtab as shtab from . import _strings @@ -296,6 +298,7 @@ def as_str(x: Any) -> Tuple[str, ...]: def _rich_tag_if_enabled(x: str, tag: str): + x = rich.markup.escape(x) return x if not USE_RICH else f"[{tag}]{x}[/{tag}]" From 9f32a3cf7e1566bc1a6d8727459afd426dc24fcf Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 2 Oct 2022 12:18:24 -0700 Subject: [PATCH 159/491] Add `dcargs.extras.get_parser()`, bump version --- dcargs/_cli.py | 76 ++++++++++++++++++++++++++++++++++----- dcargs/extras/__init__.py | 4 ++- pyproject.toml | 2 +- tests/test_helptext.py | 4 +++ 4 files changed, 76 insertions(+), 10 deletions(-) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index 69a31e187..e6ade3992 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -53,11 +53,11 @@ def cli( default: Optional[OutT] = None, **deprecated_kwargs, ) -> OutT: - """Call `f(...)`, with arguments populated from an automatically generated CLI - interface. + """Call or instantiate `f`, with inputs populated from an automatically generated + CLI interface. - `f` should have type-annotated inputs, and can be a function or class. Note that if - `f` is a class, `dcargs.cli()` returns an instance. + `f` should have type-annotated inputs, and can be a function or type. Note that if + `f` is a type, `dcargs.cli()` returns an instance. The parser is generated by populating helptext from docstrings and types from annotations; a broad range of core type annotations are supported. @@ -87,11 +87,12 @@ def cli( - Optional unions over nested structures (optional subparsers). - Generics (including nested generics). - Completion scripts for interactive shells is also provided. To print a script that - can be used for tab completion, pass in `--dcargs-print-completion {bash/zsh/tcsh}`. + Completion script generation for interactive shells is also provided. To print a + script that can be used for tab completion, pass in `--dcargs-print-completion + {bash/zsh/tcsh}`. Args: - f: Callable. + f: Function or type. prog: The name of the program printed in helptext. Mirrors argument from `argparse.ArgumentParser()`. description: Description text for the parser, displayed when the --help flag is @@ -105,7 +106,63 @@ def cli( loaded from a yaml file) Returns: - The output of `f(...)`. + The output of `f(...)`, or an instance `f`. If `f` is a class, the two are + typically equivalent. + """ + return cast( + OutT, + _cli_impl( + f, + prog=prog, + description=description, + args=args, + default=default, + return_parser=False, + **deprecated_kwargs, + ), + ) + + +def get_parser( + f: Union[Type[OutT], Callable[..., OutT]], + *, + # Note that we have no `args` argument, since this is only used when + # parser.parse_args() is called. + prog: Optional[str] = None, + description: Optional[str] = None, + default: Optional[OutT] = None, +) -> argparse.ArgumentParser: + """Get the `argparse.ArgumentParser` object generated under-the-hood by + `dcargs.cli()`. Useful for tools like `sphinx-argparse`, `argcomplete`, etc. + + For tab completion, we recommend using `dcargs.cli()`'s built-in `--dcargs-print-completion` + flag.""" + return cast( + argparse.ArgumentParser, + _cli_impl( + f, + prog=prog, + description=description, + args=None, + default=default, + return_parser=True, + ), + ) + + +def _cli_impl( + f: Union[Type[OutT], Callable[..., OutT]], + *, + prog: Optional[str] = None, + description: Optional[str], + args: Optional[Sequence[str]], + default: Optional[OutT], + return_parser: bool, + **deprecated_kwargs, +) -> Union[OutT, argparse.ArgumentParser]: + """Helper for stitching the `dcargs` pipeline together. + + Converts `f` into a """ if "default_instance" in deprecated_kwargs: warnings.warn( @@ -193,6 +250,9 @@ def fix_arg(arg: str) -> str: ) parser_definition.apply(parser) + if return_parser: + return parser + if print_completion: _arguments.USE_RICH = True assert completion_shell in ("bash", "zsh", "tcsh",), ( diff --git a/dcargs/extras/__init__.py b/dcargs/extras/__init__.py index 9a332d059..fbeb0b3f0 100644 --- a/dcargs/extras/__init__.py +++ b/dcargs/extras/__init__.py @@ -3,12 +3,14 @@ Compared to the core interface, APIs here are more likely to be changed or deprecated. """ from .._argparse_formatter import set_accent_color +from .._cli import get_parser from ._base_configs import subcommand_type_from_defaults from ._serialization import from_yaml, to_yaml __all__ = [ "set_accent_color", "subcommand_type_from_defaults", - "to_yaml", + "get_parser", "from_yaml", + "to_yaml", ] diff --git a/pyproject.toml b/pyproject.toml index 992b3937d..a11586d27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.11" +version = "0.3.12" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] diff --git a/tests/test_helptext.py b/tests/test_helptext.py index db03188dd..2a8579664 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -1,3 +1,4 @@ +import argparse import contextlib import dataclasses import enum @@ -21,6 +22,9 @@ def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: with pytest.raises(SystemExit), contextlib.redirect_stdout(target): dcargs.cli(f, args=args) + # Check dcargs.extras.get_parser(). + assert isinstance(dcargs.extras.get_parser(f), argparse.ArgumentParser) + # Completion scripts; just smoke test for now. with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): dcargs.cli(f, args=["--dcargs-print-completion", "bash"]) From 4a8933313381504b55b6540827e546c00dee8d8f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 2 Oct 2022 13:23:43 -0700 Subject: [PATCH 160/491] Turn off rich formatting for `dcargs.extra.get_parser()` --- .github/workflows/publish.yml | 2 +- dcargs/_cli.py | 3 ++- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2291c4d68..b98736e41 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: '3.x' + python-version: '3.8' - name: Install dependencies run: | curl -sSL https://install.python-poetry.org | python3 - diff --git a/dcargs/_cli.py b/dcargs/_cli.py index e6ade3992..d3e68140c 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -239,8 +239,9 @@ def fix_arg(arg: str) -> str: completion_shell = None if print_completion: - _arguments.USE_RICH = False completion_shell = args[1] + if print_completion or return_parser: + _arguments.USE_RICH = False # Generate parser! with _argparse_formatter.ansi_context(): diff --git a/pyproject.toml b/pyproject.toml index a11586d27..3f62dd53b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.12" +version = "0.3.13" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From 5d73a21edc12269b853a9832e859198f16fc9e32 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 2 Oct 2022 13:28:45 -0700 Subject: [PATCH 161/491] Properly reset formatting flag --- dcargs/_cli.py | 3 +++ pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dcargs/_cli.py b/dcargs/_cli.py index d3e68140c..f5d7abf56 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -242,6 +242,8 @@ def fix_arg(arg: str) -> str: completion_shell = args[1] if print_completion or return_parser: _arguments.USE_RICH = False + else: + _arguments.USE_RICH = True # Generate parser! with _argparse_formatter.ansi_context(): @@ -252,6 +254,7 @@ def fix_arg(arg: str) -> str: parser_definition.apply(parser) if return_parser: + _arguments.USE_RICH = True return parser if print_completion: diff --git a/pyproject.toml b/pyproject.toml index 3f62dd53b..851d5ea8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.13" +version = "0.3.14" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From c1735b220e139e5e2522a4797f6ef069c8c04ebf Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 2 Oct 2022 13:39:26 -0700 Subject: [PATCH 162/491] More thorough ANSI sequence stripping --- dcargs/_argparse_formatter.py | 25 ++++++++++++++++++++++--- tests/test_helptext.py | 13 ++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index c9a035cc8..e76ad155a 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -26,7 +26,7 @@ from rich.text import Text from rich.theme import Theme -from . import _strings +from . import _arguments, _strings @dataclasses.dataclass @@ -135,7 +135,11 @@ def __init__(self, prog: str): indent_increment = 4 width = shutil.get_terminal_size().columns - 2 max_help_position = 24 - self._fixed_help_position = False + self._strip_ansi_sequences = not _arguments.USE_RICH + + # TODO: hacky. Refactor this. + self._fixed_help_position = _arguments.USE_RICH + super().__init__(prog, indent_increment, max_help_position, width) def _format_args(self, action, default_metavar): @@ -178,6 +182,16 @@ def _split_lines(self, text, width): def _fill_text(self, text, width, indent): return "".join(indent + line for line in text.splitlines(keepends=True)) + def format_usage(self): + formatter = self._get_formatter() + formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) + out = formatter.format_help() + + if self._strip_ansi_sequences: + return _strings.strip_ansi_sequences(out) + else: + return out + def format_help(self): # Try with and without a fixed help position, then return the shorter help # message. @@ -192,7 +206,12 @@ def format_help(self): self._fixed_help_position = True help2 = super().format_help() - return help1 if help1.count("\n") < help2.count("\n") else help2 + out = help1 if help1.count("\n") < help2.count("\n") else help2 + + if self._strip_ansi_sequences: + return _strings.strip_ansi_sequences(out) + else: + return out class _Section(object): def __init__(self, formatter, parent, heading=None): diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 2a8579664..54b53c2cf 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -23,7 +23,18 @@ def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: dcargs.cli(f, args=args) # Check dcargs.extras.get_parser(). - assert isinstance(dcargs.extras.get_parser(f), argparse.ArgumentParser) + parser = dcargs.extras.get_parser(f) + assert isinstance(parser, argparse.ArgumentParser) + + # Returned parser should have formatting information stripped. External tools rarely + # support ANSI sequences. + unformatted_helptext = parser.format_help() + assert ( + dcargs._strings.strip_ansi_sequences(unformatted_helptext) + == unformatted_helptext + ) + unformatted_usage = parser.format_usage() + assert dcargs._strings.strip_ansi_sequences(unformatted_usage) == unformatted_usage # Completion scripts; just smoke test for now. with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): From 0a0c631af70b78fa414a19c2a192c859f8e400ef Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 2 Oct 2022 13:50:39 -0700 Subject: [PATCH 163/491] Strip ANSI sequences even when private argparse API is used --- dcargs/_argparse_formatter.py | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index e76ad155a..d88b69771 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -135,10 +135,10 @@ def __init__(self, prog: str): indent_increment = 4 width = shutil.get_terminal_size().columns - 2 max_help_position = 24 - self._strip_ansi_sequences = not _arguments.USE_RICH + self._fixed_help_position = False # TODO: hacky. Refactor this. - self._fixed_help_position = _arguments.USE_RICH + self._strip_ansi_sequences = not _arguments.USE_RICH super().__init__(prog, indent_increment, max_help_position, width) @@ -150,12 +150,18 @@ def _format_args(self, action, default_metavar): out = get_metavar(1)[0] if isinstance(out, str): # Can result in an failed argparse assertion if we turn off soft wrapping. - return str_from_rich( - Text( - out, - style=THEME.metavar_fixed if out == "{fixed}" else THEME.metavar, - ), - soft_wrap=True, + return ( + out + if self._strip_ansi_sequences + else str_from_rich( + Text( + out, + style=THEME.metavar_fixed + if out == "{fixed}" + else THEME.metavar, + ), + soft_wrap=True, + ) ) return out @@ -182,16 +188,6 @@ def _split_lines(self, text, width): def _fill_text(self, text, width, indent): return "".join(indent + line for line in text.splitlines(keepends=True)) - def format_usage(self): - formatter = self._get_formatter() - formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) - out = formatter.format_help() - - if self._strip_ansi_sequences: - return _strings.strip_ansi_sequences(out) - else: - return out - def format_help(self): # Try with and without a fixed help position, then return the shorter help # message. From 242b1c527545b7feeba79f531eca3da395333f53 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 2 Oct 2022 18:29:36 -0700 Subject: [PATCH 164/491] Print help message by default --- dcargs/_argparse_formatter.py | 2 -- dcargs/_cli.py | 7 ++++++- dcargs/_parsers.py | 25 +++++++++++++++---------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index d88b69771..ecef25375 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -8,8 +8,6 @@ This is largely built by fussing around in argparse implementation details, and is by far the hackiest part of `dcargs`. """ - - import argparse import contextlib import dataclasses diff --git a/dcargs/_cli.py b/dcargs/_cli.py index f5d7abf56..d5d41b97d 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -206,7 +206,7 @@ def _cli_impl( dummy_wrapped = False # Map a callable to the relevant CLI arguments + subparsers. - parser_definition = _parsers.ParserSpecification.from_callable( + parser_definition = _parsers.ParserSpecification.from_callable_or_type( f, description=description, parent_classes=set(), # Used for recursive calls. @@ -253,6 +253,11 @@ def fix_arg(arg: str) -> str: ) parser_definition.apply(parser) + # Print help message when no arguments are passed in. (but arguments are + # expected) + if len(args) == 0 and parser_definition.has_required_args: + args = ["--help"] + if return_parser: _arguments.USE_RICH = True return parser diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 303031f37..5eee4959b 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -32,9 +32,10 @@ class ParserSpecification: helptext_from_nested_class_field_name: Dict[str, Optional[str]] subparsers_from_name: Dict[str, SubparsersSpecification] prefix: str + has_required_args: bool @staticmethod - def from_callable( + def from_callable_or_type( f: Callable[..., T], description: Optional[str], parent_classes: Set[Type], @@ -44,7 +45,7 @@ def from_callable( ], prefix: str, ) -> ParserSpecification: - """Create a parser definition from a callable.""" + """Create a parser definition from a callable or type.""" # Resolve generic types. f, type_from_typevar = _resolver.resolve_generic_types(f) @@ -68,6 +69,7 @@ def from_callable( # cleaned up. parent_classes = parent_classes | {cast(Type, f)} + has_required_args = False args = [] helptext_from_nested_class_field_name = {} subparsers_from_name = {} @@ -122,7 +124,7 @@ def from_callable( field.default, ), ) - nested_parser = ParserSpecification.from_callable( + nested_parser = ParserSpecification.from_callable_or_type( field.typ, description=None, parent_classes=parent_classes, @@ -155,19 +157,21 @@ def from_callable( continue # (3) Handle primitive or fixed types. These produce a single argument! - args.append( - _arguments.ArgumentDefinition( - prefix=prefix, - field=field, - type_from_typevar=type_from_typevar, - ) + arg = _arguments.ArgumentDefinition( + prefix=prefix, + field=field, + type_from_typevar=type_from_typevar, ) + args.append(arg) + if arg.lowered.required: + has_required_args = True # If a later subparser is required, all previous ones should be as well. subparsers_required = False for name, subparsers in list(subparsers_from_name.items())[::-1]: if subparsers.required: subparsers_required = True + has_required_args = True subparsers_from_name[name] = dataclasses.replace( subparsers, required=subparsers_required ) @@ -181,6 +185,7 @@ def from_callable( helptext_from_nested_class_field_name=helptext_from_nested_class_field_name, subparsers_from_name=subparsers_from_name, prefix=prefix, + has_required_args=has_required_args, ) def apply(self, parser: argparse.ArgumentParser) -> None: @@ -299,7 +304,7 @@ def from_field( ), ) - subparser = ParserSpecification.from_callable( + subparser = ParserSpecification.from_callable_or_type( option, description=found_subcommand_configs[0].description, parent_classes=parent_classes, From 50b114ed41bf7dba7dc1396ffdc2d2dea8c6ec8c Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 2 Oct 2022 23:44:21 -0700 Subject: [PATCH 165/491] Add `dcargs.conf.OmitSubcommandPrefixes`, version bump --- dcargs/_arguments.py | 60 +++++++++++++++++++++++++++-------------- dcargs/_parsers.py | 11 ++++++-- dcargs/conf/__init__.py | 10 ++++++- dcargs/conf/_markers.py | 12 +++++++++ pyproject.toml | 2 +- tests/test_metadata.py | 43 +++++++++++++++++++++++++++++ 6 files changed, 114 insertions(+), 24 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 8d06d00e5..ab2d295bc 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -36,25 +36,12 @@ from backports.cached_property import cached_property # type: ignore -class _PatchedList(list): - """Custom tuple type, for avoiding "default not in choices" errors when the default - is set to MISSING_NONPROP. - - This solves a choices error raised by argparse in a very specific edge case: - literals in containers as positional arguments.""" - - def __init__(self, li): - super(_PatchedList, self).__init__(li) - - def __contains__(self, x: Any) -> bool: - return list.__contains__(self, x) or x is _fields.MISSING_NONPROP - - @dataclasses.dataclass(frozen=True) class ArgumentDefinition: """Structure containing everything needed to define an argument.""" prefix: str # Prefix for nesting. + subcommand_prefix: str # Prefix for nesting. field: _fields.FieldDefinition type_from_typevar: Dict[TypeVar, Type] @@ -77,10 +64,7 @@ def add_argument( # the field default to a string format, then back to the desired type. kwargs["default"] = _fields.MISSING_NONPROP - if "choices" in kwargs: - kwargs["choices"] = _PatchedList(kwargs["choices"]) - - # Note that the name must be passed in as a position argument. + # Add argument! Note that the name must be passed in as a position argument. arg = parser.add_argument(name_or_flag, **kwargs) # Do our best to tab complete paths. @@ -119,8 +103,9 @@ def lowered(self) -> LoweredArgumentDefinition: _rule_recursive_instantiator_from_type, _rule_convert_defaults_to_strings, _rule_generate_helptext, - _rule_set_name_or_flag, + _rule_set_name_or_flag_and_dest, _rule_positional_special_handling, + _rule_static_cast_choices_to_patched_list, ) return functools.reduce( lambda lowered, rule: rule(self, lowered), @@ -362,19 +347,30 @@ def _rule_generate_helptext( return dataclasses.replace(lowered, help=" ".join(help_parts)) -def _rule_set_name_or_flag( +def _rule_set_name_or_flag_and_dest( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: + # Positional arguments: no -- prefix. if arg.field.is_positional(): name_or_flag = _strings.make_field_name([arg.prefix, arg.field.name]) + # Negated booleans. elif lowered.action == "store_false": name_or_flag = "--" + _strings.make_field_name( [arg.prefix, "no-" + arg.field.name] ) + # Prefix keyword arguments with --. else: name_or_flag = "--" + _strings.make_field_name([arg.prefix, arg.field.name]) + # Strip. + if name_or_flag.startswith("--") and arg.subcommand_prefix != "": + # This will run even when unused because we want the assert. + strip_prefix = "--" + arg.subcommand_prefix + "." + assert name_or_flag.startswith(strip_prefix) + if _markers.OMIT_SUBCOMMAND_PREFIXES in arg.field.markers: + name_or_flag = "--" + name_or_flag[len(strip_prefix) :] + return dataclasses.replace( lowered, name_or_flag=name_or_flag, @@ -410,3 +406,27 @@ def _rule_positional_special_handling( metavar=metavar, nargs=nargs, ) + + +class _PatchedList(list): + """Custom list type, for avoiding "default not in choices" errors when the default + is set to MISSING_NONPROP. + + This solves a choices error raised by argparse in a very specific edge case: + literals in containers as positional arguments.""" + + def __init__(self, li): + super(_PatchedList, self).__init__(li) + + def __contains__(self, x: Any) -> bool: + return list.__contains__(self, x) or x is _fields.MISSING_NONPROP + + +def _rule_static_cast_choices_to_patched_list( + arg: ArgumentDefinition, + lowered: LoweredArgumentDefinition, +) -> LoweredArgumentDefinition: + return dataclasses.replace( + lowered, + choices=_PatchedList(lowered.choices) if lowered.choices is not None else None, + ) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 5eee4959b..eddd5f02e 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -6,7 +6,7 @@ import itertools from typing import Any, Callable, Dict, List, Optional, Set, Type, TypeVar, Union, cast -from typing_extensions import get_args, get_origin +from typing_extensions import Annotated, get_args, get_origin from . import ( _argparse_formatter, @@ -44,6 +44,7 @@ def from_callable_or_type( T, _fields.PropagatingMissingType, _fields.NonpropagatingMissingType ], prefix: str, + subcommand_prefix: str = "", ) -> ParserSpecification: """Create a parser definition from a callable or type.""" @@ -131,6 +132,7 @@ def from_callable_or_type( parent_type_from_typevar=type_from_typevar, default_instance=field.default, prefix=_strings.make_field_name([prefix, field.name]), + subcommand_prefix=subcommand_prefix, ) args.extend(nested_parser.args) @@ -159,6 +161,7 @@ def from_callable_or_type( # (3) Handle primitive or fixed types. These produce a single argument! arg = _arguments.ArgumentDefinition( prefix=prefix, + subcommand_prefix=subcommand_prefix, field=field, type_from_typevar=type_from_typevar, ) @@ -305,12 +308,16 @@ def from_field( ) subparser = ParserSpecification.from_callable_or_type( - option, + # Recursively apply markers. + Annotated.__class_getitem__((option,) + tuple(field.markers)) # type: ignore + if len(field.markers) > 0 + else option, description=found_subcommand_configs[0].description, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, default_instance=found_subcommand_configs[0].default, prefix=prefix, + subcommand_prefix=prefix, ) # Apply prefix to helptext in nested classes in subparsers. diff --git a/dcargs/conf/__init__.py b/dcargs/conf/__init__.py index 743339f8f..59c85af33 100644 --- a/dcargs/conf/__init__.py +++ b/dcargs/conf/__init__.py @@ -5,13 +5,21 @@ Features here are supported, but generally unnecessary and should be used sparingly. """ -from ._markers import AvoidSubcommands, Fixed, FlagConversionOff, Positional, Suppress +from ._markers import ( + AvoidSubcommands, + Fixed, + FlagConversionOff, + OmitSubcommandPrefixes, + Positional, + Suppress, +) from ._subcommands import subcommand __all__ = [ "AvoidSubcommands", "Fixed", "FlagConversionOff", + "OmitSubcommandPrefixes", "Positional", "Suppress", "subcommand", diff --git a/dcargs/conf/_markers.py b/dcargs/conf/_markers.py index 4b8c146da..882bca69e 100644 --- a/dcargs/conf/_markers.py +++ b/dcargs/conf/_markers.py @@ -54,3 +54,15 @@ def __repr__(self): Can be used directly on union types, `AvoidSubcommands[Union[...]]`, or recursively applied to nested types.""" + +OMIT_SUBCOMMAND_PREFIXES = _make_marker("OmitSubcommandPrefixes") +OmitSubcommandPrefixes = Annotated[T, OMIT_SUBCOMMAND_PREFIXES] +"""Make flags used for keyword arguments in subcommands shorter by omitting prefixes. + +If we have a structure with the field: + + cmd: Union[Commit, Checkout] + +By default, --cmd.branch may be generated as a flag for each dataclass in the union. +If subcommand prefixes are omitted, we would instead simply have --branch. +""" diff --git a/pyproject.toml b/pyproject.toml index 851d5ea8e..a9ea8a8fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.14" +version = "0.3.15" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] diff --git a/tests/test_metadata.py b/tests/test_metadata.py index b7c8b3941..7cda8cc83 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -7,6 +7,49 @@ import dcargs +def test_omit_subcommand_prefix(): + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + bc: dcargs.conf.OmitSubcommandPrefixes[ + Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + ] + + assert ( + dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--y", "5"], + ) + == dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--y", "5"], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) + ) + assert ( + dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--y", "8"], + ) + == dcargs.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--y", "8"], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + ) + + def test_avoid_subparser_with_default(): @dataclasses.dataclass class DefaultInstanceHTTPServer: From e63cf5b536e8c4febf83876c9091f847f67506f6 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 3 Oct 2022 00:16:47 -0700 Subject: [PATCH 166/491] Marker simplification --- dcargs/_arguments.py | 8 ++--- dcargs/_fields.py | 4 +-- dcargs/_parsers.py | 4 +-- dcargs/conf/_markers.py | 66 ++++++++++++++++++++++++----------------- pyproject.toml | 2 +- tests/test_metadata.py | 4 +-- 6 files changed, 48 insertions(+), 40 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index ab2d295bc..afb9a2dd0 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -170,7 +170,7 @@ def _rule_handle_boolean_flags( if ( arg.field.default in _fields.MISSING_SINGLETONS or arg.field.is_positional() - or _markers.FLAG_CONVERSION_OFF in arg.field.markers + or _markers.FlagConversionOff in arg.field.markers ): # Treat bools as a normal parameter. return lowered @@ -207,7 +207,7 @@ def _rule_recursive_instantiator_from_type( Conversions from strings to our desired types happen in the instantiator; this is a bit more flexible, and lets us handle more complex types like enums and multi-type tuples.""" - if _markers.FIXED in arg.field.markers: + if _markers.Fixed in arg.field.markers: return dataclasses.replace( lowered, instantiator=None, @@ -294,7 +294,7 @@ def _rule_generate_helptext( """Generate helptext from docstring, argument name, default values.""" # If the suppress marker is attached, hide the argument. - if _markers.SUPPRESS in arg.field.markers: + if _markers.Suppress in arg.field.markers: return dataclasses.replace(lowered, help=argparse.SUPPRESS) help_parts = [] @@ -368,7 +368,7 @@ def _rule_set_name_or_flag_and_dest( # This will run even when unused because we want the assert. strip_prefix = "--" + arg.subcommand_prefix + "." assert name_or_flag.startswith(strip_prefix) - if _markers.OMIT_SUBCOMMAND_PREFIXES in arg.field.markers: + if _markers.OmitSubcommandPrefixes in arg.field.markers: name_or_flag = "--" + name_or_flag[len(strip_prefix) :] return dataclasses.replace( diff --git a/dcargs/_fields.py b/dcargs/_fields.py index 95048efc0..915d6ef70 100644 --- a/dcargs/_fields.py +++ b/dcargs/_fields.py @@ -79,7 +79,7 @@ def add_markers(self, markers: Tuple[_markers.Marker, ...]) -> FieldDefinition: def is_positional(self) -> bool: return ( # Explicit positionals. - _markers.POSITIONAL in self.markers + _markers.Positional in self.markers # Dummy dataclasses should have a single positional field. or self.name == _strings.dummy_field_name ) @@ -555,7 +555,7 @@ def _field_list_from_params( typ=hints[param.name], default=default, helptext=helptext, - markers=(_markers.POSITIONAL,) + markers=(_markers.Positional,) if param.kind is inspect.Parameter.POSITIONAL_ONLY else (), ) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index eddd5f02e..3cf6645ad 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -95,7 +95,7 @@ def from_callable_or_type( f"Field {field.name} has an unbound TypeVar: {field.typ}." ) - if _markers.FIXED not in field.markers: + if _markers.Fixed not in field.markers: # (1) Handle Unions over callables; these result in subparsers. subparsers_attempt = SubparsersSpecification.from_field( field, @@ -106,7 +106,7 @@ def from_callable_or_type( if subparsers_attempt is not None: if ( not subparsers_attempt.required - and _markers.AVOID_SUBCOMMANDS in field.markers + and _markers.AvoidSubcommands in field.markers ): # Don't make a subparser. field = dataclasses.replace(field, typ=type(field.default)) diff --git a/dcargs/conf/_markers.py b/dcargs/conf/_markers.py index 882bca69e..866ddccec 100644 --- a/dcargs/conf/_markers.py +++ b/dcargs/conf/_markers.py @@ -1,62 +1,45 @@ -from typing import Type, TypeVar +from typing import TYPE_CHECKING, Type, TypeVar from typing_extensions import Annotated from .. import _singleton +# Current design issue: _dynamic_marker_types are applied recursively to nested +# structures, but can't be unapplied. -class Marker(_singleton.Singleton): - pass - - -def _make_marker(description: str) -> Marker: - class _InnerMarker(Marker): - def __repr__(self): - return description - - return _InnerMarker() - - -# Current design issue: markers are applied recursively to nested structures, but can't -# be unapplied. +# Note that all Annotated[T, None] values are just for static checkers. The real marker +# singletons are instantiated dynamically below. T = TypeVar("T", bound=Type) -POSITIONAL = _make_marker("Positional") -Positional = Annotated[T, POSITIONAL] +Positional = Annotated[T, None] """A type `T` can be annotated as `Positional[T]` if we want to parse it as a positional argument.""" - -FIXED = _make_marker("Fixed") -Fixed = Annotated[T, FIXED] +Fixed = Annotated[T, None] """A type `T` can be annotated as `Fixed[T]` to prevent `dcargs.cli` from parsing it; a default value should be set instead. Note that fields with defaults that can't be parsed will also be marked as fixed automatically.""" -SUPPRESS = _make_marker("Suppress") -Suppress = Annotated[T, FIXED, SUPPRESS] +Suppress = Annotated[T, None] """A type `T` can be annotated as `Suppress[T]` to prevent `dcargs.cli` from parsing it, and to prevent it from showing up in helptext.""" -FLAG_CONVERSION_OFF = _make_marker("FlagConversionOff") -FlagConversionOff = Annotated[T, FLAG_CONVERSION_OFF] +FlagConversionOff = Annotated[T, None] """Turn off flag conversion for booleans with default values. Instead, types annotated with `bool` will expect an explicit True or False. Can be used directly on boolean annotations, `FlagConversionOff[bool]`, or recursively applied to nested types.""" -AVOID_SUBCOMMANDS = _make_marker("AvoidSubcommands") -AvoidSubcommands = Annotated[T, AVOID_SUBCOMMANDS] +AvoidSubcommands = Annotated[T, None] """Avoid creating subcommands when a default is provided for unions over nested types. This simplifies CLI interfaces, but makes them less expressive. Can be used directly on union types, `AvoidSubcommands[Union[...]]`, or recursively applied to nested types.""" -OMIT_SUBCOMMAND_PREFIXES = _make_marker("OmitSubcommandPrefixes") -OmitSubcommandPrefixes = Annotated[T, OMIT_SUBCOMMAND_PREFIXES] +OmitSubcommandPrefixes = Annotated[T, None] """Make flags used for keyword arguments in subcommands shorter by omitting prefixes. If we have a structure with the field: @@ -66,3 +49,30 @@ def __repr__(self): By default, --cmd.branch may be generated as a flag for each dataclass in the union. If subcommand prefixes are omitted, we would instead simply have --branch. """ + + +# Dynamically generate marker singletons. +# These can be used one of two ways: +# - Marker[T] +# - Annotated[T, Marker] +class Marker(_singleton.Singleton): + def __getitem__(self, key): + print(key) + return Annotated.__class_getitem__((key, self)) # type: ignore + + +if not TYPE_CHECKING: + + def _make_marker(description: str) -> Marker: + class _InnerMarker(Marker): + def __repr__(self): + return description + + return _InnerMarker() + + _dynamic_marker_types = {} + for k, v in dict(globals()).items(): + if v == Annotated[T, None]: + _dynamic_marker_types[k] = _make_marker(k) + globals().update(_dynamic_marker_types) + del _dynamic_marker_types diff --git a/pyproject.toml b/pyproject.toml index a9ea8a8fc..af92231b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.15" +version = "0.3.16" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 7cda8cc83..5b030339f 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -355,9 +355,7 @@ class A: with pytest.raises(SystemExit): assert dcargs.cli( - dcargs.conf.Fixed[ - dcargs.conf.FlagConversionOff[A], - ], + dcargs.conf.Fixed[dcargs.conf.FlagConversionOff[A]], args=["--x", "True"], default=A(False), ) == A(True) From e7fb17a150fb9816d0610055d47ed6fe71e63724 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 3 Oct 2022 00:55:15 -0700 Subject: [PATCH 167/491] Remove debug print --- dcargs/conf/_markers.py | 1 - pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dcargs/conf/_markers.py b/dcargs/conf/_markers.py index 866ddccec..199932174 100644 --- a/dcargs/conf/_markers.py +++ b/dcargs/conf/_markers.py @@ -57,7 +57,6 @@ # - Annotated[T, Marker] class Marker(_singleton.Singleton): def __getitem__(self, key): - print(key) return Annotated.__class_getitem__((key, self)) # type: ignore diff --git a/pyproject.toml b/pyproject.toml index af92231b4..7c004ce4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.16" +version = "0.3.17" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From ef944b0d177aaf838dc7868639ba8482529ef990 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 3 Oct 2022 11:40:03 -0700 Subject: [PATCH 168/491] Add `dcargs.conf.SuppressFixed[]` --- dcargs/_arguments.py | 5 ++++- dcargs/conf/__init__.py | 2 ++ dcargs/conf/_markers.py | 20 +++++++++++++------- tests/test_helptext.py | 14 ++++++++++++++ 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index afb9a2dd0..dee8547b7 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -294,7 +294,10 @@ def _rule_generate_helptext( """Generate helptext from docstring, argument name, default values.""" # If the suppress marker is attached, hide the argument. - if _markers.Suppress in arg.field.markers: + if _markers.Suppress in arg.field.markers or ( + _markers.SuppressFixed in arg.field.markers + and _markers.Fixed in arg.field.markers + ): return dataclasses.replace(lowered, help=argparse.SUPPRESS) help_parts = [] diff --git a/dcargs/conf/__init__.py b/dcargs/conf/__init__.py index 59c85af33..7cf5c38c3 100644 --- a/dcargs/conf/__init__.py +++ b/dcargs/conf/__init__.py @@ -12,6 +12,7 @@ OmitSubcommandPrefixes, Positional, Suppress, + SuppressFixed, ) from ._subcommands import subcommand @@ -22,5 +23,6 @@ "OmitSubcommandPrefixes", "Positional", "Suppress", + "SuppressFixed", "subcommand", ] diff --git a/dcargs/conf/_markers.py b/dcargs/conf/_markers.py index 199932174..5359637d5 100644 --- a/dcargs/conf/_markers.py +++ b/dcargs/conf/_markers.py @@ -12,34 +12,40 @@ T = TypeVar("T", bound=Type) -Positional = Annotated[T, None] +# This could ideally be Annotated[T, None], but SpecialForm aliases are not well supported by static analysis tools. +StaticPlaceholder = Annotated + +Positional = StaticPlaceholder[T, None] """A type `T` can be annotated as `Positional[T]` if we want to parse it as a positional argument.""" -Fixed = Annotated[T, None] +Fixed = StaticPlaceholder[T, None] """A type `T` can be annotated as `Fixed[T]` to prevent `dcargs.cli` from parsing it; a default value should be set instead. Note that fields with defaults that can't be parsed will also be marked as fixed automatically.""" -Suppress = Annotated[T, None] +Suppress = StaticPlaceholder[T, None] """A type `T` can be annotated as `Suppress[T]` to prevent `dcargs.cli` from parsing it, and to prevent it from showing up in helptext.""" -FlagConversionOff = Annotated[T, None] +SuppressFixed = StaticPlaceholder[T, None] +"""Hide fields that are either manually or automatically marked as fixed.""" + +FlagConversionOff = StaticPlaceholder[T, None] """Turn off flag conversion for booleans with default values. Instead, types annotated with `bool` will expect an explicit True or False. Can be used directly on boolean annotations, `FlagConversionOff[bool]`, or recursively applied to nested types.""" -AvoidSubcommands = Annotated[T, None] +AvoidSubcommands = StaticPlaceholder[T, None] """Avoid creating subcommands when a default is provided for unions over nested types. This simplifies CLI interfaces, but makes them less expressive. Can be used directly on union types, `AvoidSubcommands[Union[...]]`, or recursively applied to nested types.""" -OmitSubcommandPrefixes = Annotated[T, None] +OmitSubcommandPrefixes = StaticPlaceholder[T, None] """Make flags used for keyword arguments in subcommands shorter by omitting prefixes. If we have a structure with the field: @@ -71,7 +77,7 @@ def __repr__(self): _dynamic_marker_types = {} for k, v in dict(globals()).items(): - if v == Annotated[T, None]: + if v == StaticPlaceholder[T, None]: _dynamic_marker_types[k] = _make_marker(k) globals().update(_dynamic_marker_types) del _dynamic_marker_types diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 54b53c2cf..61b64c111 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -624,3 +624,17 @@ def main(x: Any = Struct()): helptext = _get_helptext(main) assert "--x.a" in helptext assert "--x.b" not in helptext + + +def test_suppress_fixed(): + @dataclasses.dataclass + class Struct: + a: int = 5 + b: dcargs.conf.SuppressFixed[dcargs.conf.Fixed[str]] = "7" + + def main(x: Any = Struct()): + pass + + helptext = _get_helptext(main) + assert "--x.a" in helptext + assert "--x.b" not in helptext From 913155a92929d5298fb28b2cf1668c634349c572 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 3 Oct 2022 12:37:01 -0700 Subject: [PATCH 169/491] Fix rich escaping for subcommands, fix mypy --- dcargs/_parsers.py | 4 +++- dcargs/conf/__init__.py | 3 +++ dcargs/conf/_markers.py | 25 ++++++++++++------------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 3cf6645ad..440cbe659 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -1,4 +1,5 @@ """Interface for generating `argparse.ArgumentParser()` definitions from callables.""" + from __future__ import annotations import argparse @@ -6,6 +7,7 @@ import itertools from typing import Any, Callable, Dict, List, Optional, Set, Type, TypeVar, Union, cast +import rich.markup from typing_extensions import Annotated, get_args, get_origin from . import ( @@ -455,7 +457,7 @@ def apply( subparser = argparse_subparsers.add_parser( name, formatter_class=_argparse_formatter.DcargsArgparseHelpFormatter, - help=subparser_def.description, + help=rich.markup.escape(subparser_def.description), ) subparser_def.apply(subparser) diff --git a/dcargs/conf/__init__.py b/dcargs/conf/__init__.py index 7cf5c38c3..1c74f72a4 100644 --- a/dcargs/conf/__init__.py +++ b/dcargs/conf/__init__.py @@ -2,6 +2,9 @@ configuration metadata to types via [PEP 593](https://peps.python.org/pep-0593/) runtime annotations. +Configuration flags are applied recursively, and should generally be subscripted: +`Fixed[T]`, `Suppress[T]`, etc. + Features here are supported, but generally unnecessary and should be used sparingly. """ diff --git a/dcargs/conf/_markers.py b/dcargs/conf/_markers.py index 5359637d5..aca503c58 100644 --- a/dcargs/conf/_markers.py +++ b/dcargs/conf/_markers.py @@ -4,48 +4,47 @@ from .. import _singleton -# Current design issue: _dynamic_marker_types are applied recursively to nested -# structures, but can't be unapplied. +# Current design issue: markers are applied recursively to nested structures, but can't +# be unapplied. # Note that all Annotated[T, None] values are just for static checkers. The real marker # singletons are instantiated dynamically below. +# +# An alias could ideally be made, but SpecialForm aliases are not well supported by static analysis tools. T = TypeVar("T", bound=Type) -# This could ideally be Annotated[T, None], but SpecialForm aliases are not well supported by static analysis tools. -StaticPlaceholder = Annotated - -Positional = StaticPlaceholder[T, None] +Positional = Annotated[T, None] """A type `T` can be annotated as `Positional[T]` if we want to parse it as a positional argument.""" -Fixed = StaticPlaceholder[T, None] +Fixed = Annotated[T, None] """A type `T` can be annotated as `Fixed[T]` to prevent `dcargs.cli` from parsing it; a default value should be set instead. Note that fields with defaults that can't be parsed will also be marked as fixed automatically.""" -Suppress = StaticPlaceholder[T, None] +Suppress = Annotated[T, None] """A type `T` can be annotated as `Suppress[T]` to prevent `dcargs.cli` from parsing it, and to prevent it from showing up in helptext.""" -SuppressFixed = StaticPlaceholder[T, None] +SuppressFixed = Annotated[T, None] """Hide fields that are either manually or automatically marked as fixed.""" -FlagConversionOff = StaticPlaceholder[T, None] +FlagConversionOff = Annotated[T, None] """Turn off flag conversion for booleans with default values. Instead, types annotated with `bool` will expect an explicit True or False. Can be used directly on boolean annotations, `FlagConversionOff[bool]`, or recursively applied to nested types.""" -AvoidSubcommands = StaticPlaceholder[T, None] +AvoidSubcommands = Annotated[T, None] """Avoid creating subcommands when a default is provided for unions over nested types. This simplifies CLI interfaces, but makes them less expressive. Can be used directly on union types, `AvoidSubcommands[Union[...]]`, or recursively applied to nested types.""" -OmitSubcommandPrefixes = StaticPlaceholder[T, None] +OmitSubcommandPrefixes = Annotated[T, None] """Make flags used for keyword arguments in subcommands shorter by omitting prefixes. If we have a structure with the field: @@ -77,7 +76,7 @@ def __repr__(self): _dynamic_marker_types = {} for k, v in dict(globals()).items(): - if v == StaticPlaceholder[T, None]: + if v == Annotated[T, None]: _dynamic_marker_types[k] = _make_marker(k) globals().update(_dynamic_marker_types) del _dynamic_marker_types From 515dd00e9c240f3fce629b46242031906fcc35ae Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 4 Oct 2022 10:11:53 -0700 Subject: [PATCH 170/491] Support types from PEP 585, 604 --- dcargs/_resolver.py | 23 ++++++- tests/test_kw_only_only_py310.py | 23 ------- .../test_new_style_annotations_only_py310.py | 60 +++++++++++++++++++ 3 files changed, 82 insertions(+), 24 deletions(-) delete mode 100644 tests/test_kw_only_only_py310.py create mode 100644 tests/test_new_style_annotations_only_py310.py diff --git a/dcargs/_resolver.py b/dcargs/_resolver.py index 8848eade0..1abcaf3fb 100644 --- a/dcargs/_resolver.py +++ b/dcargs/_resolver.py @@ -1,14 +1,16 @@ """Utilities for resolving types and forward references.""" - import collections.abc import copy import dataclasses +import sys from typing import ( Any, Callable, Dict, + FrozenSet, List, Optional, + Set, Tuple, Type, TypeVar, @@ -171,6 +173,25 @@ def apply_type_from_typevar( if get_origin(typ) is collections.abc.Callable: assert isinstance(args[0], list) args = tuple(args[0]) + args[1:] + + # Convert Python 3.10-style types to their typing library equivalents, which + # support `.copy_with()`. + if sys.version_info[:2] >= (3, 10): + import types + + for new, old in { + # PEP 585 + tuple: Tuple, + list: List, + dict: Dict, + set: Set, + frozenset: FrozenSet, + # PEP 604 + types.UnionType: Union, + }.items(): + if isinstance(typ, new) or get_origin(typ) is new: # type: ignore + typ = old.__getitem__(args) # type: ignore + return typ.copy_with(tuple(apply_type_from_typevar(x, type_from_typevar) for x in args)) # type: ignore return typ diff --git a/tests/test_kw_only_only_py310.py b/tests/test_kw_only_only_py310.py deleted file mode 100644 index f7cef225a..000000000 --- a/tests/test_kw_only_only_py310.py +++ /dev/null @@ -1,23 +0,0 @@ -import dataclasses - -import pytest - -import dcargs - - -@dataclasses.dataclass -class Args: - a: int - b: int - _: dataclasses.KW_ONLY # type: ignore - c: int = 7 - d: int # type: ignore - - -def test_kw_only(): - assert dcargs.cli(Args, args="--a 5 --b 3 --c 2 --d 1".split(" ")) == Args( - 5, 3, c=2, d=1 - ) - assert dcargs.cli(Args, args="--a 5 --b 3 --d 1".split(" ")) == Args(5, 3, c=7, d=1) - with pytest.raises(SystemExit): - dcargs.cli(Args, args="--a 5 --b 3".split(" ")) diff --git a/tests/test_new_style_annotations_only_py310.py b/tests/test_new_style_annotations_only_py310.py new file mode 100644 index 000000000..162c2064a --- /dev/null +++ b/tests/test_new_style_annotations_only_py310.py @@ -0,0 +1,60 @@ +import dataclasses +from typing import Any, Literal + +import pytest + +import dcargs + + +def test_union_basic(): + def main(x: int | str) -> int | str: + return x + + assert dcargs.cli(main, args=["--x", "5"]) == 5 + assert dcargs.cli(main, args=["--x", "6"]) == 6 + assert dcargs.cli(main, args=["--x", "five"]) == "five" + + +def test_union_with_list(): + def main(x: int | str | list[bool]) -> Any: + return x + + assert dcargs.cli(main, args=["--x", "5"]) == 5 + assert dcargs.cli(main, args=["--x", "6"]) == 6 + assert dcargs.cli(main, args=["--x", "five"]) == "five" + assert dcargs.cli(main, args=["--x", "True"]) == "True" + assert dcargs.cli(main, args=["--x", "True", "False"]) == [True, False] + + +def test_union_literal(): + def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: + return x + + assert dcargs.cli(main, args=["--x", "5"]) == 5 + assert dcargs.cli(main, args=["--x", "6"]) == "6" + assert dcargs.cli(main, args=["--x", "five"]) == "five" + + +def test_super_nested(): + def main( + x: None + | list[ + tuple[ + None | int, + Literal[3, 4], + tuple[int, int] | tuple[str, str], + ] + ] = None + ) -> Any: + return x + + assert dcargs.cli(main, args=[]) is None + assert dcargs.cli(main, args="--x None".split(" ")) is None + assert dcargs.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] + assert dcargs.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] + assert dcargs.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ + (2, 3, ("x", "2")), + (2, 3, (1, 2)), + ] + with pytest.raises(SystemExit): + dcargs.cli(main, args=["--help"]) From 86aa11a14d01de426db8d6b13d923429456dfc05 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 4 Oct 2022 17:29:10 -0700 Subject: [PATCH 171/491] Bump version, global state fix + TODO notes --- dcargs/_arguments.py | 2 ++ dcargs/_cli.py | 23 +++++++++++++---------- pyproject.toml | 2 +- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index dee8547b7..dfa94a1c4 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -279,6 +279,8 @@ def as_str(x: Any) -> Tuple[str, ...]: # This can be turned off when we don't want rich-based formatting. (notably for # completion scripts) +# +# TODO: the global state here is gross. Should be revisited. USE_RICH = True diff --git a/dcargs/_cli.py b/dcargs/_cli.py index d5d41b97d..300dd38f6 100644 --- a/dcargs/_cli.py +++ b/dcargs/_cli.py @@ -205,16 +205,6 @@ def _cli_impl( else: dummy_wrapped = False - # Map a callable to the relevant CLI arguments + subparsers. - parser_definition = _parsers.ParserSpecification.from_callable_or_type( - f, - description=description, - parent_classes=set(), # Used for recursive calls. - parent_type_from_typevar=None, # Used for recursive calls. - default_instance=default_instance_internal, # Overrides for default values. - prefix="", # Used for recursive calls. - ) - # Read and fix arguments. If the user passes in --field_name instead of # --field-name, correct for them. args = sys.argv[1:] if args is None else args @@ -237,6 +227,9 @@ def fix_arg(arg: str) -> str: # goal, but manual parsing of argv is convenient for turning off formatting. print_completion = len(args) >= 2 and args[0] == "--dcargs-print-completion" + # Note: setting USE_RICH must happen before the parser specification is generated. + # TODO: revisit this. Ideally we should be able to eliminate the global state + # changes. completion_shell = None if print_completion: completion_shell = args[1] @@ -245,6 +238,16 @@ def fix_arg(arg: str) -> str: else: _arguments.USE_RICH = True + # Map a callable to the relevant CLI arguments + subparsers. + parser_definition = _parsers.ParserSpecification.from_callable_or_type( + f, + description=description, + parent_classes=set(), # Used for recursive calls. + parent_type_from_typevar=None, # Used for recursive calls. + default_instance=default_instance_internal, # Overrides for default values. + prefix="", # Used for recursive calls. + ) + # Generate parser! with _argparse_formatter.ansi_context(): parser = argparse.ArgumentParser( diff --git a/pyproject.toml b/pyproject.toml index 7c004ce4b..1245b58fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.17" +version = "0.3.18" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] From d5e6f8095d3000ab0ae89335b1c3b3dab199c080 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 4 Oct 2022 17:45:54 -0700 Subject: [PATCH 172/491] Fix `dcargs.conf.SuppressFixed` bug + test --- dcargs/_arguments.py | 3 +-- pyproject.toml | 2 +- tests/test_helptext.py | 16 +++++++++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index dfa94a1c4..8efeaa9ba 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -297,8 +297,7 @@ def _rule_generate_helptext( # If the suppress marker is attached, hide the argument. if _markers.Suppress in arg.field.markers or ( - _markers.SuppressFixed in arg.field.markers - and _markers.Fixed in arg.field.markers + _markers.SuppressFixed in arg.field.markers and lowered.is_fixed() ): return dataclasses.replace(lowered, help=argparse.SUPPRESS) diff --git a/pyproject.toml b/pyproject.toml index 1245b58fb..89a201550 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.18" +version = "0.3.19" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 61b64c111..2246f110a 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -626,7 +626,7 @@ def main(x: Any = Struct()): assert "--x.b" not in helptext -def test_suppress_fixed(): +def test_suppress_manual_fixed(): @dataclasses.dataclass class Struct: a: int = 5 @@ -638,3 +638,17 @@ def main(x: Any = Struct()): helptext = _get_helptext(main) assert "--x.a" in helptext assert "--x.b" not in helptext + + +def test_suppress_auto_fixed(): + @dataclasses.dataclass + class Struct: + a: int = 5 + b: Callable = lambda x: 5 + + def main(x: dcargs.conf.SuppressFixed[Any] = Struct()): + pass + + helptext = _get_helptext(main) + assert "--x.a" in helptext + assert "--x.b" not in helptext From a04dfddfe403cd59e3764aef16a990d7dd4efece Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 4 Oct 2022 18:22:07 -0700 Subject: [PATCH 173/491] Consider indentation for help positions --- dcargs/_argparse_formatter.py | 5 +++-- pyproject.toml | 2 +- tests/test_helptext.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index ecef25375..e1a311b60 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -293,12 +293,13 @@ def _dcargs_format_root(self): def _format_action(self, action: argparse.Action): invocation = self.formatter._format_action_invocation(action) + indent = self.formatter._current_indent help_position = min( - self.formatter._action_max_length + 4, self.formatter._max_help_position + self.formatter._action_max_length + 4 + indent, + self.formatter._max_help_position, ) if self.formatter._fixed_help_position: help_position = 4 - indent = self.formatter._current_indent item_parts: List[RenderableType] = [] diff --git a/pyproject.toml b/pyproject.toml index 89a201550..1d259f784 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.19" +version = "0.3.20" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 2246f110a..0fb6b1c9b 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -609,7 +609,8 @@ def main2(x: Callable = nn.ReLU): helptext = _get_helptext(main2) assert "--x {fixed}" in helptext - assert "(fixed to: Date: Tue, 4 Oct 2022 18:38:23 -0700 Subject: [PATCH 174/491] Add `dcargs.extras.literal_type_from_choices()` --- dcargs/extras/__init__.py | 2 ++ dcargs/extras/_choices_type.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 dcargs/extras/_choices_type.py diff --git a/dcargs/extras/__init__.py b/dcargs/extras/__init__.py index fbeb0b3f0..e70de2144 100644 --- a/dcargs/extras/__init__.py +++ b/dcargs/extras/__init__.py @@ -5,12 +5,14 @@ from .._argparse_formatter import set_accent_color from .._cli import get_parser from ._base_configs import subcommand_type_from_defaults +from ._choices_type import literal_type_from_choices from ._serialization import from_yaml, to_yaml __all__ = [ "set_accent_color", "subcommand_type_from_defaults", "get_parser", + "literal_type_from_choices", "from_yaml", "to_yaml", ] diff --git a/dcargs/extras/_choices_type.py b/dcargs/extras/_choices_type.py new file mode 100644 index 000000000..96a06b033 --- /dev/null +++ b/dcargs/extras/_choices_type.py @@ -0,0 +1,18 @@ +import enum +from typing import Iterable, Type, TypeVar, Union + +from typing_extensions import Literal + +T = TypeVar("T", bound=Union[int, str, bool, enum.Enum]) + + +def literal_type_from_choices(choices: Iterable[T]) -> Type[T]: + """Generate a typing.Literal[] type that constrains values to a set of choices. + + Using Literal[...] directly should generally be preferred, but this helper can be + used in the rare case that choices are generated dynamically. (for example, the keys + of a dictionary) + + Using the returned type as an annotation currently breaks for mypy, but is + understood by Pyright.""" + return Literal.__getitem__(tuple(choices)) # type: ignore From f5a10d3fcb95ed61c72c2ac9eb398cf4f200bc73 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 4 Oct 2022 20:11:45 -0700 Subject: [PATCH 175/491] v0.3.21: improve handling of non-rich ANSI sequences in inputs --- dcargs/_argparse_formatter.py | 14 +++++++++----- dcargs/_arguments.py | 9 +++++++-- dcargs/_parsers.py | 5 ++++- docs/update_example_docs.py | 6 ++++-- pyproject.toml | 2 +- tests/test_dcargs.py | 15 +++++++++++++++ 6 files changed, 40 insertions(+), 11 deletions(-) diff --git a/dcargs/_argparse_formatter.py b/dcargs/_argparse_formatter.py index e1a311b60..b9840008f 100644 --- a/dcargs/_argparse_formatter.py +++ b/dcargs/_argparse_formatter.py @@ -122,6 +122,8 @@ def inner() -> Generator[None, None, None]: def str_from_rich( renderable: RenderableType, width: Optional[int] = None, soft_wrap: bool = False ) -> str: + # TODO: everywhere that this function is used is a little bit sketchy. Could use + # re-thinking. console = Console(width=width, theme=THEME.as_rich_theme()) with console.capture() as out: console.print(renderable, soft_wrap=soft_wrap) @@ -152,7 +154,7 @@ def _format_args(self, action, default_metavar): out if self._strip_ansi_sequences else str_from_rich( - Text( + Text.from_ansi( out, style=THEME.metavar_fixed if out == "{fixed}" @@ -309,7 +311,9 @@ def _format_action(self, action: argparse.Action): # with the helptext strings defined via docstrings and set by # _arguments.py. assert action.help is not None - action.help = "[helptext]" + action.help + "[/helptext]" + action.help = str_from_rich( + Text.from_markup("[helptext]" + action.help + "[/helptext]") + ) if ( action.help and len(_strings.strip_ansi_sequences(invocation)) < help_position - 1 @@ -324,7 +328,7 @@ def _format_action(self, action: argparse.Action): style=THEME.invocation, ), # Unescape % signs, which need special handling in argparse. - Text.from_markup(action.help.replace("%%", "%")), + Text.from_ansi(action.help.replace("%%", "%")), ) item_parts.append(table) @@ -340,7 +344,7 @@ def _format_action(self, action: argparse.Action): item_parts.append( Padding( # Unescape % signs, which need special handling in argparse. - Text.from_markup(action.help.replace("%%", "%")), + Text.from_ansi(action.help.replace("%%", "%")), pad=(0, 0, 0, help_position), ) ) @@ -423,7 +427,7 @@ def _dcargs_format_nonroot(self): # Note: we don't use rich.rule.Rule() because this will make all of # the panels expand to fill the full width of the console. (this only # impacts single-column layouts) - self.formatter._dcargs_rule = Text( + self.formatter._dcargs_rule = Text.from_ansi( "─" * max_width, style=THEME.border, overflow="crop" ) elif len(self.formatter._dcargs_rule._text[0]) < max_width: diff --git a/dcargs/_arguments.py b/dcargs/_arguments.py index 8efeaa9ba..b005a6810 100644 --- a/dcargs/_arguments.py +++ b/dcargs/_arguments.py @@ -22,8 +22,9 @@ ) import rich.markup +from rich.text import Text -from . import _fields, _instantiators, _resolver +from . import _argparse_formatter, _fields, _instantiators, _resolver from . import _shtab as shtab from . import _strings from .conf import _markers @@ -286,7 +287,11 @@ def as_str(x: Any) -> Tuple[str, ...]: def _rich_tag_if_enabled(x: str, tag: str): x = rich.markup.escape(x) - return x if not USE_RICH else f"[{tag}]{x}[/{tag}]" + return ( + x + if not USE_RICH + else _argparse_formatter.str_from_rich(Text.from_markup(f"[{tag}]{x}[/{tag}]")) + ) def _rule_generate_helptext( diff --git a/dcargs/_parsers.py b/dcargs/_parsers.py index 440cbe659..8dcab4e24 100644 --- a/dcargs/_parsers.py +++ b/dcargs/_parsers.py @@ -106,6 +106,8 @@ def from_callable_or_type( prefix=_strings.make_field_name([prefix, field.name]), ) if subparsers_attempt is not None: + if subparsers_attempt.required: + has_required_args = True if ( not subparsers_attempt.required and _markers.AvoidSubcommands in field.markers @@ -136,6 +138,8 @@ def from_callable_or_type( prefix=_strings.make_field_name([prefix, field.name]), subcommand_prefix=subcommand_prefix, ) + if nested_parser.has_required_args: + has_required_args = True args.extend(nested_parser.args) # Include nested subparsers. @@ -176,7 +180,6 @@ def from_callable_or_type( for name, subparsers in list(subparsers_from_name.items())[::-1]: if subparsers.required: subparsers_required = True - has_required_args = True subparsers_from_name[name] = dataclasses.replace( subparsers, required=subparsers_required ) diff --git a/docs/update_example_docs.py b/docs/update_example_docs.py index 7e0ffa334..c026dd892 100644 --- a/docs/update_example_docs.py +++ b/docs/update_example_docs.py @@ -110,8 +110,10 @@ def main( ).write_text( "\n".join( [ - ".. Comment: this file is automatically generated by" - " `update_example_docs.py`.", + ( + ".. Comment: this file is automatically generated by" + " `update_example_docs.py`." + ), " It should not be modified manually.", "", f"{ex.index}. {ex.title}", diff --git a/pyproject.toml b/pyproject.toml index 1d259f784..485a00efd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcargs" -version = "0.3.20" +version = "0.3.21" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./dcargs/**/*"] diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 5cdf0adc8..b6dc19712 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -307,6 +307,21 @@ class A: assert dcargs.cli(A, args=["--x", "3"]) +# Hack for mypy. Not needed for pyright. +Choices = int +Choices = dcargs.extras.literal_type_from_choices([0, 1, 2]) # type: ignore + + +def test_dynamic_literal(): + @dataclasses.dataclass + class A: + x: Choices + + assert dcargs.cli(A, args=["--x", "1"]) == A(x=1) + with pytest.raises(SystemExit): + assert dcargs.cli(A, args=["--x", "3"]) + + def test_literal_bool(): def main(x: Literal[True]) -> bool: return x From f96c8559848739a5a397f792d15bcf4d9ea19665 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Oct 2022 12:53:47 -0700 Subject: [PATCH 176/491] Rename: `dcargs` -> `tyro` (#16) * Initial rename * Update README.md * Update README.md --- README.md | 124 ++++++++--- .../_static/css/compact_table_header.css | 8 + docs/source/_static/logo-dark.svg | 55 +++++ docs/source/_static/logo-light.svg | 56 +++++ docs/source/conf.py | 41 ++-- docs/source/examples/01_functions.rst | 6 +- docs/source/examples/02_dataclasses.rst | 6 +- .../examples/03_enums_and_containers.rst | 4 +- docs/source/examples/04_flags.rst | 6 +- .../examples/05_hierarchical_configs.rst | 6 +- .../examples/06_literals_and_unions.rst | 4 +- docs/source/examples/07_positional_args.rst | 6 +- docs/source/examples/08_subcommands.rst | 8 +- .../examples/09_multiple_subcommands.rst | 4 +- docs/source/examples/10_base_configs.rst | 20 +- docs/source/examples/11_dictionaries.rst | 4 +- docs/source/examples/12_tuples.rst | 6 +- docs/source/examples/13_standard_classes.rst | 4 +- docs/source/examples/14_generics.rst | 4 +- .../examples/15_nesting_in_containers.rst | 4 +- .../examples/16_advanced_configuration.rst | 20 +- docs/source/examples/17_flax_modules.rst | 6 +- docs/source/goals_and_alternatives.md | 10 +- docs/source/helptext_generation.md | 4 +- docs/source/index.rst | 65 +++--- docs/source/installation.md | 8 +- docs/source/tab_completion.md | 12 +- docs/update_example_docs.py | 4 +- examples/01_functions.py | 6 +- examples/02_dataclasses.py | 6 +- examples/03_enums_and_containers.py | 4 +- examples/04_flags.py | 6 +- examples/05_hierarchical_configs.py | 6 +- examples/06_literals_and_unions.py | 4 +- examples/07_positional_args.py | 6 +- examples/08_subcommands.py | 8 +- examples/09_multiple_subcommands.py | 4 +- examples/10_base_configs.py | 20 +- examples/11_dictionaries.py | 4 +- examples/12_tuples.py | 6 +- examples/13_standard_classes.py | 4 +- examples/14_generics.py | 4 +- examples/15_nesting_in_containers.py | 4 +- examples/16_advanced_configuration.py | 20 +- examples/17_flax_modules.py | 6 +- examples/_rename_example.py | 4 +- pyproject.toml | 12 +- tests/test_collections.py | 198 ++++++++--------- tests/test_dcargs.py | 186 ++++++++-------- tests/test_dict_namedtuple.py | 84 ++++---- tests/test_dynamic_dataclasses.py | 6 +- tests/test_errors.py | 42 ++-- tests/test_flax_ignore_py310.py | 8 +- tests/test_forward_ref.py | 26 +-- tests/test_generics_and_serialization.py | 56 ++--- tests/test_helptext.py | 37 ++-- tests/test_is_nested_type.py | 2 +- tests/test_metadata.py | 94 ++++---- tests/test_missing.py | 32 +-- tests/test_nested.py | 200 +++++++++--------- tests/test_nested_in_containers.py | 86 ++++---- .../test_new_style_annotations_only_py310.py | 37 ++-- tests/test_positional_ignore_py37.py | 48 ++--- tests/test_print_completion.py | 8 +- tests/test_strings.py | 2 +- tests/test_union_from_mapping.py | 26 +-- tests/test_unsupported_but_should_work.py | 32 +-- {dcargs => tyro}/__init__.py | 0 {dcargs => tyro}/_argparse_formatter.py | 26 +-- {dcargs => tyro}/_arguments.py | 0 {dcargs => tyro}/_calling.py | 0 {dcargs => tyro}/_cli.py | 18 +- {dcargs => tyro}/_deprecated.py | 0 {dcargs => tyro}/_docstrings.py | 0 {dcargs => tyro}/_fields.py | 4 +- {dcargs => tyro}/_instantiators.py | 0 {dcargs => tyro}/_parsers.py | 2 +- {dcargs => tyro}/_resolver.py | 0 {dcargs => tyro}/_shtab/LICENCE | 0 {dcargs => tyro}/_shtab/README.md | 0 {dcargs => tyro}/_shtab/__init__.py | 0 {dcargs => tyro}/_shtab/__main__.py | 0 {dcargs => tyro}/_shtab/_dist_ver.py | 0 {dcargs => tyro}/_shtab/main.py | 0 {dcargs => tyro}/_shtab/py.typed | 0 {dcargs => tyro}/_singleton.py | 0 {dcargs => tyro}/_strings.py | 4 +- {dcargs => tyro}/conf/__init__.py | 2 +- {dcargs => tyro}/conf/_markers.py | 4 +- {dcargs => tyro}/conf/_subcommands.py | 6 +- {dcargs => tyro}/extras/__init__.py | 2 +- {dcargs => tyro}/extras/_base_configs.py | 12 +- {dcargs => tyro}/extras/_choices_type.py | 0 {dcargs => tyro}/extras/_serialization.py | 18 +- {dcargs => tyro}/py.typed | 0 95 files changed, 1052 insertions(+), 895 deletions(-) create mode 100644 docs/source/_static/logo-dark.svg create mode 100644 docs/source/_static/logo-light.svg rename {dcargs => tyro}/__init__.py (100%) rename {dcargs => tyro}/_argparse_formatter.py (96%) rename {dcargs => tyro}/_arguments.py (100%) rename {dcargs => tyro}/_calling.py (100%) rename {dcargs => tyro}/_cli.py (93%) rename {dcargs => tyro}/_deprecated.py (100%) rename {dcargs => tyro}/_docstrings.py (100%) rename {dcargs => tyro}/_fields.py (99%) rename {dcargs => tyro}/_instantiators.py (100%) rename {dcargs => tyro}/_parsers.py (99%) rename {dcargs => tyro}/_resolver.py (100%) rename {dcargs => tyro}/_shtab/LICENCE (100%) rename {dcargs => tyro}/_shtab/README.md (100%) rename {dcargs => tyro}/_shtab/__init__.py (100%) rename {dcargs => tyro}/_shtab/__main__.py (100%) rename {dcargs => tyro}/_shtab/_dist_ver.py (100%) rename {dcargs => tyro}/_shtab/main.py (100%) rename {dcargs => tyro}/_shtab/py.typed (100%) rename {dcargs => tyro}/_singleton.py (100%) rename {dcargs => tyro}/_strings.py (97%) rename {dcargs => tyro}/conf/__init__.py (89%) rename {dcargs => tyro}/conf/_markers.py (93%) rename {dcargs => tyro}/conf/_subcommands.py (91%) rename {dcargs => tyro}/extras/__init__.py (84%) rename {dcargs => tyro}/extras/_base_configs.py (84%) rename {dcargs => tyro}/extras/_choices_type.py (100%) rename {dcargs => tyro}/extras/_serialization.py (92%) rename {dcargs => tyro}/py.typed (100%) diff --git a/README.md b/README.md index 07c04092b..516510c71 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,86 @@ -

dcargs

+

+tyro logo +

-

- Documentation +

+ Documentation   •   - pip install dcargs + pip install tyro

-

- build - mypy - lint - - codecov + +

+ build + mypy + lint + + codecov - - codecov + + codecov

-

- dcargs is a library for typed CLI interfaces - and configuration objects. -

+
-

- Our core interface, dcargs.cli(), generates argument parsers from type-annotated -
callables: functions, dataclasses, classes, and nested dataclasses and classes. -

+tyro is a library for building CLI interfaces, +configuration objects, and configuration _systems_ with modern, type-annotated +Python. -

- This can be used as a replacement for argparse: -

+Our core interface consists of just one function, `tyro.cli()`, which translates +Python callables and types into fully-featured argument parsers and +configuration objects. + +To get started, we recommend visiting the examples in our +[documentation](https://brentyi.github.io/tyro). If you're familiar with +alternative libraries, we also include comparisons between [tyro and argparse]() +and [tyro and hydra](). + +### Why `tyro`? + +1. **Strong typing.** + + Unlike tools dependent on dictionaries, YAML, or dynamic namespaces, + arguments populated by `tyro` benefit from IDE and language server-supported + operations — think tab completion, rename, jump-to-def, docstrings on hover — + as well as static checking tools like `pyright` and `mypy`. + +2. **Minimal overhead.** + + Standard Python type annotations, docstrings, and default values are parsed + to automatically generate command-line interfaces with informative helptext. + + If you're familiar with type annotations and docstrings in Python, you + already know how to use `tyro`! If you're not, learning to use `tyro` reduces + to learning to write modern Python. + + Hate `tyro`? Just remove one line of code, and you're left with beautiful, + type-annotated, and documented vanilla Python that can be used with a range + of other configuration libraries. + +3. **Automatic helptext generation.** + + `tyro` parses docstrings, types, and defaults to generate carefully formatted + helptext. Longer messages are organized into responsive multi-column layouts. + +4. **Modularity.** + + `tyro` supports hierarchically nested configuration structures, which make it + easy to distribute definitions, defaults, and documentation of configurable + fields across modules or source files. + +5. **Tab completion.** + + By extending [shtab](https://github.com/iterative/shtab), `tyro` + automatically generates tab completion scripts for bash, zsh, and tcsh! + +### A minimal example + +As a replacement for `argparse`: - +
with argparsewith dcargswith tyro
@@ -64,32 +110,32 @@ print(args.a + args.b) ```python """Sum two numbers by calling a -function with dcargs.""" +function with tyro.""" -import dcargs +import tyro def main(a: int, b: int = 3) -> None: print(a + b) -dcargs.cli(main) +tyro.cli(main) ``` --- ```python """Sum two numbers by instantiating -a dataclass with dcargs.""" +a dataclass with tyro.""" from dataclasses import dataclass -import dcargs +import tyro @dataclass class Args: a: int b: int = 3 -args = dcargs.cli(Args) +args = tyro.cli(Args) print(args.a + args.b) ``` @@ -97,7 +143,17 @@ print(args.a + args.b)
-

- For more sophisticated examples, see - our documentation. -

+For more examples, see our [documentation](https://brentyi.github.io/tyro). + +### In the wild + +`tyro` is still a new library, but being stress tested in several projects! + +- [nerfstudio](https://github.com/nerfstudio-project/nerfstudio/) uses `tyro` + both to build compact command-line utilities and for YAML-free experiment + configuration. +- [obj2mjcf](https://github.com/kevinzakka/obj2mjcf) uses `tyro` to build a CLI + for processing composite Wavefront OBJ files for Mujoco. +- [tensorf-jax](https://github.com/brentyi/tensorf-jax/) implements + [Tensorial Radiance Fields](https://apchenstu.github.io/TensoRF/) in JAX, with + `tyro` for configuration. diff --git a/docs/source/_static/css/compact_table_header.css b/docs/source/_static/css/compact_table_header.css index 37af2d7c2..2861b5888 100644 --- a/docs/source/_static/css/compact_table_header.css +++ b/docs/source/_static/css/compact_table_header.css @@ -1,3 +1,11 @@ +.sidebar-logo { + max-width: 150px; + margin: 0; +} +.sidebar-brand-text { + display: None; +} + /* This stylesheet is currently overfit to the comparison table. Should be * revisited if we add more tables to the documentation. */ table.docutils thead, diff --git a/docs/source/_static/logo-dark.svg b/docs/source/_static/logo-dark.svg new file mode 100644 index 000000000..5143f41ac --- /dev/null +++ b/docs/source/_static/logo-dark.svg @@ -0,0 +1,55 @@ + + + + + + + + + + diff --git a/docs/source/_static/logo-light.svg b/docs/source/_static/logo-light.svg new file mode 100644 index 000000000..7ca6345f1 --- /dev/null +++ b/docs/source/_static/logo-light.svg @@ -0,0 +1,56 @@ + + + + + + + + + + diff --git a/docs/source/conf.py b/docs/source/conf.py index 5f3eaee4f..6ecf03022 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -19,7 +19,7 @@ # -- Project information ----------------------------------------------------- -project = "dcargs" +project = "tyro" copyright = "2022" author = "brentyi" @@ -57,6 +57,7 @@ ] programoutput_use_ansi = True html_ansi_stylesheet = "black-on-white.css" +html_static_path = ["_static"] html_theme_options = { "light_css_variables": { "color-code-background": "#f4f4f4", @@ -65,7 +66,7 @@ "footer_icons": [ { "name": "GitHub", - "url": "https://github.com/brentyi/dcargs", + "url": "https://github.com/brentyi/tyro", "html": """ @@ -74,6 +75,8 @@ "class": "", }, ], + "light_logo": "logo-light.svg", + "dark_logo": "logo-dark.svg", } # Pull documentation types from hints @@ -113,7 +116,7 @@ # a list of builtin themes. # html_theme = "furo" -html_title = "dcargs" +html_title = "tyro" # Theme options are theme-specific and customize the look and feel of a theme @@ -141,7 +144,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = "dcargs_doc" +htmlhelp_basename = "tyro_doc" # -- Options for Github output ------------------------------------------------ @@ -174,8 +177,8 @@ latex_documents = [ ( master_doc, - "dcargs.tex", - "dcargs", + "tyro.tex", + "tyro", "brentyi", "manual", ), @@ -186,7 +189,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [(master_doc, "dcargs", "dcargs documentation", [author], 1)] +man_pages = [(master_doc, "tyro", "tyro documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -197,11 +200,11 @@ texinfo_documents = [ ( master_doc, - "dcargs", - "dcargs", + "tyro", + "tyro", author, - "dcargs", - "dcargs", + "tyro", + "tyro", "Miscellaneous", ), ] @@ -210,7 +213,7 @@ # -- Extension configuration -------------------------------------------------- # -- Options for autoapi extension -------------------------------------------- -autoapi_dirs = ["../../dcargs"] +autoapi_dirs = ["../../tyro"] autoapi_root = "api" autoapi_options = [ "members", @@ -231,7 +234,7 @@ # name_alias = {} # # def recurse(module, prefixes): -# if hasattr(module, "__name__") and module.__name__.startswith("dcargs"): +# if hasattr(module, "__name__") and module.__name__.startswith("tyro"): # MAX_DEPTH = 5 # if len(prefixes) > MAX_DEPTH: # # Prevent infinite loops from cyclic imports @@ -240,15 +243,15 @@ # return # # for member_name in dir(module): -# if member_name == "dcargs": +# if member_name == "tyro": # continue # # member = getattr(module, member_name) # if callable(member): -# full_name = ".".join(["dcargs"] + prefixes + [member_name]) +# full_name = ".".join(["tyro"] + prefixes + [member_name]) # -# shortened_name = "dcargs" -# current = dcargs +# shortened_name = "tyro" +# current = tyro # success = True # for p in prefixes + [member_name]: # if p.startswith("_"): @@ -267,9 +270,9 @@ # elif not member_name.startswith("__"): # recurse(member, prefixes + [member_name]) # -# import dcargs +# import tyro # -# recurse(dcargs, prefixes=[]) +# recurse(tyro, prefixes=[]) # return name_alias # # diff --git a/docs/source/examples/01_functions.rst b/docs/source/examples/01_functions.rst index 324e58d32..3ac1a2e67 100644 --- a/docs/source/examples/01_functions.rst +++ b/docs/source/examples/01_functions.rst @@ -5,7 +5,7 @@ ========================================== -In the simplest case, ``dcargs.cli()`` can be used to run a function with arguments +In the simplest case, ``tyro.cli()`` can be used to run a function with arguments populated from the CLI. @@ -14,7 +14,7 @@ populated from the CLI. :linenos: - import dcargs + import tyro def main( @@ -31,7 +31,7 @@ populated from the CLI. if __name__ == "__main__": - dcargs.cli(main) + tyro.cli(main) ------------ diff --git a/docs/source/examples/02_dataclasses.rst b/docs/source/examples/02_dataclasses.rst index f8c7c525e..ded2a0523 100644 --- a/docs/source/examples/02_dataclasses.rst +++ b/docs/source/examples/02_dataclasses.rst @@ -5,7 +5,7 @@ ========================================== -Common pattern: use ``dcargs.cli()`` to instantiate a dataclass. The outputted instance +Common pattern: use ``tyro.cli()`` to instantiate a dataclass. The outputted instance can be used as a typed alternative for an argparse namespace. @@ -16,7 +16,7 @@ can be used as a typed alternative for an argparse namespace. import dataclasses - import dcargs + import tyro @dataclasses.dataclass @@ -29,7 +29,7 @@ can be used as a typed alternative for an argparse namespace. if __name__ == "__main__": - args = dcargs.cli(Args) + args = tyro.cli(Args) print(args) ------------ diff --git a/docs/source/examples/03_enums_and_containers.rst b/docs/source/examples/03_enums_and_containers.rst index 45fd7da3e..1a4b56c6d 100644 --- a/docs/source/examples/03_enums_and_containers.rst +++ b/docs/source/examples/03_enums_and_containers.rst @@ -19,7 +19,7 @@ container types. import pathlib from typing import Optional, Tuple - import dcargs + import tyro class OptimizerType(enum.Enum): @@ -48,7 +48,7 @@ container types. if __name__ == "__main__": - config = dcargs.cli(TrainConfig) + config = tyro.cli(TrainConfig) print(config) ------------ diff --git a/docs/source/examples/04_flags.rst b/docs/source/examples/04_flags.rst index 30c217882..2dc23d0bb 100644 --- a/docs/source/examples/04_flags.rst +++ b/docs/source/examples/04_flags.rst @@ -8,7 +8,7 @@ Booleans can either be expected to be explicitly passed in, or, if given a default value, automatically converted to flags. -To turn off conversion, see :class:`dcargs.conf.FlagConversionOff`. +To turn off conversion, see :class:`tyro.conf.FlagConversionOff`. @@ -19,7 +19,7 @@ To turn off conversion, see :class:`dcargs.conf.FlagConversionOff`. import dataclasses from typing import Optional - import dcargs + import tyro @dataclasses.dataclass @@ -38,7 +38,7 @@ To turn off conversion, see :class:`dcargs.conf.FlagConversionOff`. if __name__ == "__main__": - args = dcargs.cli(Args) + args = tyro.cli(Args) print(args) ------------ diff --git a/docs/source/examples/05_hierarchical_configs.rst b/docs/source/examples/05_hierarchical_configs.rst index f05df0c19..775fe458f 100644 --- a/docs/source/examples/05_hierarchical_configs.rst +++ b/docs/source/examples/05_hierarchical_configs.rst @@ -18,7 +18,7 @@ objects. This helps with modularity and grouping in larger projects. import enum import pathlib - import dcargs + import tyro class OptimizerType(enum.Enum): @@ -72,11 +72,11 @@ objects. This helps with modularity and grouping in larger projects. print() print(f"{config=}") print() - print(dcargs.to_yaml(config)) + print(tyro.to_yaml(config)) if __name__ == "__main__": - dcargs.cli(train) + tyro.cli(train) ------------ diff --git a/docs/source/examples/06_literals_and_unions.rst b/docs/source/examples/06_literals_and_unions.rst index e85d02cc2..9031a1c7e 100644 --- a/docs/source/examples/06_literals_and_unions.rst +++ b/docs/source/examples/06_literals_and_unions.rst @@ -18,7 +18,7 @@ import enum from typing import Literal, Optional, Tuple, Union - import dcargs + import tyro class Color(enum.Enum): @@ -54,7 +54,7 @@ if __name__ == "__main__": - args = dcargs.cli(Args) + args = tyro.cli(Args) print(args) ------------ diff --git a/docs/source/examples/07_positional_args.rst b/docs/source/examples/07_positional_args.rst index 7bf7cd260..b618756dd 100644 --- a/docs/source/examples/07_positional_args.rst +++ b/docs/source/examples/07_positional_args.rst @@ -7,7 +7,7 @@ Positional-only arguments in functions are converted to positional CLI arguments. -For more general positional arguments, see :func:`dcargs.conf.Positional`. +For more general positional arguments, see :func:`tyro.conf.Positional`. @@ -22,7 +22,7 @@ For more general positional arguments, see :func:`dcargs.conf.Positional`. import pathlib from typing import Tuple - import dcargs + import tyro def main( @@ -66,7 +66,7 @@ For more general positional arguments, see :func:`dcargs.conf.Positional`. if __name__ == "__main__": - dcargs.cli(main) + tyro.cli(main) ------------ diff --git a/docs/source/examples/08_subcommands.rst b/docs/source/examples/08_subcommands.rst index eae5801e5..77bc6a108 100644 --- a/docs/source/examples/08_subcommands.rst +++ b/docs/source/examples/08_subcommands.rst @@ -8,7 +8,7 @@ Unions over nested types (classes or dataclasses) are populated using subcommands. For configuring subcommands beyond what can be expressed with type annotations, see -:func:`dcargs.conf.subcommand()`. +:func:`tyro.conf.subcommand()`. @@ -21,7 +21,7 @@ For configuring subcommands beyond what can be expressed with type annotations, import dataclasses from typing import Union - import dcargs + import tyro @dataclasses.dataclass(frozen=True) @@ -45,9 +45,9 @@ For configuring subcommands beyond what can be expressed with type annotations, if __name__ == "__main__": # Note that we can also pass `Union[Checkout, Command]` directly into - # `dcargs.cli()`; this is understood by dcargs and pyright, but unfortunately not by + # `tyro.cli()`; this is understood by tyro and pyright, but unfortunately not by # mypy. - dcargs.cli(main) + tyro.cli(main) ------------ diff --git a/docs/source/examples/09_multiple_subcommands.rst b/docs/source/examples/09_multiple_subcommands.rst index 7522d6a8c..41e4d416c 100644 --- a/docs/source/examples/09_multiple_subcommands.rst +++ b/docs/source/examples/09_multiple_subcommands.rst @@ -18,7 +18,7 @@ Multiple unions over nested types are populated using a series of subcommands. import dataclasses from typing import Literal, Tuple, Union - import dcargs + import tyro # Possible dataset configurations. @@ -70,7 +70,7 @@ Multiple unions over nested types are populated using a series of subcommands. if __name__ == "__main__": - dcargs.cli(train) + tyro.cli(train) ------------ diff --git a/docs/source/examples/10_base_configs.rst b/docs/source/examples/10_base_configs.rst index e3135ac79..d1b905bf7 100644 --- a/docs/source/examples/10_base_configs.rst +++ b/docs/source/examples/10_base_configs.rst @@ -5,7 +5,7 @@ ========================================== -We can integrate ``dcargs.cli()`` into common configuration patterns: here, we select +We can integrate ``tyro.cli()`` into common configuration patterns: here, we select one of multiple possible base configurations, create a subcommand for each one, and then use the CLI to either override (existing) or fill in (missing) values. @@ -26,7 +26,7 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. from torch import nn - import dcargs + import tyro @dataclass(frozen=True) @@ -80,9 +80,9 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. num_layers=4, units=64, train_steps=30_000, - # The dcargs.MISSING sentinel allows us to specify that the seed should have no + # The tyro.MISSING sentinel allows us to specify that the seed should have no # default, and needs to be populated from the CLI. - seed=dcargs.MISSING, + seed=tyro.MISSING, activation=nn.ReLU, ) @@ -95,22 +95,22 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. num_layers=8, units=256, train_steps=100_000, - seed=dcargs.MISSING, + seed=tyro.MISSING, activation=nn.GELU, ) if __name__ == "__main__": - config = dcargs.cli( - dcargs.extras.subcommand_type_from_defaults(base_configs, descriptions), + config = tyro.cli( + tyro.extras.subcommand_type_from_defaults(base_configs, descriptions), ) # ^Note that this is equivalent to: # - # config = dcargs.cli( + # config = tyro.cli( # Union[ # Annotated[ # ExperimentConfig, - # dcargs.conf.subcommand( + # tyro.conf.subcommand( # "small", # default=base_configs["small"], # description=descriptions["small"], @@ -118,7 +118,7 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. # ], # Annotated[ # ExperimentConfig, - # dcargs.conf.subcommand( + # tyro.conf.subcommand( # "big", # default=base_configs["big"], # description=descriptions["big"], diff --git a/docs/source/examples/11_dictionaries.rst b/docs/source/examples/11_dictionaries.rst index a5092ed5a..d063b2b5c 100644 --- a/docs/source/examples/11_dictionaries.rst +++ b/docs/source/examples/11_dictionaries.rst @@ -18,7 +18,7 @@ or a ``TypedDict`` subclass. from frozendict import frozendict # type: ignore - import dcargs + import tyro class DictionarySchema( @@ -53,7 +53,7 @@ or a ``TypedDict`` subclass. if __name__ == "__main__": - dcargs.cli(main) + tyro.cli(main) ------------ diff --git a/docs/source/examples/12_tuples.rst b/docs/source/examples/12_tuples.rst index 46381befa..719e699f1 100644 --- a/docs/source/examples/12_tuples.rst +++ b/docs/source/examples/12_tuples.rst @@ -5,7 +5,7 @@ ========================================== -Example using ``dcargs.cli()`` to instantiate tuple types. +Example using ``tyro.cli()`` to instantiate tuple types. @@ -15,7 +15,7 @@ Example using ``dcargs.cli()`` to instantiate tuple types. from typing import NamedTuple, Tuple - import dcargs + import tyro # Named tuples are interpreted as nested structures. @@ -37,7 +37,7 @@ Example using ``dcargs.cli()`` to instantiate tuple types. if __name__ == "__main__": - x = dcargs.cli(TupleType) + x = tyro.cli(TupleType) assert isinstance(x, tuple) print(x) diff --git a/docs/source/examples/13_standard_classes.rst b/docs/source/examples/13_standard_classes.rst index 9e0da5d03..d0e350c39 100644 --- a/docs/source/examples/13_standard_classes.rst +++ b/docs/source/examples/13_standard_classes.rst @@ -14,7 +14,7 @@ constructors of) standard Python classes. :linenos: - import dcargs + import tyro class Args: @@ -35,7 +35,7 @@ constructors of) standard Python classes. if __name__ == "__main__": - args = dcargs.cli(Args) + args = tyro.cli(Args) print(args.data) ------------ diff --git a/docs/source/examples/14_generics.rst b/docs/source/examples/14_generics.rst index 48145e493..627b4a656 100644 --- a/docs/source/examples/14_generics.rst +++ b/docs/source/examples/14_generics.rst @@ -16,7 +16,7 @@ Example of parsing for generic dataclasses. import dataclasses from typing import Generic, TypeVar - import dcargs + import tyro ScalarType = TypeVar("ScalarType", int, float) ShapeType = TypeVar("ShapeType") @@ -43,7 +43,7 @@ Example of parsing for generic dataclasses. if __name__ == "__main__": - args = dcargs.cli(Args[Triangle]) + args = tyro.cli(Args[Triangle]) print(args) ------------ diff --git a/docs/source/examples/15_nesting_in_containers.rst b/docs/source/examples/15_nesting_in_containers.rst index 148536f4a..13c2a2e09 100644 --- a/docs/source/examples/15_nesting_in_containers.rst +++ b/docs/source/examples/15_nesting_in_containers.rst @@ -19,7 +19,7 @@ parsing default values. import dataclasses from typing import Dict, Tuple - import dcargs + import tyro class Color: @@ -63,7 +63,7 @@ parsing default values. if __name__ == "__main__": - args = dcargs.cli(Args) + args = tyro.cli(Args) print(args) ------------ diff --git a/docs/source/examples/16_advanced_configuration.rst b/docs/source/examples/16_advanced_configuration.rst index 55fde3935..f5ce74d22 100644 --- a/docs/source/examples/16_advanced_configuration.rst +++ b/docs/source/examples/16_advanced_configuration.rst @@ -5,7 +5,7 @@ ========================================== -The :mod:`dcargs.conf` module contains utilities that can be used to configure +The :mod:`tyro.conf` module contains utilities that can be used to configure command-line interfaces beyond what is expressible via static type annotations. Features here are supported, but generally unnecessary and should be used sparingly. @@ -21,7 +21,7 @@ Features here are supported, but generally unnecessary and should be used sparin from typing_extensions import Annotated - import dcargs + import tyro @dataclasses.dataclass(frozen=True) @@ -42,32 +42,32 @@ Features here are supported, but generally unnecessary and should be used sparin @dataclasses.dataclass class Args: # A boolean field with flag conversion turned off. - boolean: dcargs.conf.FlagConversionOff[bool] = False + boolean: tyro.conf.FlagConversionOff[bool] = False # A numeric field parsed as a positional argument. - positional: dcargs.conf.Positional[int] = 3 + positional: tyro.conf.Positional[int] = 3 # A numeric field that can't be changed via the CLI. - fixed: dcargs.conf.Fixed[int] = 5 + fixed: tyro.conf.Fixed[int] = 5 # A union over nested structures, but without subcommand generation. When a default # is provided, the type is simply fixed to that default. - union_without_subcommand: dcargs.conf.AvoidSubcommands[ + union_without_subcommand: tyro.conf.AvoidSubcommands[ Union[CheckoutArgs, CommitArgs] ] = CheckoutArgs("main") - # `dcargs.conf.subcommand()` can be used to configure subcommands in a Union. Here, + # `tyro.conf.subcommand()` can be used to configure subcommands in a Union. Here, # we make the subcommand names more succinct. renamed_subcommand: Union[ Annotated[ - CheckoutArgs, dcargs.conf.subcommand(name="checkout", prefix_name=False) + CheckoutArgs, tyro.conf.subcommand(name="checkout", prefix_name=False) ], - Annotated[CommitArgs, dcargs.conf.subcommand(name="commit", prefix_name=False)], + Annotated[CommitArgs, tyro.conf.subcommand(name="commit", prefix_name=False)], ] = CheckoutArgs("main") if __name__ == "__main__": - print(dcargs.cli(Args)) + print(tyro.cli(Args)) ------------ diff --git a/docs/source/examples/17_flax_modules.rst b/docs/source/examples/17_flax_modules.rst index 64b6ef71d..c93ffd3b8 100644 --- a/docs/source/examples/17_flax_modules.rst +++ b/docs/source/examples/17_flax_modules.rst @@ -6,7 +6,7 @@ If you use `Flax `_\ , modules can be instantiated -directly from ``dcargs.cli``. +directly from ``tyro.cli``. @@ -17,7 +17,7 @@ directly from ``dcargs.cli``. from flax import linen as nn from jax import numpy as jnp - import dcargs + import tyro class Classifier(nn.Module): @@ -57,7 +57,7 @@ directly from ``dcargs.cli``. if __name__ == "__main__": - dcargs.cli(train) + tyro.cli(train) ------------ diff --git a/docs/source/goals_and_alternatives.md b/docs/source/goals_and_alternatives.md index e1667ac85..900172b9a 100644 --- a/docs/source/goals_and_alternatives.md +++ b/docs/source/goals_and_alternatives.md @@ -2,12 +2,12 @@ ## Design goals -The core functionality of `dcargs` — generating argument parsers from type +The core functionality of `tyro` — generating argument parsers from type annotations — overlaps significantly with features offered by other libraries. Usage distinctions are the result of two API goals: - **One uninvasive function.** For all core functionality, learning to use - `dcargs` should reduce to learning to write (type-annotated) Python. For + `tyro` should reduce to learning to write (type-annotated) Python. For example, types are specified using standard annotations, helptext using docstrings, choices using the standard `typing.Literal` type, subcommands with `typing.Union` of nested types, and positional arguments with `/`. @@ -37,7 +37,7 @@ More concretely, we can also compare specific features. A noncomprehensive set: | [pyrallis][pyrallis] | ✓ | | | ✓ | ✓ | | | ✓ | | | | [yahp][yahp] | ✓ | | | ~[^yahp_docstrings] | ✓ | ✓ | ~[^yahp_unions_nested] | ✓ | | | | [omegaconf][omegaconf] | ✓ | | | | ✓ | | | ✓ | ✓ | | -| **dcargs** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| **tyro** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | @@ -62,7 +62,7 @@ More concretely, we can also compare specific features. A noncomprehensive set: [^simp_literals]: Not supported for mixed (eg `Literal[5, "five"]`) or in container (eg `List[Literal[1, 2]]`) types. [^datargs_literals]: Not supported for mixed types (eg `Literal[5, "five"]`). [^typer_containers]: `typer` uses positional arguments for all required fields, which means that only one variable-length argument (such as `List[int]`) without a default is supported per argument parser. -[^yahp_docstrings]: Via the `hp.auto()` function, which can parse docstrings from external classes. Usage is different from the more direct parsing that `dcargs`, `tap`, and `simple-parsing`/`pyrallis` support. +[^yahp_docstrings]: Via the `hp.auto()` function, which can parse docstrings from external classes. Usage is different from the more direct parsing that `tyro`, `tap`, and `simple-parsing`/`pyrallis` support. @@ -86,7 +86,7 @@ integration, which include syntax and logic for **(1)** defining configurable fields, **(2)** saving and choosing between a set of base configurations, and **(3)** overriding configurable values at the command-line. -`dcargs` is meant to take advantage of modern Python features for **(1)** and +`tyro` is meant to take advantage of modern Python features for **(1)** and focus completely on **(3)** in a way that's agnostic to a project's preferred approach for **(2)**. **(2)** is left as an exercise to the user; it tends to be the most open-ended and varied in terms of project and personal preferences, but diff --git a/docs/source/helptext_generation.md b/docs/source/helptext_generation.md index f341e59d9..8b46e5882 100644 --- a/docs/source/helptext_generation.md +++ b/docs/source/helptext_generation.md @@ -1,6 +1,6 @@ # Helptext generation -In addition to type annotations, :func:`dcargs.cli()` will also parse docstrings +In addition to type annotations, :func:`tyro.cli()` will also parse docstrings and comments. These are used to automatically generate helptext; see examples for how these end up being formatted. @@ -30,7 +30,7 @@ def main( For types defined using class attributes, enumerating each argument list in the class docstring can be cumbersome. -If they are unavailable, :func:`dcargs.cli` will generate helptext from +If they are unavailable, :func:`tyro.cli` will generate helptext from docstrings and comments on attributes. These are parsed via source code inspection. diff --git a/docs/source/index.rst b/docs/source/index.rst index 0ab9f4429..23604bab4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,11 +1,11 @@ -dcargs +tyro ========================================== |build| |nbsp| |mypy| |nbsp| |lint| |nbsp| |coverage| |nbsp| |versions| -:code:`dcargs` is a library for typed CLI interfaces and configuration objects. +:code:`tyro` is a library for typed CLI interfaces and configuration objects. -Our core interface, :func:`dcargs.cli()`, generates argument parsers from +Our core interface, :func:`tyro.cli()`, generates argument parsers from type-annotated callables: functions, classes, dataclasses, and *nested* dataclasses and classes. @@ -28,30 +28,30 @@ This can be used as a replacement for :code:`argparse`: .. code-block:: - """Sum two numbers by calling a function with dcargs.""" + """Sum two numbers by calling a function with tyro.""" - import dcargs + import tyro def main(a: int, b: int = 3) -> None: print(a + b) - dcargs.cli(main) + tyro.cli(main) .. code-block:: - """Sum two numbers by instantiating a dataclass with dcargs.""" + """Sum two numbers by instantiating a dataclass with tyro.""" from dataclasses import dataclass - import dcargs + import tyro @dataclass class Args: a: int b: int = 3 - args = dcargs.cli(Args) + args = tyro.cli(Args) print(args.a + args.b) @@ -62,7 +62,7 @@ The broader goal is also a replacement for tools like :code:`hydra`, values are parsed to automatically generate command-line interfaces with informative helptext. -- **Expressive.** :func:`dcargs.cli` understands functions, classes, +- **Expressive.** :func:`tyro.cli` understands functions, classes, dataclasses, and *nested* classes and dataclasses, as well as frequently used annotations like unions, literals, and collections, which can be composed into hierarchical configuration objects built on standard Python features. @@ -90,15 +90,6 @@ The broader goal is also a replacement for tools like :code:`hydra`, installation -.. toctree:: - :caption: Examples - :maxdepth: 1 - :hidden: - :titlesonly: - :glob: - - examples/* - .. toctree:: :caption: Notes :maxdepth: 5 @@ -109,34 +100,42 @@ The broader goal is also a replacement for tools like :code:`hydra`, tab_completion goals_and_alternatives - .. toctree:: :caption: API Reference :maxdepth: 5 :hidden: :titlesonly: - api/dcargs/index + api/tyro/index + +.. toctree:: + :caption: Examples + :maxdepth: 1 + :hidden: + :titlesonly: + :glob: + + examples/* -.. |build| image:: https://github.com/brentyi/dcargs/workflows/build/badge.svg +.. |build| image:: https://github.com/brentyi/tyro/workflows/build/badge.svg :alt: Build status icon - :target: https://github.com/brentyi/dcargs -.. |mypy| image:: https://github.com/brentyi/dcargs/workflows/mypy/badge.svg?branch=master + :target: https://github.com/brentyi/tyro +.. |mypy| image:: https://github.com/brentyi/tyro/workflows/mypy/badge.svg?branch=master :alt: Mypy status icon - :target: https://github.com/brentyi/dcargs -.. |lint| image:: https://github.com/brentyi/dcargs/workflows/lint/badge.svg + :target: https://github.com/brentyi/tyro +.. |lint| image:: https://github.com/brentyi/tyro/workflows/lint/badge.svg :alt: Lint status icon - :target: https://github.com/brentyi/dcargs -.. |coverage| image:: https://codecov.io/gh/brentyi/dcargs/branch/master/graph/badge.svg + :target: https://github.com/brentyi/tyro +.. |coverage| image:: https://codecov.io/gh/brentyi/tyro/branch/master/graph/badge.svg :alt: Test coverage status icon - :target: https://codecov.io/gh/brentyi/dcargs -.. |downloads| image:: https://pepy.tech/badge/dcargs + :target: https://codecov.io/gh/brentyi/tyro +.. |downloads| image:: https://pepy.tech/badge/tyro :alt: Download count icon - :target: https://pypi.org/project/dcargs/ -.. |versions| image:: https://img.shields.io/pypi/pyversions/dcargs + :target: https://pypi.org/project/tyro/ +.. |versions| image:: https://img.shields.io/pypi/pyversions/tyro :alt: Version icon - :target: https://pypi.org/project/dcargs/ + :target: https://pypi.org/project/tyro/ .. |nbsp| unicode:: 0xA0 :trim: diff --git a/docs/source/installation.md b/docs/source/installation.md index efe0d88fc..ad8b0e4b3 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -6,18 +6,18 @@ Installation is supported on Python >=3.7 via pip. This is typically all that's required. ```bash -pip install dcargs +pip install tyro ``` ## Development -If you're interested in development, the recommended way to install `dcargs` is +If you're interested in development, the recommended way to install `tyro` is via [poetry](https://github.com/python-poetry/poetry). ```bash # Clone repository and install. -git clone git@github.com:brentyi/dcargs.git -cd dcargs +git clone git@github.com:brentyi/tyro.git +cd tyro poetry install # Run tests. diff --git a/docs/source/tab_completion.md b/docs/source/tab_completion.md index 559e693d5..14153cf3c 100644 --- a/docs/source/tab_completion.md +++ b/docs/source/tab_completion.md @@ -1,10 +1,10 @@ # Tab completion -Interfaces built with :func:`dcargs.cli()` can be tab completed in interactive +Interfaces built with :func:`tyro.cli()` can be tab completed in interactive shells without any source code modification. Completion scripts can be generated by passing the -`--dcargs-print-completion {bash/zsh/tcsh}` flag to a dcargs CLI. This generates +`--tyro-print-completion {bash/zsh/tcsh}` flag to a tyro CLI. This generates a completion script via [shtab](https://docs.iterative.ai/shtab/) and prints it to stdout. To set up tab completion, the printed script simply needs to be written somewhere where your shell will find it. @@ -16,14 +16,14 @@ For zsh, one option is to emulate the pattern used for completions in ```bash # Set up zsh autocompletion for 01_functions.py, which is located in -# dcargs/examples. +# tyro/examples. # (1) Make directory for local completions. mkdir -p ~/.zfunc # (2) Write completion script. The name here (_01_functions_py) doesn't matter, # as long as it's prefixed with an underscore. -python 01_functions.py --dcargs-print-completion zsh > ~/.zfunc/_01_functions_py +python 01_functions.py --tyro-print-completion zsh > ~/.zfunc/_01_functions_py ``` And if it's not in your `.zshrc` already: @@ -58,7 +58,7 @@ Borrowing from the `bash-completion` source[^bash_completion], we can run: ```bash # Set up bash autocompletion for 01_functions.py, which is located in -# dcargs/examples. +# tyro/examples. # (1) Find and make completion directory. completion_dir=${BASH_COMPLETION_USER_DIR:-${XDG_DATA_HOME:-$HOME/.local/share}/bash-completion}/completions/ @@ -66,7 +66,7 @@ mkdir -p $completion_dir # (2) Write completion scripts. Note that the name of the completion script must # match the name of the file. -python 01_functions.py --dcargs-print-completion bash > ${completion_dir}/01_functions.py +python 01_functions.py --tyro-print-completion bash > ${completion_dir}/01_functions.py ``` In contrast to zsh, tab completion in bash requires that scripts are either set diff --git a/docs/update_example_docs.py b/docs/update_example_docs.py index c026dd892..05e06b929 100644 --- a/docs/update_example_docs.py +++ b/docs/update_example_docs.py @@ -10,7 +10,7 @@ import m2r2 -import dcargs +import tyro @dataclasses.dataclass @@ -137,4 +137,4 @@ def main( if __name__ == "__main__": - dcargs.cli(main, description=__doc__) + tyro.cli(main, description=__doc__) diff --git a/examples/01_functions.py b/examples/01_functions.py index c51af61f8..9d4d7048b 100755 --- a/examples/01_functions.py +++ b/examples/01_functions.py @@ -1,4 +1,4 @@ -"""In the simplest case, `dcargs.cli()` can be used to run a function with arguments +"""In the simplest case, `tyro.cli()` can be used to run a function with arguments populated from the CLI. Usage: @@ -7,7 +7,7 @@ `python ./01_functions.py --field1 hello --field2 10` """ -import dcargs +import tyro def main( @@ -24,4 +24,4 @@ def main( if __name__ == "__main__": - dcargs.cli(main) + tyro.cli(main) diff --git a/examples/02_dataclasses.py b/examples/02_dataclasses.py index 4993138bc..2958fa7ee 100644 --- a/examples/02_dataclasses.py +++ b/examples/02_dataclasses.py @@ -1,4 +1,4 @@ -"""Common pattern: use `dcargs.cli()` to instantiate a dataclass. The outputted instance +"""Common pattern: use `tyro.cli()` to instantiate a dataclass. The outputted instance can be used as a typed alternative for an argparse namespace. Usage: @@ -9,7 +9,7 @@ import dataclasses -import dcargs +import tyro @dataclasses.dataclass @@ -22,5 +22,5 @@ class Args: if __name__ == "__main__": - args = dcargs.cli(Args) + args = tyro.cli(Args) print(args) diff --git a/examples/03_enums_and_containers.py b/examples/03_enums_and_containers.py index fe0facba4..d7c7a3a5b 100644 --- a/examples/03_enums_and_containers.py +++ b/examples/03_enums_and_containers.py @@ -12,7 +12,7 @@ import pathlib from typing import Optional, Tuple -import dcargs +import tyro class OptimizerType(enum.Enum): @@ -41,5 +41,5 @@ class TrainConfig: if __name__ == "__main__": - config = dcargs.cli(TrainConfig) + config = tyro.cli(TrainConfig) print(config) diff --git a/examples/04_flags.py b/examples/04_flags.py index 3df1d0c7e..a19244acc 100644 --- a/examples/04_flags.py +++ b/examples/04_flags.py @@ -1,7 +1,7 @@ """Booleans can either be expected to be explicitly passed in, or, if given a default value, automatically converted to flags. -To turn off conversion, see :class:`dcargs.conf.FlagConversionOff`. +To turn off conversion, see :class:`tyro.conf.FlagConversionOff`. Usage: `python ./04_flags.py --help` @@ -13,7 +13,7 @@ import dataclasses from typing import Optional -import dcargs +import tyro @dataclasses.dataclass @@ -32,5 +32,5 @@ class Args: if __name__ == "__main__": - args = dcargs.cli(Args) + args = tyro.cli(Args) print(args) diff --git a/examples/05_hierarchical_configs.py b/examples/05_hierarchical_configs.py index c0a1be98c..69d4429a5 100644 --- a/examples/05_hierarchical_configs.py +++ b/examples/05_hierarchical_configs.py @@ -11,7 +11,7 @@ import enum import pathlib -import dcargs +import tyro class OptimizerType(enum.Enum): @@ -65,8 +65,8 @@ def train( print() print(f"{config=}") print() - print(dcargs.to_yaml(config)) + print(tyro.to_yaml(config)) if __name__ == "__main__": - dcargs.cli(train) + tyro.cli(train) diff --git a/examples/06_literals_and_unions.py b/examples/06_literals_and_unions.py index a46fff65c..d3fc32c1a 100644 --- a/examples/06_literals_and_unions.py +++ b/examples/06_literals_and_unions.py @@ -9,7 +9,7 @@ import enum from typing import Literal, Optional, Tuple, Union -import dcargs +import tyro class Color(enum.Enum): @@ -45,5 +45,5 @@ class Args: if __name__ == "__main__": - args = dcargs.cli(Args) + args = tyro.cli(Args) print(args) diff --git a/examples/07_positional_args.py b/examples/07_positional_args.py index fc81f7126..63902e1f7 100644 --- a/examples/07_positional_args.py +++ b/examples/07_positional_args.py @@ -1,6 +1,6 @@ """Positional-only arguments in functions are converted to positional CLI arguments. -For more general positional arguments, see :func:`dcargs.conf.Positional`. +For more general positional arguments, see :func:`tyro.conf.Positional`. Usage: `python ./07_positional_args.py --help` @@ -14,7 +14,7 @@ import pathlib from typing import Tuple -import dcargs +import tyro def main( @@ -58,4 +58,4 @@ class OptimizerConfig: if __name__ == "__main__": - dcargs.cli(main) + tyro.cli(main) diff --git a/examples/08_subcommands.py b/examples/08_subcommands.py index 7ae6b5db9..bf8bb9616 100644 --- a/examples/08_subcommands.py +++ b/examples/08_subcommands.py @@ -1,7 +1,7 @@ """Unions over nested types (classes or dataclasses) are populated using subcommands. For configuring subcommands beyond what can be expressed with type annotations, see -:func:`dcargs.conf.subcommand()`. +:func:`tyro.conf.subcommand()`. Usage: `python ./08_subcommands.py --help` @@ -16,7 +16,7 @@ import dataclasses from typing import Union -import dcargs +import tyro @dataclasses.dataclass(frozen=True) @@ -40,6 +40,6 @@ def main(cmd: Union[Checkout, Commit]) -> None: if __name__ == "__main__": # Note that we can also pass `Union[Checkout, Command]` directly into - # `dcargs.cli()`; this is understood by dcargs and pyright, but unfortunately not by + # `tyro.cli()`; this is understood by tyro and pyright, but unfortunately not by # mypy. - dcargs.cli(main) + tyro.cli(main) diff --git a/examples/09_multiple_subcommands.py b/examples/09_multiple_subcommands.py index cede20a18..3b689559f 100644 --- a/examples/09_multiple_subcommands.py +++ b/examples/09_multiple_subcommands.py @@ -11,7 +11,7 @@ import dataclasses from typing import Literal, Tuple, Union -import dcargs +import tyro # Possible dataset configurations. @@ -63,4 +63,4 @@ def train( if __name__ == "__main__": - dcargs.cli(train) + tyro.cli(train) diff --git a/examples/10_base_configs.py b/examples/10_base_configs.py index b994bac60..2b2273a09 100644 --- a/examples/10_base_configs.py +++ b/examples/10_base_configs.py @@ -1,4 +1,4 @@ -"""We can integrate `dcargs.cli()` into common configuration patterns: here, we select +"""We can integrate `tyro.cli()` into common configuration patterns: here, we select one of multiple possible base configurations, create a subcommand for each one, and then use the CLI to either override (existing) or fill in (missing) values. @@ -21,7 +21,7 @@ from torch import nn -import dcargs +import tyro @dataclass(frozen=True) @@ -75,9 +75,9 @@ class ExperimentConfig: num_layers=4, units=64, train_steps=30_000, - # The dcargs.MISSING sentinel allows us to specify that the seed should have no + # The tyro.MISSING sentinel allows us to specify that the seed should have no # default, and needs to be populated from the CLI. - seed=dcargs.MISSING, + seed=tyro.MISSING, activation=nn.ReLU, ) @@ -90,22 +90,22 @@ class ExperimentConfig: num_layers=8, units=256, train_steps=100_000, - seed=dcargs.MISSING, + seed=tyro.MISSING, activation=nn.GELU, ) if __name__ == "__main__": - config = dcargs.cli( - dcargs.extras.subcommand_type_from_defaults(base_configs, descriptions), + config = tyro.cli( + tyro.extras.subcommand_type_from_defaults(base_configs, descriptions), ) # ^Note that this is equivalent to: # - # config = dcargs.cli( + # config = tyro.cli( # Union[ # Annotated[ # ExperimentConfig, - # dcargs.conf.subcommand( + # tyro.conf.subcommand( # "small", # default=base_configs["small"], # description=descriptions["small"], @@ -113,7 +113,7 @@ class ExperimentConfig: # ], # Annotated[ # ExperimentConfig, - # dcargs.conf.subcommand( + # tyro.conf.subcommand( # "big", # default=base_configs["big"], # description=descriptions["big"], diff --git a/examples/11_dictionaries.py b/examples/11_dictionaries.py index 617622c09..3534c2a9e 100644 --- a/examples/11_dictionaries.py +++ b/examples/11_dictionaries.py @@ -11,7 +11,7 @@ from frozendict import frozendict # type: ignore -import dcargs +import tyro class DictionarySchema( @@ -46,4 +46,4 @@ def main( if __name__ == "__main__": - dcargs.cli(main) + tyro.cli(main) diff --git a/examples/12_tuples.py b/examples/12_tuples.py index f85f4f884..214e18303 100644 --- a/examples/12_tuples.py +++ b/examples/12_tuples.py @@ -1,4 +1,4 @@ -"""Example using `dcargs.cli()` to instantiate tuple types. +"""Example using `tyro.cli()` to instantiate tuple types. Usage: `python ./12_tuples.py --help` @@ -8,7 +8,7 @@ from typing import NamedTuple, Tuple -import dcargs +import tyro # Named tuples are interpreted as nested structures. @@ -30,6 +30,6 @@ class TupleType(NamedTuple): if __name__ == "__main__": - x = dcargs.cli(TupleType) + x = tyro.cli(TupleType) assert isinstance(x, tuple) print(x) diff --git a/examples/13_standard_classes.py b/examples/13_standard_classes.py index c4d942e45..b276337a9 100644 --- a/examples/13_standard_classes.py +++ b/examples/13_standard_classes.py @@ -6,7 +6,7 @@ `python ./13_standard_classes.py --field1 hello --field2 7` """ -import dcargs +import tyro class Args: @@ -27,5 +27,5 @@ def __init__( if __name__ == "__main__": - args = dcargs.cli(Args) + args = tyro.cli(Args) print(args.data) diff --git a/examples/14_generics.py b/examples/14_generics.py index 2450cd7d5..9e2b342cb 100644 --- a/examples/14_generics.py +++ b/examples/14_generics.py @@ -7,7 +7,7 @@ import dataclasses from typing import Generic, TypeVar -import dcargs +import tyro ScalarType = TypeVar("ScalarType", int, float) ShapeType = TypeVar("ShapeType") @@ -34,5 +34,5 @@ class Args(Generic[ShapeType]): if __name__ == "__main__": - args = dcargs.cli(Args[Triangle]) + args = tyro.cli(Args[Triangle]) print(args) diff --git a/examples/15_nesting_in_containers.py b/examples/15_nesting_in_containers.py index f5416d05a..d412059eb 100644 --- a/examples/15_nesting_in_containers.py +++ b/examples/15_nesting_in_containers.py @@ -10,7 +10,7 @@ import dataclasses from typing import Dict, Tuple -import dcargs +import tyro class Color: @@ -54,5 +54,5 @@ class Args: if __name__ == "__main__": - args = dcargs.cli(Args) + args = tyro.cli(Args) print(args) diff --git a/examples/16_advanced_configuration.py b/examples/16_advanced_configuration.py index 8f0c58151..6d76e8272 100644 --- a/examples/16_advanced_configuration.py +++ b/examples/16_advanced_configuration.py @@ -1,4 +1,4 @@ -"""The :mod:`dcargs.conf` module contains utilities that can be used to configure +"""The :mod:`tyro.conf` module contains utilities that can be used to configure command-line interfaces beyond what is expressible via static type annotations. Features here are supported, but generally unnecessary and should be used sparingly. @@ -12,7 +12,7 @@ from typing_extensions import Annotated -import dcargs +import tyro @dataclasses.dataclass(frozen=True) @@ -33,29 +33,29 @@ class CommitArgs: @dataclasses.dataclass class Args: # A boolean field with flag conversion turned off. - boolean: dcargs.conf.FlagConversionOff[bool] = False + boolean: tyro.conf.FlagConversionOff[bool] = False # A numeric field parsed as a positional argument. - positional: dcargs.conf.Positional[int] = 3 + positional: tyro.conf.Positional[int] = 3 # A numeric field that can't be changed via the CLI. - fixed: dcargs.conf.Fixed[int] = 5 + fixed: tyro.conf.Fixed[int] = 5 # A union over nested structures, but without subcommand generation. When a default # is provided, the type is simply fixed to that default. - union_without_subcommand: dcargs.conf.AvoidSubcommands[ + union_without_subcommand: tyro.conf.AvoidSubcommands[ Union[CheckoutArgs, CommitArgs] ] = CheckoutArgs("main") - # `dcargs.conf.subcommand()` can be used to configure subcommands in a Union. Here, + # `tyro.conf.subcommand()` can be used to configure subcommands in a Union. Here, # we make the subcommand names more succinct. renamed_subcommand: Union[ Annotated[ - CheckoutArgs, dcargs.conf.subcommand(name="checkout", prefix_name=False) + CheckoutArgs, tyro.conf.subcommand(name="checkout", prefix_name=False) ], - Annotated[CommitArgs, dcargs.conf.subcommand(name="commit", prefix_name=False)], + Annotated[CommitArgs, tyro.conf.subcommand(name="commit", prefix_name=False)], ] = CheckoutArgs("main") if __name__ == "__main__": - print(dcargs.cli(Args)) + print(tyro.cli(Args)) diff --git a/examples/17_flax_modules.py b/examples/17_flax_modules.py index 37ebb3d17..4ef9e80a4 100644 --- a/examples/17_flax_modules.py +++ b/examples/17_flax_modules.py @@ -1,5 +1,5 @@ """If you use [Flax](https://github.com/google/flax), modules can be instantiated -directly from `dcargs.cli`. +directly from `tyro.cli`. Usage: `python ./17_flax_modules.py --help` @@ -9,7 +9,7 @@ from flax import linen as nn from jax import numpy as jnp -import dcargs +import tyro class Classifier(nn.Module): @@ -49,4 +49,4 @@ def train(model: Classifier, num_iterations: int = 1000) -> None: if __name__ == "__main__": - dcargs.cli(train) + tyro.cli(train) diff --git a/examples/_rename_example.py b/examples/_rename_example.py index 324558e88..0c741a514 100644 --- a/examples/_rename_example.py +++ b/examples/_rename_example.py @@ -1,6 +1,6 @@ import pathlib -import dcargs +import tyro def main(source: pathlib.Path, dest: pathlib.Path, /) -> None: @@ -12,4 +12,4 @@ def main(source: pathlib.Path, dest: pathlib.Path, /) -> None: if __name__ == "__main__": - dcargs.cli(main) + tyro.cli(main) diff --git a/pyproject.toml b/pyproject.toml index 485a00efd..0b4882f61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] -name = "dcargs" +name = "tyro" version = "0.3.21" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] -include = ["./dcargs/**/*"] +include = ["./tyro/**/*"] readme = "README.md" -repository = "https://github.com/brentyi/dcargs" -homepage = "https://github.com/brentyi/dcargs" -documentation = "https://brentyi.github.io/dcargs/" +repository = "https://github.com/brentyi/tyro" +homepage = "https://github.com/brentyi/tyro" +documentation = "https://brentyi.github.io/tyro/" [tool.poetry.dependencies] python = "^3.7" @@ -47,7 +47,7 @@ warn_unused_configs = true [tool.coverage.run] omit = [ - "dcargs/_shtab/*", + "tyro/_shtab/*", ] [tool.coverage.report] diff --git a/tests/test_collections.py b/tests/test_collections.py index 6145ee693..4e670e607 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -19,7 +19,7 @@ import pytest from typing_extensions import Literal -import dcargs +import tyro def test_tuples_fixed(): @@ -27,11 +27,11 @@ def test_tuples_fixed(): class A: x: Tuple[int, int, int] - assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_tuples_fixed_mixed(): @@ -39,11 +39,11 @@ def test_tuples_fixed_mixed(): class A: x: Tuple[int, str, int] - assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, "2", 3)) + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, "2", 3)) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_tuples_with_default(): @@ -51,10 +51,10 @@ def test_tuples_with_default(): class A: x: Tuple[int, int, int] = (0, 1, 2) - assert dcargs.cli(A, args=[]) == A(x=(0, 1, 2)) - assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + assert tyro.cli(A, args=[]) == A(x=(0, 1, 2)) + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) def test_tuple_with_literal_and_default(): @@ -62,25 +62,25 @@ def test_tuple_with_literal_and_default(): class A: x: Tuple[Literal[1, 2, 3], ...] = (1, 2) - assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) - assert dcargs.cli(A, args=[]) == A(x=(1, 2)) + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + assert tyro.cli(A, args=[]) == A(x=(1, 2)) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x", "1", "2", "3", "4"]) + tyro.cli(A, args=["--x", "1", "2", "3", "4"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) def test_positional_tuple_with_literal_and_default(): @dataclasses.dataclass class A: - x: dcargs.conf.Positional[Tuple[Literal[1, 2, 3], ...]] = (1, 2) + x: tyro.conf.Positional[Tuple[Literal[1, 2, 3], ...]] = (1, 2) - assert dcargs.cli(A, args=["1", "2", "3"]) == A(x=(1, 2, 3)) - assert dcargs.cli(A, args=[]) == A(x=(1, 2)) + assert tyro.cli(A, args=["1", "2", "3"]) == A(x=(1, 2, 3)) + assert tyro.cli(A, args=[]) == A(x=(1, 2)) target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stderr(target): - dcargs.cli(A, args=["1", "2", "3", "4"]) + tyro.cli(A, args=["1", "2", "3", "4"]) assert "invalid choice" in target.getvalue() @@ -89,11 +89,11 @@ def test_tuples_fixed_multitype(): class A: x: Tuple[int, str, float] - assert dcargs.cli(A, args=["--x", "1", "2", "3.5"]) == A(x=(1, "2", 3.5)) + assert tyro.cli(A, args=["--x", "1", "2", "3.5"]) == A(x=(1, "2", 3.5)) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_tuples_fixed_bool(): @@ -101,13 +101,13 @@ def test_tuples_fixed_bool(): class A: x: Tuple[bool, bool, bool] - assert dcargs.cli(A, args=["--x", "True", "True", "False"]) == A( + assert tyro.cli(A, args=["--x", "True", "True", "False"]) == A( x=(True, True, False) ) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_tuples_variable(): @@ -115,11 +115,11 @@ def test_tuples_variable(): class A: x: Tuple[int, ...] - assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_tuples_variable_bool(): @@ -127,13 +127,13 @@ def test_tuples_variable_bool(): class A: x: Tuple[bool, ...] - assert dcargs.cli(A, args=["--x", "True", "True", "False"]) == A( + assert tyro.cli(A, args=["--x", "True", "True", "False"]) == A( x=(True, True, False) ) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_tuples_variable_optional(): @@ -141,10 +141,10 @@ def test_tuples_variable_optional(): class A: x: Optional[Tuple[int, ...]] = None - assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) - assert dcargs.cli(A, args=[]) == A(x=None) + tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=[]) == A(x=None) def test_sequences(): @@ -152,11 +152,11 @@ def test_sequences(): class A: x: Sequence[int] - assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_lists(): @@ -164,11 +164,11 @@ def test_lists(): class A: x: List[int] - assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_list_with_literal(): @@ -176,13 +176,13 @@ def test_list_with_literal(): class A: x: List[Literal[1, 2, 3]] - assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x", "1", "2", "3", "4"]) + tyro.cli(A, args=["--x", "1", "2", "3", "4"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_list_with_enums(): @@ -195,15 +195,15 @@ class Color(enum.Enum): class A: x: List[Color] - assert dcargs.cli(A, args=["--x", "RED", "RED", "BLUE"]) == A( + assert tyro.cli(A, args=["--x", "RED", "RED", "BLUE"]) == A( x=[Color.RED, Color.RED, Color.BLUE] ) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x", "RED", "RED", "YELLOW"]) + tyro.cli(A, args=["--x", "RED", "RED", "YELLOW"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_lists_with_default(): @@ -218,8 +218,8 @@ class A: default_factory=[Color.RED, Color.GREEN].copy ) - assert dcargs.cli(A, args=[]) == A(x=[Color.RED, Color.GREEN]) - assert dcargs.cli(A, args=["--x", "RED", "GREEN", "BLUE"]) == A( + assert tyro.cli(A, args=[]) == A(x=[Color.RED, Color.GREEN]) + assert tyro.cli(A, args=["--x", "RED", "GREEN", "BLUE"]) == A( x=[Color.RED, Color.GREEN, Color.BLUE] ) @@ -229,13 +229,13 @@ def test_lists_bool(): class A: x: List[bool] - assert dcargs.cli(A, args=["--x", "True", "False", "True"]) == A( + assert tyro.cli(A, args=["--x", "True", "False", "True"]) == A( x=[True, False, True] ) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_sets(): @@ -243,11 +243,11 @@ def test_sets(): class A: x: Set[int] - assert dcargs.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) + assert tyro.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_frozen_sets(): @@ -255,11 +255,11 @@ def test_frozen_sets(): class A: x: FrozenSet[int] - assert dcargs.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x=frozenset({1, 2, 3})) + assert tyro.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x=frozenset({1, 2, 3})) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_deque(): @@ -267,13 +267,13 @@ def test_deque(): class A: x: Deque[int] - assert dcargs.cli(A, args=["--x", "1", "2", "3", "3"]) == A( + assert tyro.cli(A, args=["--x", "1", "2", "3", "3"]) == A( x=collections.deque([1, 2, 3, 3]) ) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_sets_with_default(): @@ -281,10 +281,10 @@ def test_sets_with_default(): class A: x: Set[int] = dataclasses.field(default_factory={0, 1, 2}.copy) - assert dcargs.cli(A, args=[]) == A(x={0, 1, 2}) - assert dcargs.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) + assert tyro.cli(A, args=[]) == A(x={0, 1, 2}) + assert tyro.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) + tyro.cli(A, args=["--x"]) def test_optional_sequences(): @@ -292,10 +292,10 @@ def test_optional_sequences(): class A: x: Optional[Sequence[int]] = None - assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) - assert dcargs.cli(A, args=[]) == A(x=None) + tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=[]) == A(x=None) def test_optional_lists(): @@ -303,10 +303,10 @@ def test_optional_lists(): class A: x: Optional[List[int]] = None - assert dcargs.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x"]) - assert dcargs.cli(A, args=[]) == A(x=None) + tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=[]) == A(x=None) def test_nested_optional_types(): @@ -318,40 +318,40 @@ def test_nested_optional_types(): class A: x: Tuple[Optional[int], ...] - assert dcargs.cli(A, args=["--x", "0", "1"]) == A((0, 1)) - assert dcargs.cli(A, args=["--x", "0", "None", "1"]) == A((0, None, 1)) + assert tyro.cli(A, args=["--x", "0", "1"]) == A((0, 1)) + assert tyro.cli(A, args=["--x", "0", "None", "1"]) == A((0, None, 1)) def test_union_over_collections(): def main(a: Union[Tuple[float, ...], Tuple[int, ...]]) -> Any: return a - assert dcargs.cli(main, args="--a 3.3 3.3 7.0".split(" ")) == (3.3, 3.3, 7.0) - assert dcargs.cli(main, args="--a 3 3 7".split(" ")) == (3, 3, 7) + assert tyro.cli(main, args="--a 3.3 3.3 7.0".split(" ")) == (3.3, 3.3, 7.0) + assert tyro.cli(main, args="--a 3 3 7".split(" ")) == (3, 3, 7) def test_union_over_collections_2(): def main(a: Union[Tuple[str, float, str], Tuple[str, str, float]]) -> Any: return a - assert dcargs.cli(main, args="--a 3.3 hey 7.0".split(" ")) == ("3.3", "hey", 7.0) - assert dcargs.cli(main, args="--a 3 3 hey".split(" ")) == ("3", 3.0, "hey") + assert tyro.cli(main, args="--a 3.3 hey 7.0".split(" ")) == ("3.3", "hey", 7.0) + assert tyro.cli(main, args="--a 3 3 hey".split(" ")) == ("3", 3.0, "hey") def test_union_over_collections_3(): def main(a: Union[Tuple[int, int], Tuple[int, int, int]]) -> Tuple[int, ...]: return a - assert dcargs.cli(main, args=["--a", "5", "5"]) == (5, 5) - assert dcargs.cli(main, args=["--a", "1", "2", "3"]) == (1, 2, 3) + assert tyro.cli(main, args=["--a", "5", "5"]) == (5, 5) + assert tyro.cli(main, args=["--a", "1", "2", "3"]) == (1, 2, 3) with pytest.raises(SystemExit): - dcargs.cli(main, args=["--a", "5", "5", "2", "1"]) + tyro.cli(main, args=["--a", "5", "5", "2", "1"]) with pytest.raises(SystemExit): - dcargs.cli(main, args=["--a"]) + tyro.cli(main, args=["--a"]) with pytest.raises(SystemExit): - dcargs.cli(main, args=[]) + tyro.cli(main, args=[]) def test_choices_in_tuples_0(): @@ -359,7 +359,7 @@ def test_choices_in_tuples_0(): class A: x: Tuple[bool, bool] - assert dcargs.cli(A, args=["--x", "True", "False"]) == A((True, False)) + assert tyro.cli(A, args=["--x", "True", "False"]) == A((True, False)) def test_choices_in_tuples_1(): @@ -367,7 +367,7 @@ def test_choices_in_tuples_1(): class A: x: Tuple[bool, Literal["True", "False"]] - assert dcargs.cli(A, args=["--x", "True", "False"]) == A((True, "False")) + assert tyro.cli(A, args=["--x", "True", "False"]) == A((True, "False")) def test_choices_in_tuples_2(): @@ -375,10 +375,10 @@ def test_choices_in_tuples_2(): class A: x: Tuple[bool, Literal["True", "False", "None"]] - assert dcargs.cli(A, args=["--x", "True", "False"]).x == (True, "False") - assert dcargs.cli(A, args=["--x", "False", "None"]).x == (False, "None") + assert tyro.cli(A, args=["--x", "True", "False"]).x == (True, "False") + assert tyro.cli(A, args=["--x", "False", "None"]).x == (False, "None") with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x", "None", "False"]) + tyro.cli(A, args=["--x", "None", "False"]) def test_nested_tuple_types(): @@ -386,16 +386,16 @@ def test_nested_tuple_types(): class A: x: Tuple[Tuple[int, int], Tuple[str, str]] - assert dcargs.cli(A, args="--x 5 5 5 5".split(" ")).x == ((5, 5), ("5", "5")) + assert tyro.cli(A, args="--x 5 5 5 5".split(" ")).x == ((5, 5), ("5", "5")) def test_variable_nested_tuple(): def main(x: Tuple[Tuple[int, str], ...]) -> tuple: return x - assert dcargs.cli(main, args="--x 1 1 2 2".split(" ")) == ((1, "1"), (2, "2")) + assert tyro.cli(main, args="--x 1 1 2 2".split(" ")) == ((1, "1"), (2, "2")) with pytest.raises(SystemExit): - dcargs.cli(main, args="--x 1 1 2".split(" ")) + tyro.cli(main, args="--x 1 1 2".split(" ")) def test_super_nested(): @@ -412,24 +412,24 @@ def main( ) -> Any: return x - assert dcargs.cli(main, args=[]) is None - assert dcargs.cli(main, args="--x None".split(" ")) is None - assert dcargs.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] - assert dcargs.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] - assert dcargs.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ + assert tyro.cli(main, args=[]) is None + assert tyro.cli(main, args="--x None".split(" ")) is None + assert tyro.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] + assert tyro.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] + assert tyro.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ (2, 3, ("x", "2")), (2, 3, (1, 2)), ] with pytest.raises(SystemExit): - dcargs.cli(main, args=["--help"]) + tyro.cli(main, args=["--help"]) def test_dict_no_annotation(): def main(x: Dict[str, Any] = {"int": 5, "str": "5"}): return x - assert dcargs.cli(main, args=[]) == {"int": 5, "str": "5"} - assert dcargs.cli(main, args="--x.int 3 --x.str 7".split(" ")) == { + assert tyro.cli(main, args=[]) == {"int": 5, "str": "5"} + assert tyro.cli(main, args="--x.int 3 --x.str 7".split(" ")) == { "int": 3, "str": "7", } @@ -443,8 +443,8 @@ def main( ): return x - assert dcargs.cli(main, args=[]) == {"wow": {"int": 5, "str": "5"}} - assert dcargs.cli(main, args="--x.wow.int 3 --x.wow.str 7".split(" ")) == { + assert tyro.cli(main, args=[]) == {"wow": {"int": 5, "str": "5"}} + assert tyro.cli(main, args="--x.wow.int 3 --x.wow.str 7".split(" ")) == { "wow": { "int": 3, "str": "7", diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index b6dc19712..c589e5059 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -20,16 +20,16 @@ import torch from typing_extensions import Annotated, Final, Literal, TypeAlias -import dcargs +import tyro def test_no_args(): def main() -> int: return 5 - assert dcargs.cli(main, args=[]) == 5 + assert tyro.cli(main, args=[]) == 5 with pytest.raises(SystemExit): - dcargs.cli(main, args=["3"]) + tyro.cli(main, args=["3"]) def test_basic(): @@ -40,8 +40,8 @@ class ManyTypes: f: float p: pathlib.Path - # We can directly pass a dataclass to `dcargs.cli()`: - assert dcargs.cli( + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( ManyTypes, args=[ "--i", @@ -55,11 +55,11 @@ class ManyTypes: ], ) == ManyTypes(i=5, s="5", f=5.0, p=pathlib.Path("~")) - # We can directly pass a function to `dcargs.cli()`: + # We can directly pass a function to `tyro.cli()`: def function(i: int, s: str, f: float, p: pathlib.Path) -> ManyTypes: return ManyTypes(i=i, s=s, f=f, p=p) - assert dcargs.cli( + assert tyro.cli( function, args=[ "--i", @@ -73,12 +73,12 @@ def function(i: int, s: str, f: float, p: pathlib.Path) -> ManyTypes: ], ) == ManyTypes(i=5, s="5", f=5.0, p=pathlib.Path("~")) - # We can directly pass a generic class to `dcargs.cli()`: + # We can directly pass a generic class to `tyro.cli()`: class Wrapper: def __init__(self, i: int, s: str, f: float, p: pathlib.Path): self.inner = ManyTypes(i=i, s=s, f=f, p=p) - assert dcargs.cli( + assert tyro.cli( Wrapper, args=[ "--i", @@ -102,7 +102,7 @@ class InitFalseDataclass: dir: pathlib.Path ignored: str = dataclasses.field(default="hello", init=False) - assert dcargs.cli( + assert tyro.cli( InitFalseDataclass, args=[ "--i", @@ -117,7 +117,7 @@ class InitFalseDataclass: ) == InitFalseDataclass(i=5, s="5", f=5.0, dir=pathlib.Path("~")) with pytest.raises(SystemExit): - dcargs.cli( + tyro.cli( InitFalseDataclass, args=[ "--i", @@ -140,7 +140,7 @@ class A: x: int with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) def test_flag(): @@ -151,19 +151,19 @@ class A: x: bool with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) + tyro.cli(A, args=[]) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x", "1"]) + tyro.cli(A, args=["--x", "1"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x", "true"]) - assert dcargs.cli(A, args=["--x", "True"]) == A(True) + tyro.cli(A, args=["--x", "true"]) + assert tyro.cli(A, args=["--x", "True"]) == A(True) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x", "0"]) + tyro.cli(A, args=["--x", "0"]) with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x", "false"]) - assert dcargs.cli(A, args=["--x", "False"]) == A(False) + tyro.cli(A, args=["--x", "false"]) + assert tyro.cli(A, args=["--x", "False"]) == A(False) def test_flag_default_false(): @@ -173,8 +173,8 @@ def test_flag_default_false(): class A: x: bool = False - assert dcargs.cli(A, args=[]) == A(False) - assert dcargs.cli(A, args=["--x"]) == A(True) + assert tyro.cli(A, args=[]) == A(False) + assert tyro.cli(A, args=["--x"]) == A(True) def test_flag_default_true(): @@ -184,8 +184,8 @@ def test_flag_default_true(): class A: x: bool = True - assert dcargs.cli(A, args=[]) == A(True) - assert dcargs.cli(A, args=["--no-x"]) == A(False) + assert tyro.cli(A, args=[]) == A(True) + assert tyro.cli(A, args=["--no-x"]) == A(False) def test_flag_default_true_nested(): @@ -199,8 +199,8 @@ class NestedDefaultTrue: class A: x: NestedDefaultTrue - assert dcargs.cli(A, args=[]) == A(NestedDefaultTrue(True)) - assert dcargs.cli(A, args=["--x.no-x"]) == A(NestedDefaultTrue(False)) + assert tyro.cli(A, args=[]) == A(NestedDefaultTrue(True)) + assert tyro.cli(A, args=["--x.no-x"]) == A(NestedDefaultTrue(False)) def test_default(): @@ -208,7 +208,7 @@ def test_default(): class A: x: int = 5 - assert dcargs.cli(A, args=[]) == A() + assert tyro.cli(A, args=[]) == A() def test_default_factory(): @@ -216,7 +216,7 @@ def test_default_factory(): class A: x: int = dataclasses.field(default_factory=lambda: 5) - assert dcargs.cli(A, args=[]) == A() + assert tyro.cli(A, args=[]) == A() def test_optional(): @@ -224,36 +224,36 @@ def test_optional(): class A: x: Optional[int] = None - assert dcargs.cli(A, args=[]) == A(x=None) + assert tyro.cli(A, args=[]) == A(x=None) def test_union_basic(): def main(x: Union[int, str]) -> Union[int, str]: return x - assert dcargs.cli(main, args=["--x", "5"]) == 5 - assert dcargs.cli(main, args=["--x", "6"]) == 6 - assert dcargs.cli(main, args=["--x", "five"]) == "five" + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == 6 + assert tyro.cli(main, args=["--x", "five"]) == "five" def test_union_with_list(): def main(x: Union[int, str, List[bool]]) -> Any: return x - assert dcargs.cli(main, args=["--x", "5"]) == 5 - assert dcargs.cli(main, args=["--x", "6"]) == 6 - assert dcargs.cli(main, args=["--x", "five"]) == "five" - assert dcargs.cli(main, args=["--x", "True"]) == "True" - assert dcargs.cli(main, args=["--x", "True", "False"]) == [True, False] + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == 6 + assert tyro.cli(main, args=["--x", "five"]) == "five" + assert tyro.cli(main, args=["--x", "True"]) == "True" + assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] def test_union_literal(): def main(x: Union[Literal[1, 2], Literal[3, 4, 5], str]) -> Union[int, str]: return x - assert dcargs.cli(main, args=["--x", "5"]) == 5 - assert dcargs.cli(main, args=["--x", "6"]) == "6" - assert dcargs.cli(main, args=["--x", "five"]) == "five" + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == "6" + assert tyro.cli(main, args=["--x", "five"]) == "five" def test_func_typevar(): @@ -262,8 +262,8 @@ def test_func_typevar(): def main(x: T) -> T: return x - assert dcargs.cli(main, args=["--x", "5"]) == 5 - assert dcargs.cli(main, args=["--x", "five"]) == "five" + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "five"]) == "five" def test_func_typevar_bound(): @@ -272,9 +272,9 @@ def test_func_typevar_bound(): def main(x: T) -> T: return x - assert dcargs.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "5"]) == 5 with pytest.raises(SystemExit): - dcargs.cli(main, args=["--x", "five"]) + tyro.cli(main, args=["--x", "five"]) def test_enum(): @@ -291,10 +291,8 @@ class EnumClassA: class EnumClassB: color: Color = Color.GREEN - assert dcargs.cli(EnumClassA, args=["--color", "RED"]) == EnumClassA( - color=Color.RED - ) - assert dcargs.cli(EnumClassB, args=[]) == EnumClassB() + assert tyro.cli(EnumClassA, args=["--color", "RED"]) == EnumClassA(color=Color.RED) + assert tyro.cli(EnumClassB, args=[]) == EnumClassB() def test_literal(): @@ -302,14 +300,14 @@ def test_literal(): class A: x: Literal[0, 1, 2] - assert dcargs.cli(A, args=["--x", "1"]) == A(x=1) + assert tyro.cli(A, args=["--x", "1"]) == A(x=1) with pytest.raises(SystemExit): - assert dcargs.cli(A, args=["--x", "3"]) + assert tyro.cli(A, args=["--x", "3"]) # Hack for mypy. Not needed for pyright. Choices = int -Choices = dcargs.extras.literal_type_from_choices([0, 1, 2]) # type: ignore +Choices = tyro.extras.literal_type_from_choices([0, 1, 2]) # type: ignore def test_dynamic_literal(): @@ -317,26 +315,26 @@ def test_dynamic_literal(): class A: x: Choices - assert dcargs.cli(A, args=["--x", "1"]) == A(x=1) + assert tyro.cli(A, args=["--x", "1"]) == A(x=1) with pytest.raises(SystemExit): - assert dcargs.cli(A, args=["--x", "3"]) + assert tyro.cli(A, args=["--x", "3"]) def test_literal_bool(): def main(x: Literal[True]) -> bool: return x - assert dcargs.cli(main, args=["--x", "True"]) is True + assert tyro.cli(main, args=["--x", "True"]) is True with pytest.raises(SystemExit): - dcargs.cli(main, args=["--x", "False"]) + tyro.cli(main, args=["--x", "False"]) def main2(x: Literal[True, False]) -> bool: return x - assert dcargs.cli(main2, args=["--x", "True"]) is True - assert dcargs.cli(main2, args=["--x", "False"]) is False + assert tyro.cli(main2, args=["--x", "True"]) is True + assert tyro.cli(main2, args=["--x", "False"]) is False with pytest.raises(SystemExit): - dcargs.cli(main2, args=["--x", "Tru"]) + tyro.cli(main2, args=["--x", "Tru"]) def test_literal_enum(): @@ -349,10 +347,10 @@ class Color(enum.Enum): class A: x: Literal[Color.RED, Color.GREEN] - assert dcargs.cli(A, args=["--x", "RED"]) == A(x=Color.RED) - assert dcargs.cli(A, args=["--x", "GREEN"]) == A(x=Color.GREEN) + assert tyro.cli(A, args=["--x", "RED"]) == A(x=Color.RED) + assert tyro.cli(A, args=["--x", "GREEN"]) == A(x=Color.GREEN) with pytest.raises(SystemExit): - assert dcargs.cli(A, args=["--x", "BLUE"]) + assert tyro.cli(A, args=["--x", "BLUE"]) def test_optional_literal(): @@ -360,20 +358,20 @@ def test_optional_literal(): class A: x: Optional[Literal[0, 1, 2]] = None - assert dcargs.cli(A, args=["--x", "1"]) == A(x=1) + assert tyro.cli(A, args=["--x", "1"]) == A(x=1) with pytest.raises(SystemExit): - assert dcargs.cli(A, args=["--x", "3"]) - assert dcargs.cli(A, args=[]) == A(x=None) + assert tyro.cli(A, args=["--x", "3"]) + assert tyro.cli(A, args=[]) == A(x=None) def test_multitype_literal(): def main(x: Literal[0, "5"]) -> Any: return x - assert dcargs.cli(main, args=["--x", "0"]) == 0 - assert dcargs.cli(main, args=["--x", "5"]) == "5" + assert tyro.cli(main, args=["--x", "0"]) == 0 + assert tyro.cli(main, args=["--x", "5"]) == "5" with pytest.raises(SystemExit): - dcargs.cli(main, args=["--x", "6"]) + tyro.cli(main, args=["--x", "6"]) def test_annotated(): @@ -383,7 +381,7 @@ def test_annotated(): class A: x: Annotated[int, "some label"] = 3 - assert dcargs.cli(A, args=["--x", "5"]) == A(x=5) + assert tyro.cli(A, args=["--x", "5"]) == A(x=5) def test_annotated_optional(): @@ -393,8 +391,8 @@ def test_annotated_optional(): class A: x: Annotated[Optional[int], "some label"] = 3 - assert dcargs.cli(A, args=[]) == A(x=3) - assert dcargs.cli(A, args=["--x", "5"]) == A(x=5) + assert tyro.cli(A, args=[]) == A(x=3) + assert tyro.cli(A, args=["--x", "5"]) == A(x=5) def test_optional_annotated(): @@ -404,8 +402,8 @@ def test_optional_annotated(): class A: x: Optional[Annotated[int, "some label"]] = 3 - assert dcargs.cli(A, args=[]) == A(x=3) - assert dcargs.cli(A, args=["--x", "5"]) == A(x=5) + assert tyro.cli(A, args=[]) == A(x=3) + assert tyro.cli(A, args=["--x", "5"]) == A(x=5) def test_final(): @@ -415,7 +413,7 @@ def test_final(): class A: x: Final[int] = 3 - assert dcargs.cli(A, args=["--x", "5"]) == A(x=5) + assert tyro.cli(A, args=["--x", "5"]) == A(x=5) def test_final_optional(): @@ -423,8 +421,8 @@ def test_final_optional(): class A: x: Final[Optional[int]] = 3 - assert dcargs.cli(A, args=[]) == A(x=3) - assert dcargs.cli(A, args=["--x", "5"]) == A(x=5) + assert tyro.cli(A, args=[]) == A(x=3) + assert tyro.cli(A, args=["--x", "5"]) == A(x=5) def test_classvar(): @@ -435,8 +433,8 @@ class A: x: ClassVar[int] = 5 with pytest.raises(SystemExit): - dcargs.cli(A, args=["--x", "1"]) - assert dcargs.cli(A, args=[]) == A() + tyro.cli(A, args=["--x", "1"]) + assert tyro.cli(A, args=[]) == A() def test_parse_empty_description(): @@ -446,7 +444,7 @@ def test_parse_empty_description(): class A: x: int = 0 - assert dcargs.cli(A, description=None, args=[]) == A(x=0) + assert tyro.cli(A, description=None, args=[]) == A(x=0) SomeTypeAlias: TypeAlias = int @@ -456,7 +454,7 @@ def test_type_alias(): def add(a: SomeTypeAlias, b: SomeTypeAlias) -> SomeTypeAlias: return a + b - assert dcargs.cli(add, args=["--a", "5", "--b", "7"]) == 12 + assert tyro.cli(add, args=["--a", "5", "--b", "7"]) == 12 @pytest.mark.filterwarnings("ignore::Warning") @@ -464,14 +462,14 @@ def test_any(): def main(x: Any = 5) -> Any: return x - assert dcargs.cli(main, args=[]) == 5 + assert tyro.cli(main, args=[]) == 5 def test_bytes(): def main(x: bytes) -> bytes: return x - assert dcargs.cli(main, args=["--x", "hello"]) == b"hello" + assert tyro.cli(main, args=["--x", "hello"]) == b"hello" def test_any_str(): @@ -479,17 +477,17 @@ def main(x: AnyStr) -> AnyStr: return x # Use bytes when provided ascii-compatible inputs. - assert dcargs.cli(main, args=["--x", "hello"]) == b"hello" - assert dcargs.cli(main, args=["--x", "hello„"]) == "hello„" + assert tyro.cli(main, args=["--x", "hello"]) == b"hello" + assert tyro.cli(main, args=["--x", "hello„"]) == "hello„" def test_fixed(): def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]: return x - assert dcargs.cli(main, args=[])(3) == 6 + assert tyro.cli(main, args=[])(3) == 6 with pytest.raises(SystemExit): - dcargs.cli(main, args=["--x", "something"]) + tyro.cli(main, args=["--x", "something"]) def test_fixed_dataclass_type(): @@ -500,43 +498,43 @@ class Dummy: def main(x: Callable = Dummy) -> Callable: return x - assert dcargs.cli(main, args=[]) is Dummy + assert tyro.cli(main, args=[]) is Dummy with pytest.raises(SystemExit): - dcargs.cli(main, args=["--x", "something"]) + tyro.cli(main, args=["--x", "something"]) def test_missing_singleton(): - assert dcargs.MISSING is copy.deepcopy(dcargs.MISSING) + assert tyro.MISSING is copy.deepcopy(tyro.MISSING) def test_torch_device(): def main(device: torch.device) -> torch.device: return device - assert dcargs.cli(main, args=["--device", "cpu"]) == torch.device("cpu") + assert tyro.cli(main, args=["--device", "cpu"]) == torch.device("cpu") def test_torch_device_2(): - assert dcargs.cli(torch.device, args=["cpu"]) == torch.device("cpu") + assert tyro.cli(torch.device, args=["cpu"]) == torch.device("cpu") def test_just_int(): - assert dcargs.cli(int, args=["123"]) == 123 + assert tyro.cli(int, args=["123"]) == 123 def test_just_dict(): - assert dcargs.cli(Dict[str, str], args="key value key2 value2".split(" ")) == { + assert tyro.cli(Dict[str, str], args="key value key2 value2".split(" ")) == { "key": "value", "key2": "value2", } def test_just_list(): - assert dcargs.cli(List[int], args="1 2 3 4".split(" ")) == [1, 2, 3, 4] + assert tyro.cli(List[int], args="1 2 3 4".split(" ")) == [1, 2, 3, 4] def test_just_tuple(): - assert dcargs.cli(Tuple[int, int, int, int], args="1 2 3 4".split(" ")) == ( + assert tyro.cli(Tuple[int, int, int, int], args="1 2 3 4".split(" ")) == ( 1, 2, 3, @@ -549,4 +547,4 @@ def main() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() return parser - assert isinstance(dcargs.cli(main, args=[]), argparse.ArgumentParser) + assert isinstance(tyro.cli(main, args=[]), argparse.ArgumentParser) diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index 0156ea766..f7741e1ea 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -7,48 +7,48 @@ import pytest from typing_extensions import Literal, TypedDict -import dcargs -import dcargs._strings +import tyro +import tyro._strings def test_basic_dict(): def main(params: Dict[str, int]) -> Dict[str, int]: return params - assert dcargs.cli(main, args="--params hey 5 hello 2".split(" ")) == { + assert tyro.cli(main, args="--params hey 5 hello 2".split(" ")) == { "hey": 5, "hello": 2, } - assert dcargs.cli(main, args="--params hey 5 hello 2".split(" ")) == { + assert tyro.cli(main, args="--params hey 5 hello 2".split(" ")) == { "hey": 5, "hello": 2, } with pytest.raises(SystemExit): - dcargs.cli(main, args="--params hey 5 hello hey".split(" ")) + tyro.cli(main, args="--params hey 5 hello hey".split(" ")) with pytest.raises(SystemExit): - dcargs.cli(main, args="--params hey 5 hello".split(" ")) + tyro.cli(main, args="--params hey 5 hello".split(" ")) with pytest.raises(SystemExit): - dcargs.cli(main, args="--params".split(" ")) + tyro.cli(main, args="--params".split(" ")) def test_dict_with_default(): def main(params: Mapping[Literal[1, 3, 5, 7], bool] = {5: False, 1: True}) -> Any: return params - assert dcargs.cli(main, args=[]) == {5: False, 1: True} - assert dcargs.cli(main, args="--params.5 --params.no-1".split(" ")) == { + assert tyro.cli(main, args=[]) == {5: False, 1: True} + assert tyro.cli(main, args="--params.5 --params.no-1".split(" ")) == { 5: True, 1: False, } with pytest.raises(SystemExit): - dcargs.cli(main, args="--params".split(" ")) + tyro.cli(main, args="--params".split(" ")) def test_tuple_in_dict(): def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> dict: return x - assert dcargs.cli(main, args="--x 1 1 2 2 3 3 4 4".split(" ")) == { + assert tyro.cli(main, args="--x 1 1 2 2 3 3 4 4".split(" ")) == { (1, 1): (2, 2), (3, 3): (4, 4), } @@ -59,16 +59,16 @@ class ManyTypesTypedDict(TypedDict): i: int s: str - assert dcargs.cli( + assert tyro.cli( ManyTypesTypedDict, args="--i 5 --s 5".split(" "), ) == dict(i=5, s="5") with pytest.raises(SystemExit): - dcargs.cli(ManyTypesTypedDict, args="--i 5".split(" ")) + tyro.cli(ManyTypesTypedDict, args="--i 5".split(" ")) with pytest.raises(SystemExit): - dcargs.cli(ManyTypesTypedDict, args="--s 5".split(" ")) + tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) def test_total_false_typeddict(): @@ -76,13 +76,13 @@ class ManyTypesTypedDict(TypedDict, total=False): i: int s: str - assert dcargs.cli( + assert tyro.cli( ManyTypesTypedDict, args="--i 5 --s 5".split(" "), ) == dict(i=5, s="5") - assert dcargs.cli(ManyTypesTypedDict, args="--i 5".split(" ")) == dict(i=5) - assert dcargs.cli(ManyTypesTypedDict, args="--s 5".split(" ")) == dict(s="5") + assert tyro.cli(ManyTypesTypedDict, args="--i 5".split(" ")) == dict(i=5) + assert tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) == dict(s="5") def test_total_false_nested_typeddict(): @@ -93,15 +93,15 @@ class ChildTypedDict(TypedDict, total=False): class ParentTypedDict(TypedDict, total=False): child: ChildTypedDict - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli( + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli( ParentTypedDict, args="--child.i 5 --child.s 5".split(" "), ) - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): + with pytest.raises(tyro.UnsupportedTypeAnnotationError): assert ( - dcargs.cli( + tyro.cli( ParentTypedDict, args=[""], ) @@ -118,14 +118,14 @@ class ManyTypesTypedDict(TypedDict, total=False): i: int s: Inner - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli( + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli( ManyTypesTypedDict, args="".split(" "), ) - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli( + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli( ManyTypesTypedDict, args="--x.i 5 --x.s 5 5".split(" "), ) @@ -137,14 +137,14 @@ class ManyTypesTypedDict(TypedDict, total=False): s: Tuple[str, str] assert ( - dcargs.cli( + tyro.cli( ManyTypesTypedDict, args=[], ) == dict() ) - assert dcargs.cli( + assert tyro.cli( ManyTypesTypedDict, args="--i 5 --s 5 5".split(" "), ) == dict(i=5, s=("5", "5")) @@ -158,11 +158,11 @@ class NestedTypedDict(TypedDict): x: int b: ChildTypedDict - assert dcargs.cli(NestedTypedDict, args=["--x", "1", "--b.y", "3"]) == dict( + assert tyro.cli(NestedTypedDict, args=["--x", "1", "--b.y", "3"]) == dict( x=1, b=dict(y=3) ) with pytest.raises(SystemExit): - dcargs.cli(NestedTypedDict, args=["--x", "1"]) + tyro.cli(NestedTypedDict, args=["--x", "1"]) def test_helptext_and_default_typeddict(): @@ -180,8 +180,8 @@ class HelptextTypedDict(TypedDict): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.cli(HelptextTypedDict, default={"z": 3}, args=["--help"]) - helptext = dcargs._strings.strip_ansi_sequences(f.getvalue()) + tyro.cli(HelptextTypedDict, default={"z": 3}, args=["--help"]) + helptext = tyro._strings.strip_ansi_sequences(f.getvalue()) assert cast(str, HelptextTypedDict.__doc__) in helptext assert "--x INT" in helptext assert "--y INT" in helptext @@ -198,7 +198,7 @@ class ManyTypesNamedTuple(NamedTuple): f: float p: pathlib.Path - assert dcargs.cli( + assert tyro.cli( ManyTypesNamedTuple, args=[ "--i", @@ -221,11 +221,11 @@ class NestedNamedTuple(NamedTuple): x: int b: ChildNamedTuple - assert dcargs.cli( + assert tyro.cli( NestedNamedTuple, args=["--x", "1", "--b.y", "3"] ) == NestedNamedTuple(x=1, b=ChildNamedTuple(y=3)) with pytest.raises(SystemExit): - dcargs.cli(NestedNamedTuple, args=["--x", "1"]) + tyro.cli(NestedNamedTuple, args=["--x", "1"]) def test_helptext_and_default_namedtuple(): @@ -243,8 +243,8 @@ class HelptextNamedTupleDefault(NamedTuple): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.cli(HelptextNamedTupleDefault, args=["--help"]) - helptext = dcargs._strings.strip_ansi_sequences(f.getvalue()) + tyro.cli(HelptextNamedTupleDefault, args=["--help"]) + helptext = tyro._strings.strip_ansi_sequences(f.getvalue()) assert cast(str, HelptextNamedTupleDefault.__doc__) in helptext assert "--x INT" in helptext assert "--y INT" in helptext @@ -267,25 +267,25 @@ class HelptextNamedTuple(NamedTuple): """Documentation 3""" with pytest.raises(SystemExit): - dcargs.cli( + tyro.cli( HelptextNamedTuple, - default=dcargs.MISSING, + default=tyro.MISSING, args=[], ) f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.cli( + tyro.cli( HelptextNamedTuple, default=HelptextNamedTuple( - x=dcargs.MISSING, - y=dcargs.MISSING, + x=tyro.MISSING, + y=tyro.MISSING, z=3, ), args=["--help"], ) - helptext = dcargs._strings.strip_ansi_sequences(f.getvalue()) + helptext = tyro._strings.strip_ansi_sequences(f.getvalue()) assert cast(str, HelptextNamedTuple.__doc__) in helptext assert "Documentation 1 (required)" in helptext assert "Documentation 2 (required)" in helptext diff --git a/tests/test_dynamic_dataclasses.py b/tests/test_dynamic_dataclasses.py index c46093914..b0f30091a 100644 --- a/tests/test_dynamic_dataclasses.py +++ b/tests/test_dynamic_dataclasses.py @@ -2,7 +2,7 @@ import pytest -import dcargs +import tyro def test_dynamic(): @@ -10,5 +10,5 @@ def test_dynamic(): A = make_dataclass("A", [("b", B, field())]) with pytest.raises(SystemExit): - dcargs.cli(A, args=[]) - assert dcargs.cli(A, args=["--b.c", "5"]) == A(b=B(c=5)) + tyro.cli(A, args=[]) + assert tyro.cli(A, args=["--b.c", "5"]) == A(b=B(c=5)) diff --git a/tests/test_errors.py b/tests/test_errors.py index 4d71aa2c2..814b89630 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -3,7 +3,7 @@ import pytest -import dcargs +import tyro def test_ambiguous_collection_0(): @@ -11,24 +11,24 @@ def test_ambiguous_collection_0(): class A: x: Tuple[Tuple[int, ...], ...] - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(A, args=["--x", "0", "1"]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(A, args=["--x", "0", "1"]) def test_ambiguous_collection_1(): def main(x: Tuple[List[str], List[str]]) -> None: pass - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main, args=["--help"]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) def test_ambiguous_collection_2(): def main(x: List[Union[Tuple[int, int], Tuple[int, int, int]]]) -> None: pass - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main, args=["--help"]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) # Must be global. @@ -38,16 +38,16 @@ class _CycleDataclass: def test_cycle(): - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(_CycleDataclass, args=[]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(_CycleDataclass, args=[]) def test_uncallable_annotation(): def main(arg: 5) -> None: # type: ignore pass - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main, args=[]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=[]) def test_nested_annotation(): @@ -58,8 +58,8 @@ class OneIntArg: def main(arg: List[OneIntArg]) -> List[OneIntArg]: # type: ignore return arg - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main, args=[]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=[]) @dataclasses.dataclass class OneStringArg: @@ -68,8 +68,8 @@ class OneStringArg: def main(arg: List[OneStringArg]) -> List[OneStringArg]: # type: ignore return arg - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main, args=["--arg", "0", "1", "2"]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--arg", "0", "1", "2"]) @dataclasses.dataclass class TwoStringArg: @@ -79,21 +79,21 @@ class TwoStringArg: def main(arg: List[TwoStringArg]) -> None: pass - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main, args=[]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=[]) def test_missing_annotation_1(): def main(a, b) -> None: pass - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main, args=["--help"]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) def test_missing_annotation_2(): def main(*, a) -> None: pass - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main, args=["--help"]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) diff --git a/tests/test_flax_ignore_py310.py b/tests/test_flax_ignore_py310.py index fa2cb027f..9642692e2 100644 --- a/tests/test_flax_ignore_py310.py +++ b/tests/test_flax_ignore_py310.py @@ -1,10 +1,10 @@ -"""Tests initializing flax modules directly via dcargs.""" +"""Tests initializing flax modules directly via tyro.""" import jax import pytest from flax import linen as nn from jax import numpy as jnp -import dcargs +import tyro class Classifier(nn.Module): @@ -30,7 +30,7 @@ def __call__(self, x: jnp.ndarray) -> jnp.ndarray: # type: ignore def test_ok(): - network = dcargs.cli( + network = tyro.cli( Classifier, args=[ "--layers", @@ -49,7 +49,7 @@ def test_ok(): def test_missing(): with pytest.raises(SystemExit): - dcargs.cli( + tyro.cli( Classifier, args=[ "--layers", diff --git a/tests/test_forward_ref.py b/tests/test_forward_ref.py index 3bf4230f9..ccd29863e 100644 --- a/tests/test_forward_ref.py +++ b/tests/test_forward_ref.py @@ -3,7 +3,7 @@ import pytest -import dcargs +import tyro @dataclasses.dataclass @@ -29,28 +29,20 @@ class C: def test_forward_ref_1(): - assert dcargs.cli(A1, args=["--x", "1", "bc:b", "--bc.y", "3"]) == A1( - x=1, bc=B(y=3) - ) - assert dcargs.cli(A1, args=["--x", "1", "bc:c", "--bc.z", "3"]) == A1( - x=1, bc=C(z=3) - ) + assert tyro.cli(A1, args=["--x", "1", "bc:b", "--bc.y", "3"]) == A1(x=1, bc=B(y=3)) + assert tyro.cli(A1, args=["--x", "1", "bc:c", "--bc.z", "3"]) == A1(x=1, bc=C(z=3)) with pytest.raises(SystemExit): - dcargs.cli(A1, args=["--x", "1", "bc:b", "--bc.z", "3"]) + tyro.cli(A1, args=["--x", "1", "bc:b", "--bc.z", "3"]) with pytest.raises(SystemExit): - dcargs.cli(A1, args=["--x", "1", "bc:c", "--bc.y", "3"]) + tyro.cli(A1, args=["--x", "1", "bc:c", "--bc.y", "3"]) def test_forward_ref_2(): - assert dcargs.cli(A2, args=["--x", "1", "bc:b", "--bc.y", "3"]) == A2( - x=1, bc=B(y=3) - ) - assert dcargs.cli(A2, args=["--x", "1", "bc:c", "--bc.z", "3"]) == A2( - x=1, bc=C(z=3) - ) + assert tyro.cli(A2, args=["--x", "1", "bc:b", "--bc.y", "3"]) == A2(x=1, bc=B(y=3)) + assert tyro.cli(A2, args=["--x", "1", "bc:c", "--bc.z", "3"]) == A2(x=1, bc=C(z=3)) with pytest.raises(SystemExit): - dcargs.cli(A2, args=["--x", "1", "bc:b", "--bc.z", "3"]) + tyro.cli(A2, args=["--x", "1", "bc:b", "--bc.z", "3"]) with pytest.raises(SystemExit): - dcargs.cli(A2, args=["--x", "1", "bc:c", "--bc.y", "3"]) + tyro.cli(A2, args=["--x", "1", "bc:c", "--bc.y", "3"]) diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 4fe599354..af5a2679e 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -8,13 +8,13 @@ import yaml from typing_extensions import Annotated -import dcargs +import tyro T = TypeVar("T") def _check_serialization_identity(cls: Type[T], instance: T) -> None: - assert dcargs.extras.from_yaml(cls, dcargs.extras.to_yaml(instance)) == instance + assert tyro.extras.from_yaml(cls, tyro.extras.to_yaml(instance)) == instance ScalarType = TypeVar("ScalarType") @@ -25,7 +25,7 @@ def test_tuple_generic_variable(): class TupleGenericVariable(Generic[ScalarType]): xyz: Tuple[ScalarType, ...] - assert dcargs.cli( + assert tyro.cli( TupleGenericVariable[int], args=["--xyz", "1", "2", "3"] ) == TupleGenericVariable((1, 2, 3)) @@ -40,7 +40,7 @@ class TupleGenericVariableHelptext(Generic[ScalarType]): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.cli(TupleGenericVariableHelptext[int], args=["--help"]) + tyro.cli(TupleGenericVariableHelptext[int], args=["--help"]) helptext = f.getvalue() assert "Helptext!" in helptext @@ -53,7 +53,7 @@ class TupleGenericVariableNoHelptext(Generic[ScalarType]): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.cli(TupleGenericVariableNoHelptext[int], args=["--help"]) + tyro.cli(TupleGenericVariableNoHelptext[int], args=["--help"]) helptext = f.getvalue() assert "Helptext!" not in helptext @@ -66,7 +66,7 @@ def test_tuple_generic_fixed(): class TupleGenericFixed(Generic[ScalarType]): xyz: Tuple[ScalarType, ScalarType, ScalarType] - assert dcargs.cli( + assert tyro.cli( TupleGenericFixed[int], args=["--xyz", "1", "2", "3"] ) == TupleGenericFixed((1, 2, 3)) @@ -90,7 +90,7 @@ class SimpleGeneric: point_continuous: Point3[float] point_discrete: Point3[int] - parsed_instance = dcargs.cli( + parsed_instance = tyro.cli( SimpleGeneric, args=[ "--point-continuous.x", @@ -119,7 +119,7 @@ class SimpleGeneric: with pytest.raises(SystemExit): # Accidentally pass in floats instead of ints for discrete - dcargs.cli( + tyro.cli( SimpleGeneric, args=[ "--point-continuous.x", @@ -149,7 +149,7 @@ class Triangle(Generic[ScalarType]): b: Point3[ScalarType] c: Point3[ScalarType] - parsed_instance = dcargs.cli( + parsed_instance = tyro.cli( Triangle[float], args=[ "--a.x", @@ -195,7 +195,7 @@ class LineSegment(Generic[ScalarType]): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.cli(LineSegment[int], args=["--help"]) + tyro.cli(LineSegment[int], args=["--help"]) helptext = f.getvalue() # Check that we don't accidentally grab docstrings from the generic alias! @@ -214,7 +214,7 @@ class Child: class DataclassGeneric(Generic[T]): child: T - parsed_instance = dcargs.cli( + parsed_instance = tyro.cli( DataclassGeneric[Child], args=["--child.a", "5", "--child.b", "7"] ) assert parsed_instance == DataclassGeneric(Child(5, 7)) @@ -239,7 +239,7 @@ class DataclassGeneric(Generic[T]): f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.cli(DataclassGeneric[Child], args=["--help"]) + tyro.cli(DataclassGeneric[Child], args=["--help"]) helptext = f.getvalue() # Check that we don't accidentally grab docstrings from the generic alias! @@ -262,7 +262,7 @@ class CommandTwo: class Subparser(Generic[T1, T2]): command: Union[T1, T2] - parsed_instance = dcargs.cli( + parsed_instance = tyro.cli( Subparser[CommandOne, CommandTwo], args="command:command-one --command.a 5".split(" "), ) @@ -273,7 +273,7 @@ class Subparser(Generic[T1, T2]): Subparser[CommandOne, CommandTwo], parsed_instance ) - parsed_instance = dcargs.cli( + parsed_instance = tyro.cli( Subparser[CommandOne, CommandTwo], args="command:command-two --command.b 7".split(" "), ) @@ -299,7 +299,7 @@ class Command(Generic[T]): class Subparser(Generic[T1, T2]): command: Union[T1, T2] - parsed_instance = dcargs.cli( + parsed_instance = tyro.cli( Subparser[Command[int], Command[float]], args="command:command-int --command.a 5 3".split(" "), ) @@ -312,7 +312,7 @@ class Subparser(Generic[T1, T2]): Subparser[Command[int], Command[float]], parsed_instance ) - parsed_instance = dcargs.cli( + parsed_instance = tyro.cli( Subparser[Command[int], Command[float]], args="command:command-float --command.a 7 2".split(" "), ) @@ -331,14 +331,14 @@ def test_serialize_missing(): class TupleGenericVariableMissing(Generic[ScalarType]): xyz: Tuple[ScalarType, ...] - x = TupleGenericVariableMissing[int](xyz=(dcargs.MISSING, dcargs.MISSING)) - assert dcargs.extras.to_yaml(x).count("!missing") == 2 + x = TupleGenericVariableMissing[int](xyz=(tyro.MISSING, tyro.MISSING)) + assert tyro.extras.to_yaml(x).count("!missing") == 2 _check_serialization_identity(TupleGenericVariableMissing[int], x) assert ( - dcargs.extras.from_yaml( - TupleGenericVariableMissing[int], dcargs.extras.to_yaml(x) + tyro.extras.from_yaml( + TupleGenericVariableMissing[int], tyro.extras.to_yaml(x) ).xyz[0] - is dcargs.MISSING + is tyro.MISSING ) @@ -362,11 +362,11 @@ class ChildClass(ActualParentClass[int]): def main(x: ActualParentClass[int] = ChildClass(5, 5)) -> ActualParentClass: return x - assert dcargs.cli(main, args="--x.x 3".split(" ")) == ChildClass(3, 5, 3) + assert tyro.cli(main, args="--x.x 3".split(" ")) == ChildClass(3, 5, 3) def test_pculbertson(): - # https://github.com/brentyi/dcargs/issues/7 + # https://github.com/brentyi/tyro/issues/7 from typing import Union @dataclasses.dataclass @@ -382,11 +382,11 @@ class Wrapper: subclass: Union[TypeA, TypeB] = TypeA(1) wrapper1 = Wrapper() # Create Wrapper object. - assert wrapper1 == dcargs.extras.from_yaml(Wrapper, dcargs.extras.to_yaml(wrapper1)) + assert wrapper1 == tyro.extras.from_yaml(Wrapper, tyro.extras.to_yaml(wrapper1)) def test_annotated(): - # https://github.com/brentyi/dcargs/issues/7 + # https://github.com/brentyi/tyro/issues/7 @dataclasses.dataclass class TypeA: @@ -397,11 +397,11 @@ class Wrapper: subclass: Annotated[TypeA, int] = TypeA(1) wrapper1 = Wrapper() # Create Wrapper object. - assert wrapper1 == dcargs.extras.from_yaml(Wrapper, dcargs.extras.to_yaml(wrapper1)) + assert wrapper1 == tyro.extras.from_yaml(Wrapper, tyro.extras.to_yaml(wrapper1)) def test_superclass(): - # https://github.com/brentyi/dcargs/issues/7 + # https://github.com/brentyi/tyro/issues/7 @dataclasses.dataclass class TypeA: @@ -416,4 +416,4 @@ class Wrapper: subclass: TypeA wrapper1 = Wrapper(TypeASubclass(3)) # Create Wrapper object. - assert wrapper1 == dcargs.extras.from_yaml(Wrapper, dcargs.extras.to_yaml(wrapper1)) + assert wrapper1 == tyro.extras.from_yaml(Wrapper, tyro.extras.to_yaml(wrapper1)) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 0fb6b1c9b..c50815899 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -12,50 +12,49 @@ import torch.nn as nn from typing_extensions import Annotated, Literal -import dcargs -import dcargs._arguments -import dcargs._strings +import tyro +import tyro._arguments +import tyro._strings def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): - dcargs.cli(f, args=args) + tyro.cli(f, args=args) - # Check dcargs.extras.get_parser(). - parser = dcargs.extras.get_parser(f) + # Check tyro.extras.get_parser(). + parser = tyro.extras.get_parser(f) assert isinstance(parser, argparse.ArgumentParser) # Returned parser should have formatting information stripped. External tools rarely # support ANSI sequences. unformatted_helptext = parser.format_help() assert ( - dcargs._strings.strip_ansi_sequences(unformatted_helptext) - == unformatted_helptext + tyro._strings.strip_ansi_sequences(unformatted_helptext) == unformatted_helptext ) unformatted_usage = parser.format_usage() - assert dcargs._strings.strip_ansi_sequences(unformatted_usage) == unformatted_usage + assert tyro._strings.strip_ansi_sequences(unformatted_usage) == unformatted_usage # Completion scripts; just smoke test for now. with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): - dcargs.cli(f, args=["--dcargs-print-completion", "bash"]) + tyro.cli(f, args=["--tyro-print-completion", "bash"]) with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): - dcargs.cli(f, args=["--dcargs-print-completion", "zsh"]) + tyro.cli(f, args=["--tyro-print-completion", "zsh"]) # Check helptext with vs without formatting. This can help catch text wrapping bugs # caused by ANSI sequences. target2 = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): - dcargs._arguments.USE_RICH = False - dcargs.cli(f, args=args) - dcargs._arguments.USE_RICH = True + tyro._arguments.USE_RICH = False + tyro.cli(f, args=args) + tyro._arguments.USE_RICH = True - if target2.getvalue() != dcargs._strings.strip_ansi_sequences(target.getvalue()): + if target2.getvalue() != tyro._strings.strip_ansi_sequences(target.getvalue()): raise AssertionError( "Potential wrapping bug! These two strings should match:\n" + target2.getvalue() + "\n\n" - + dcargs._strings.strip_ansi_sequences(target.getvalue()) + + tyro._strings.strip_ansi_sequences(target.getvalue()) ) return target2.getvalue() @@ -617,7 +616,7 @@ def test_suppressed(): @dataclasses.dataclass class Struct: a: int = 5 - b: dcargs.conf.Suppress[str] = "7" + b: tyro.conf.Suppress[str] = "7" def main(x: Any = Struct()): pass @@ -631,7 +630,7 @@ def test_suppress_manual_fixed(): @dataclasses.dataclass class Struct: a: int = 5 - b: dcargs.conf.SuppressFixed[dcargs.conf.Fixed[str]] = "7" + b: tyro.conf.SuppressFixed[tyro.conf.Fixed[str]] = "7" def main(x: Any = Struct()): pass @@ -647,7 +646,7 @@ class Struct: a: int = 5 b: Callable = lambda x: 5 - def main(x: dcargs.conf.SuppressFixed[Any] = Struct()): + def main(x: tyro.conf.SuppressFixed[Any] = Struct()): pass helptext = _get_helptext(main) diff --git a/tests/test_is_nested_type.py b/tests/test_is_nested_type.py index 6960d9a9f..a586756f4 100644 --- a/tests/test_is_nested_type.py +++ b/tests/test_is_nested_type.py @@ -2,7 +2,7 @@ import pathlib from typing import Any, Dict, List, Tuple -from dcargs._fields import MISSING_NONPROP, is_nested_type +from tyro._fields import MISSING_NONPROP, is_nested_type def test_is_nested_type_simple(): diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 5b030339f..3b58da893 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -4,7 +4,7 @@ import pytest from typing_extensions import Annotated -import dcargs +import tyro def test_omit_subcommand_prefix(): @@ -20,16 +20,16 @@ class DefaultInstanceSMTPServer: class DefaultInstanceSubparser: x: int # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] - bc: dcargs.conf.OmitSubcommandPrefixes[ + bc: tyro.conf.OmitSubcommandPrefixes[ Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] ] assert ( - dcargs.cli( + tyro.cli( DefaultInstanceSubparser, args=["--x", "1", "bc:default-instance-http-server", "--y", "5"], ) - == dcargs.cli( + == tyro.cli( DefaultInstanceSubparser, args=["--x", "1", "bc:default-instance-http-server", "--y", "5"], default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), @@ -37,11 +37,11 @@ class DefaultInstanceSubparser: == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) ) assert ( - dcargs.cli( + tyro.cli( DefaultInstanceSubparser, args=["--x", "1", "bc:default-instance-http-server", "--y", "8"], ) - == dcargs.cli( + == tyro.cli( DefaultInstanceSubparser, args=["--x", "1", "bc:default-instance-http-server", "--y", "8"], default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), @@ -62,16 +62,16 @@ class DefaultInstanceSMTPServer: @dataclasses.dataclass class DefaultInstanceSubparser: x: int - bc: dcargs.conf.AvoidSubcommands[ + bc: tyro.conf.AvoidSubcommands[ Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] ] assert ( - dcargs.cli( + tyro.cli( DefaultInstanceSubparser, args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], ) - == dcargs.cli( + == tyro.cli( DefaultInstanceSubparser, args=["--x", "1", "--bc.y", "5"], default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), @@ -79,11 +79,11 @@ class DefaultInstanceSubparser: == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) ) assert ( - dcargs.cli( + tyro.cli( DefaultInstanceSubparser, args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], ) - == dcargs.cli( + == tyro.cli( DefaultInstanceSubparser, args=["--bc.y", "8"], default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), @@ -107,29 +107,29 @@ class DefaultInstanceSubparser: bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] assert ( - dcargs.cli( + tyro.cli( DefaultInstanceSubparser, args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], ) - == dcargs.cli( - dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], + == tyro.cli( + tyro.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "--bc.y", "5"], default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) ) - assert dcargs.cli( + assert tyro.cli( DefaultInstanceSubparser, args=["bc:default-instance-smtp-server", "--bc.z", "3"], default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) assert ( - dcargs.cli( - dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], + tyro.cli( + tyro.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], ) - == dcargs.cli( - dcargs.conf.AvoidSubcommands[DefaultInstanceSubparser], + == tyro.cli( + tyro.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--bc.y", "8"], default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), ) @@ -150,8 +150,8 @@ class B: @dataclasses.dataclass class Nested2: subcommand: Union[ - Annotated[A, dcargs.conf.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.conf.subcommand("command-b", default=B(9))], + Annotated[A, tyro.conf.subcommand("command-a", default=A(7))], + Annotated[B, tyro.conf.subcommand("command-b", default=B(9))], ] @dataclasses.dataclass @@ -162,11 +162,11 @@ class Nested1: class Parent: nested1: Nested1 - assert dcargs.cli( + assert tyro.cli( Parent, args="nested1.nested2.subcommand:command-a".split(" "), ) == Parent(Nested1(Nested2(A(7)))) - assert dcargs.cli( + assert tyro.cli( Parent, args=( "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( @@ -175,11 +175,11 @@ class Parent: ), ) == Parent(Nested1(Nested2(A(3)))) - assert dcargs.cli( + assert tyro.cli( Parent, args="nested1.nested2.subcommand:command-b".split(" "), ) == Parent(Nested1(Nested2(B(9)))) - assert dcargs.cli( + assert tyro.cli( Parent, args=( "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( @@ -209,8 +209,8 @@ class Nested2(Generic[T]): class Nested1: nested2: Nested2[ Union[ - Annotated[A, dcargs.conf.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.conf.subcommand("command-b", default=B(9))], + Annotated[A, tyro.conf.subcommand("command-a", default=A(7))], + Annotated[B, tyro.conf.subcommand("command-b", default=B(9))], ] ] @@ -218,11 +218,11 @@ class Nested1: class Parent: nested1: Nested1 - assert dcargs.cli( + assert tyro.cli( Parent, args="nested1.nested2.subcommand:command-a".split(" "), ) == Parent(Nested1(Nested2(A(7)))) - assert dcargs.cli( + assert tyro.cli( Parent, args=( "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( @@ -231,11 +231,11 @@ class Parent: ), ) == Parent(Nested1(Nested2(A(3)))) - assert dcargs.cli( + assert tyro.cli( Parent, args="nested1.nested2.subcommand:command-b".split(" "), ) == Parent(Nested1(Nested2(B(9)))) - assert dcargs.cli( + assert tyro.cli( Parent, args=( "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( @@ -260,8 +260,8 @@ class B: @dataclasses.dataclass class Nested2(Generic[T]): subcommand: Union[ - Annotated[T, dcargs.conf.subcommand("command-a", default=A(7))], - Annotated[B, dcargs.conf.subcommand("command-b", default=B(9))], + Annotated[T, tyro.conf.subcommand("command-a", default=A(7))], + Annotated[B, tyro.conf.subcommand("command-b", default=B(9))], ] @dataclasses.dataclass @@ -272,11 +272,11 @@ class Nested1: class Parent: nested1: Nested1 - assert dcargs.cli( + assert tyro.cli( Parent, args="nested1.nested2.subcommand:command-a".split(" "), ) == Parent(Nested1(Nested2(A(7)))) - assert dcargs.cli( + assert tyro.cli( Parent, args=( "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( @@ -285,11 +285,11 @@ class Parent: ), ) == Parent(Nested1(Nested2(A(3)))) - assert dcargs.cli( + assert tyro.cli( Parent, args="nested1.nested2.subcommand:command-b".split(" "), ) == Parent(Nested1(Nested2(B(9)))) - assert dcargs.cli( + assert tyro.cli( Parent, args=( "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( @@ -306,14 +306,14 @@ def test_flag(): class A: x: bool - assert dcargs.cli( + assert tyro.cli( A, args=["--x"], default=A(False), ) == A(True) - assert dcargs.cli( - dcargs.conf.FlagConversionOff[A], + assert tyro.cli( + tyro.conf.FlagConversionOff[A], args=["--x", "True"], default=A(False), ) == A(True) @@ -324,17 +324,17 @@ def test_fixed(): @dataclasses.dataclass class A: - x: dcargs.conf.Fixed[bool] + x: tyro.conf.Fixed[bool] - assert dcargs.cli( + assert tyro.cli( A, args=[], default=A(True), ) == A(True) with pytest.raises(SystemExit): - assert dcargs.cli( - dcargs.conf.FlagConversionOff[A], + assert tyro.cli( + tyro.conf.FlagConversionOff[A], args=["--x", "True"], default=A(False), ) == A(True) @@ -347,15 +347,15 @@ def test_fixed_recursive(): class A: x: bool - assert dcargs.cli( + assert tyro.cli( A, args=["--x"], default=A(False), ) == A(True) with pytest.raises(SystemExit): - assert dcargs.cli( - dcargs.conf.Fixed[dcargs.conf.FlagConversionOff[A]], + assert tyro.cli( + tyro.conf.Fixed[tyro.conf.FlagConversionOff[A]], args=["--x", "True"], default=A(False), ) == A(True) diff --git a/tests/test_missing.py b/tests/test_missing.py index 16cee1687..75d6fa632 100644 --- a/tests/test_missing.py +++ b/tests/test_missing.py @@ -1,32 +1,32 @@ -"""Tests for dcargs.MISSING.""" +"""Tests for tyro.MISSING.""" import dataclasses from typing import Tuple import pytest -import dcargs +import tyro def test_missing(): - def main(a: int = 5, b: int = dcargs.MISSING, c: int = 3) -> Tuple[int, int, int]: + def main(a: int = 5, b: int = tyro.MISSING, c: int = 3) -> Tuple[int, int, int]: return a, b, c with pytest.raises(SystemExit): - dcargs.cli(main, args=[]) - assert dcargs.cli(main, args=["--b", "7"]) == (5, 7, 3) + tyro.cli(main, args=[]) + assert tyro.cli(main, args=["--b", "7"]) == (5, 7, 3) def test_missing_dataclass(): @dataclasses.dataclass class Args2: a: int = 5 - b: int = dcargs.MISSING + b: int = tyro.MISSING c: int = 3 with pytest.raises(SystemExit): - dcargs.cli(Args2, args=[]) - assert dcargs.cli(Args2, args=["--b", "7"]) == Args2(5, 7, 3) + tyro.cli(Args2, args=[]) + assert tyro.cli(Args2, args=["--b", "7"]) == Args2(5, 7, 3) def test_missing_default(): @@ -37,15 +37,15 @@ class Args2: c: int with pytest.raises(SystemExit): - dcargs.cli( + tyro.cli( Args2, args=[], - default=Args2(5, dcargs.MISSING, 3), + default=Args2(5, tyro.MISSING, 3), ) - assert dcargs.cli( + assert tyro.cli( Args2, args=["--b", "7"], - default=Args2(5, dcargs.MISSING, 3), + default=Args2(5, tyro.MISSING, 3), ) == Args2(5, 7, 3) @@ -61,13 +61,13 @@ class Parent: child: Child with pytest.raises(SystemExit): - dcargs.cli( + tyro.cli( Parent, args=[], - default=Parent(child=dcargs.MISSING), + default=Parent(child=tyro.MISSING), ) - assert dcargs.cli( + assert tyro.cli( Parent, args=["--child.a", "5", "--child.b", "7", "--child.c", "3"], - default=Parent(child=dcargs.MISSING), + default=Parent(child=tyro.MISSING), ) == Parent(Child(5, 7, 3)) diff --git a/tests/test_nested.py b/tests/test_nested.py index 982ead888..8df60ed6f 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -5,7 +5,7 @@ from frozendict import frozendict # type: ignore from typing_extensions import Annotated -import dcargs +import tyro def test_nested(): @@ -18,9 +18,9 @@ class Nested: x: int b: B - assert dcargs.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) + assert tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) with pytest.raises(SystemExit): - dcargs.cli(Nested, args=["--x", "1"]) + tyro.cli(Nested, args=["--x", "1"]) def test_nested_annotated(): @@ -33,9 +33,9 @@ class Nested: x: int b: Annotated[B, "this should be ignored"] - assert dcargs.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) + assert tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) with pytest.raises(SystemExit): - dcargs.cli(Nested, args=["--x", "1"]) + tyro.cli(Nested, args=["--x", "1"]) def test_nested_accidental_underscores(): @@ -49,18 +49,14 @@ class Nested: child_struct: B assert ( - dcargs.cli(Nested, args=["--x", "1", "--child-struct.arg-name", "three_five"]) - == dcargs.cli( - Nested, args=["--x", "1", "--child_struct.arg_name", "three_five"] - ) - == dcargs.cli( - Nested, args=["--x", "1", "--child_struct.arg-name", "three_five"] - ) - == dcargs.cli(Nested, args=["--x", "1", "--child_struct.arg_name=three_five"]) + tyro.cli(Nested, args=["--x", "1", "--child-struct.arg-name", "three_five"]) + == tyro.cli(Nested, args=["--x", "1", "--child_struct.arg_name", "three_five"]) + == tyro.cli(Nested, args=["--x", "1", "--child_struct.arg-name", "three_five"]) + == tyro.cli(Nested, args=["--x", "1", "--child_struct.arg_name=three_five"]) == Nested(x=1, child_struct=B(arg_name="three_five")) ) with pytest.raises(SystemExit): - dcargs.cli(Nested, args=["--x", "1"]) + tyro.cli(Nested, args=["--x", "1"]) def test_nested_default(): @@ -73,7 +69,7 @@ class Nested: x: int = 2 b: B = B() - assert dcargs.cli(Nested, args=[], default=Nested(x=1, b=B(y=2))) == Nested( + assert tyro.cli(Nested, args=[], default=Nested(x=1, b=B(y=2))) == Nested( x=1, b=B(y=2) ) @@ -90,10 +86,10 @@ class Nested: assert ( Nested(x=1, b=B(y=3)) - == dcargs.cli(Nested, args=["--x", "1", "--b.y", "3"]) - == dcargs.cli(Nested, args=[], default=Nested(x=1, b=B(y=3))) + == tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) + == tyro.cli(Nested, args=[], default=Nested(x=1, b=B(y=3))) ) - assert dcargs.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=3)) + assert tyro.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=3)) def test_default_nested(): @@ -106,8 +102,8 @@ class Nested: x: int b: B = B(y=5) - assert dcargs.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) - assert dcargs.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) + assert tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) + assert tyro.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) def test_double_default_nested(): @@ -124,10 +120,10 @@ class Grandparent: x: int b: Parent = Parent(Child(y=5)) - assert dcargs.cli(Grandparent, args=["--x", "1", "--b.c.y", "3"]) == Grandparent( + assert tyro.cli(Grandparent, args=["--x", "1", "--b.c.y", "3"]) == Grandparent( x=1, b=Parent(Child(y=3)) ) - assert dcargs.cli(Grandparent, args=["--x", "1"]) == Grandparent( + assert tyro.cli(Grandparent, args=["--x", "1"]) == Grandparent( x=1, b=Parent(Child(y=5)) ) @@ -142,8 +138,8 @@ class Nested: x: int b: B = dataclasses.field(default_factory=lambda: B(y=5)) - assert dcargs.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) - assert dcargs.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) + assert tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) + assert tyro.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) def test_optional_nested(): @@ -157,17 +153,17 @@ class OptionalNested: x: int b: Optional[OptionalNestedChild] = None - assert dcargs.cli(OptionalNested, args=["--x", "1"]) == OptionalNested(x=1, b=None) + assert tyro.cli(OptionalNested, args=["--x", "1"]) == OptionalNested(x=1, b=None) with pytest.raises(SystemExit): - dcargs.cli( + tyro.cli( OptionalNested, args=["--x", "1", "b:optional-nested-child", "--b.y", "3"] ) with pytest.raises(SystemExit): - dcargs.cli( + tyro.cli( OptionalNested, args=["--x", "1", "b:optional-nested-child", "--b.z", "3"] ) - assert dcargs.cli( + assert tyro.cli( OptionalNested, args=["--x", "1", "b:optional-nested-child", "--b.y", "2", "--b.z", "3"], ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) @@ -187,22 +183,22 @@ class Subparser: x: int bc: Union[HTTPServer, SMTPServer] - assert dcargs.cli( + assert tyro.cli( Subparser, args=["--x", "1", "bc:http-server", "--bc.y", "3"] ) == Subparser(x=1, bc=HTTPServer(y=3)) - assert dcargs.cli( + assert tyro.cli( Subparser, args=["--x", "1", "bc:smtp-server", "--bc.z", "3"] ) == Subparser(x=1, bc=SMTPServer(z=3)) with pytest.raises(SystemExit): # Missing subcommand. - dcargs.cli(Subparser, args=["--x", "1"]) + tyro.cli(Subparser, args=["--x", "1"]) with pytest.raises(SystemExit): # Wrong field. - dcargs.cli(Subparser, args=["--x", "1", "bc:http-server", "--bc.z", "3"]) + tyro.cli(Subparser, args=["--x", "1", "bc:http-server", "--bc.z", "3"]) with pytest.raises(SystemExit): # Wrong field. - dcargs.cli(Subparser, args=["--x", "1", "bc:smtp-server", "--bc.y", "3"]) + tyro.cli(Subparser, args=["--x", "1", "bc:smtp-server", "--bc.y", "3"]) def test_subparser_root(): @@ -219,7 +215,7 @@ class Subparser: x: int bc: Union[HTTPServer, SMTPServer] - assert dcargs.cli( + assert tyro.cli( Union[HTTPServer, SMTPServer], args=["http-server", "--y", "3"] # type: ignore ) == HTTPServer(y=3) @@ -241,20 +237,20 @@ class DefaultSubparser: ) assert ( - dcargs.cli( + tyro.cli( DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "5"] ) - == dcargs.cli(DefaultSubparser, args=["--x", "1"]) + == tyro.cli(DefaultSubparser, args=["--x", "1"]) == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=5)) ) - assert dcargs.cli( + assert tyro.cli( DefaultSubparser, args=["--x", "1", "bc:default-smtp-server", "--bc.z", "3"] ) == DefaultSubparser(x=1, bc=DefaultSMTPServer(z=3)) assert ( - dcargs.cli( + tyro.cli( DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "8"] ) - == dcargs.cli( + == tyro.cli( DefaultSubparser, args=[], default=DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)), @@ -263,9 +259,9 @@ class DefaultSubparser: ) with pytest.raises(SystemExit): - dcargs.cli(DefaultSubparser, args=["--x", "1", "b", "--bc.z", "3"]) + tyro.cli(DefaultSubparser, args=["--x", "1", "b", "--bc.z", "3"]) with pytest.raises(SystemExit): - dcargs.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) + tyro.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) def test_subparser_with_default_alternate(): @@ -283,33 +279,33 @@ class DefaultInstanceSubparser: bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] assert ( - dcargs.cli( + tyro.cli( DefaultInstanceSubparser, args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], ) - == dcargs.cli( + == tyro.cli( DefaultInstanceSubparser, args=[], default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), ) - == dcargs.cli( + == tyro.cli( DefaultInstanceSubparser, args=["bc:default-instance-http-server"], default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) ) - assert dcargs.cli( + assert tyro.cli( DefaultInstanceSubparser, args=["bc:default-instance-smtp-server", "--bc.z", "3"], default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) assert ( - dcargs.cli( + tyro.cli( DefaultInstanceSubparser, args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], ) - == dcargs.cli( + == tyro.cli( DefaultInstanceSubparser, args=[], default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)), @@ -318,9 +314,9 @@ class DefaultInstanceSubparser: ) with pytest.raises(SystemExit): - dcargs.cli(DefaultInstanceSubparser, args=["--x", "1", "b", "--bc.z", "3"]) + tyro.cli(DefaultInstanceSubparser, args=["--x", "1", "b", "--bc.z", "3"]) with pytest.raises(SystemExit): - dcargs.cli(DefaultInstanceSubparser, args=["--x", "1", "c", "--bc.y", "3"]) + tyro.cli(DefaultInstanceSubparser, args=["--x", "1", "c", "--bc.y", "3"]) def test_optional_subparser(): @@ -337,25 +333,25 @@ class OptionalSubparser: x: int bc: Optional[Union[OptionalHTTPServer, OptionalSMTPServer]] - assert dcargs.cli( + assert tyro.cli( OptionalSubparser, args=["--x", "1", "bc:optional-http-server", "--bc.y", "3"] ) == OptionalSubparser(x=1, bc=OptionalHTTPServer(y=3)) - assert dcargs.cli( + assert tyro.cli( OptionalSubparser, args=["--x", "1", "bc:optional-smtp-server", "--bc.z", "3"] ) == OptionalSubparser(x=1, bc=OptionalSMTPServer(z=3)) - assert dcargs.cli( + assert tyro.cli( OptionalSubparser, args=["--x", "1", "bc:None"] ) == OptionalSubparser(x=1, bc=None) with pytest.raises(SystemExit): # Wrong field. - dcargs.cli( + tyro.cli( OptionalSubparser, args=["--x", "1", "bc:optional-http-server", "--bc.z", "3"], ) with pytest.raises(SystemExit): # Wrong field. - dcargs.cli( + tyro.cli( OptionalSubparser, args=["--x", "1", "bc:optional-smtp-server", "--bc.y", "3"], ) @@ -383,8 +379,8 @@ class DefaultFactoryPostInitArgs: ) assert ( - dcargs.cli(NoDefaultPostInitArgs, args=["--inner.x", "5"]).inner - == dcargs.cli(DefaultFactoryPostInitArgs, args=["--inner.x", "5"]).inner + tyro.cli(NoDefaultPostInitArgs, args=["--inner.x", "5"]).inner + == tyro.cli(DefaultFactoryPostInitArgs, args=["--inner.x", "5"]).inner == DataclassWithDynamicDefault(x=5, y=5) ) @@ -409,27 +405,27 @@ class MultipleSubparsers: c: Union[Subcommand1, Subcommand2, Subcommand3] with pytest.raises(SystemExit): - dcargs.cli(MultipleSubparsers, args=[]) + tyro.cli(MultipleSubparsers, args=[]) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args="a:subcommand1 b:subcommand2 c:subcommand3".split(" ") ) == MultipleSubparsers(Subcommand1(), Subcommand2(), Subcommand3()) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args="a:subcommand1 --a.x 5 b:subcommand2 --b.y 7 c:subcommand3 --c.z 3".split( " " ), ) == MultipleSubparsers(Subcommand1(x=5), Subcommand2(y=7), Subcommand3(z=3)) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args="a:subcommand2 --a.y 5 b:subcommand1 --b.x 7 c:subcommand3 --c.z 3".split( " " ), ) == MultipleSubparsers(Subcommand2(y=5), Subcommand1(x=7), Subcommand3(z=3)) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args="a:subcommand3 --a.z 5 b:subcommand1 --b.x 7 c:subcommand3 --c.z 3".split( " " @@ -452,38 +448,38 @@ class Subcommand3: @dataclasses.dataclass class MultipleSubparsers: - a: Union[Subcommand1, Subcommand2, Subcommand3] = Subcommand1(dcargs.MISSING) + a: Union[Subcommand1, Subcommand2, Subcommand3] = Subcommand1(tyro.MISSING) b: Union[Subcommand1, Subcommand2, Subcommand3] = Subcommand2(7) c: Union[Subcommand1, Subcommand2, Subcommand3] = Subcommand3(3) with pytest.raises(SystemExit): - dcargs.cli( + tyro.cli( MultipleSubparsers, args=[], ) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args=["a:subcommand1", "--a.x", "5"], ) == MultipleSubparsers(Subcommand1(x=5), Subcommand2(y=7), Subcommand3(z=3)) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args="a:subcommand1 --a.x 3".split(" "), ) == MultipleSubparsers(Subcommand1(x=3), Subcommand2(y=7), Subcommand3(z=3)) with pytest.raises(SystemExit): - dcargs.cli( + tyro.cli( MultipleSubparsers, args=[], default=MultipleSubparsers( Subcommand1(), Subcommand2(), - Subcommand3(dcargs.MISSING), + Subcommand3(tyro.MISSING), ), ) with pytest.raises(SystemExit): - dcargs.cli( + tyro.cli( MultipleSubparsers, args=[ "a:subcommand1", @@ -491,45 +487,45 @@ class MultipleSubparsers: default=MultipleSubparsers( Subcommand1(), Subcommand2(), - Subcommand3(dcargs.MISSING), + Subcommand3(tyro.MISSING), ), ) with pytest.raises(SystemExit): - dcargs.cli( + tyro.cli( MultipleSubparsers, args=["a:subcommand1", "b:subcommand2"], default=MultipleSubparsers( Subcommand1(), Subcommand2(), - Subcommand3(dcargs.MISSING), + Subcommand3(tyro.MISSING), ), ) with pytest.raises(SystemExit): - dcargs.cli( + tyro.cli( MultipleSubparsers, args=["a:subcommand1", "b:subcommand2", "c:subcommand3"], default=MultipleSubparsers( Subcommand1(), Subcommand2(), - Subcommand3(dcargs.MISSING), + Subcommand3(tyro.MISSING), ), ) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args=["a:subcommand1", "b:subcommand2", "c:subcommand3", "--c.z", "3"], default=MultipleSubparsers( Subcommand1(), Subcommand2(), - Subcommand3(dcargs.MISSING), + Subcommand3(tyro.MISSING), ), ) == MultipleSubparsers(Subcommand1(x=0), Subcommand2(y=1), Subcommand3(z=3)) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args=["a:subcommand1", "b:subcommand2", "c:subcommand2"], default=MultipleSubparsers( Subcommand1(), Subcommand2(), - Subcommand3(dcargs.MISSING), + Subcommand3(tyro.MISSING), ), ) == MultipleSubparsers(Subcommand1(x=0), Subcommand2(y=1), Subcommand2(y=1)) @@ -549,23 +545,23 @@ class Subcommand2: @dataclasses.dataclass(frozen=True) class MultipleSubparsers: - a: Union[Subcommand1, Subcommand2] = Subcommand2(Subcommand1(dcargs.MISSING)) + a: Union[Subcommand1, Subcommand2] = Subcommand2(Subcommand1(tyro.MISSING)) with pytest.raises(SystemExit): - dcargs.cli(MultipleSubparsers, args=[]) + tyro.cli(MultipleSubparsers, args=[]) with pytest.raises(SystemExit): - dcargs.cli(MultipleSubparsers, args=["a:subcommand2"]) + tyro.cli(MultipleSubparsers, args=["a:subcommand2"]) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args="a:subcommand1 --a.x 3".split(" ") ) == MultipleSubparsers(Subcommand1(3)) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args="a:subcommand2 a.y:subcommand3 --a.y.z 2".split(" ") ) == MultipleSubparsers(Subcommand2(Subcommand3())) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args="a:subcommand2 a.y:subcommand3 --a.y.z 7".split(" ") ) == MultipleSubparsers(Subcommand2(Subcommand3(7))) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args="a:subcommand2 a.y:subcommand1 --a.y.x 7".split(" ") ) == MultipleSubparsers(Subcommand2(Subcommand1(7))) @@ -589,19 +585,19 @@ class MultipleSubparsers: b: Union[Subcommand1, Subcommand2] with pytest.raises(SystemExit): - dcargs.cli(MultipleSubparsers, args=[]) - assert dcargs.cli( + tyro.cli(MultipleSubparsers, args=[]) + assert tyro.cli( MultipleSubparsers, args="a:subcommand1 b:subcommand1".split(" ") ) == MultipleSubparsers(Subcommand1(), Subcommand1()) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args="a:subcommand1 b:subcommand2 b.y:subcommand1".split(" "), ) == MultipleSubparsers(Subcommand1(), Subcommand2(Subcommand1())) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args="a:subcommand2 a.y:subcommand1 b:subcommand2 b.y:subcommand1".split(" "), ) == MultipleSubparsers(Subcommand2(Subcommand1()), Subcommand2(Subcommand1())) - assert dcargs.cli( + assert tyro.cli( MultipleSubparsers, args=( "a:subcommand2 a.y:subcommand1 --a.y.x 3 b:subcommand2 b.y:subcommand1" @@ -626,7 +622,7 @@ class Location: def main(x: Tuple[Tuple[Color], Location, float]): return x - assert dcargs.cli( + assert tyro.cli( main, args=( "--x.0.0.r 255 --x.0.0.g 0 --x.0.0.b 0 --x.1.x 5.0 --x.1.y 0.0" @@ -645,14 +641,14 @@ class A(Generic[T]): def main(x: Union[A[int], A[float]]) -> Any: return x - assert dcargs.cli(main, args="x:a-float --x.x 3.2".split(" ")) == A(3.2) - assert dcargs.cli(main, args="x:a-int --x.x 3".split(" ")) == A(3) + assert tyro.cli(main, args="x:a-float --x.x 3.2".split(" ")) == A(3.2) + assert tyro.cli(main, args="x:a-int --x.x 3".split(" ")) == A(3) def main_with_default(x: Union[A[int], A[float]] = A(5)) -> Any: return x - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main_with_default, args=[]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main_with_default, args=[]) def test_generic_inherited(): @@ -675,7 +671,7 @@ class ActualParentClass(Generic[T]): class ChildClass(UnrelatedParentClass, ActualParentClass[int]): pass - assert dcargs.cli( + assert tyro.cli( ChildClass, args=["--x", "1", "--y", "2", "--z", "3"] ) == ChildClass(x=1, y=2, z=3) @@ -701,11 +697,11 @@ class Nested1: class Parent: nested1: Nested1 - assert dcargs.cli( + assert tyro.cli( Parent, args="nested1.nested2.subcommand:a --nested1.nested2.subcommand.a 3".split(" "), ) == Parent(Nested1(Nested2(A(3)))) - assert dcargs.cli( + assert tyro.cli( Parent, args="nested1.nested2.subcommand:b --nested1.nested2.subcommand.b 7".split(" "), ) == Parent(Nested1(Nested2(B(7)))) @@ -722,13 +718,13 @@ def main( ): return x - assert hash(dcargs.cli(main, args="--x.num-epochs 10".split(" "))) == hash( + assert hash(tyro.cli(main, args="--x.num-epochs 10".split(" "))) == hash( frozendict({"num_epochs": 10, "batch_size": 64}) ) def test_nested_in_subparser(): - # https://github.com/brentyi/dcargs/issues/9 + # https://github.com/brentyi/tyro/issues/9 @dataclasses.dataclass(frozen=True) class Subtype: data: int = 1 @@ -745,10 +741,8 @@ class TypeB: class Wrapper: supertype: Union[TypeA, TypeB] = TypeA() - assert dcargs.cli(Wrapper, args=[]) == Wrapper() + assert tyro.cli(Wrapper, args=[]) == Wrapper() assert ( - dcargs.cli( - Wrapper, args="supertype:type-a --supertype.subtype.data 1".split(" ") - ) + tyro.cli(Wrapper, args="supertype:type-a --supertype.subtype.data 1".split(" ")) == Wrapper() ) diff --git a/tests/test_nested_in_containers.py b/tests/test_nested_in_containers.py index 56c3dfded..bf0f1bf5e 100644 --- a/tests/test_nested_in_containers.py +++ b/tests/test_nested_in_containers.py @@ -4,7 +4,7 @@ import pytest -import dcargs +import tyro @dataclasses.dataclass(frozen=True) @@ -18,7 +18,7 @@ def test_nested_tuple_fixed_single(): def main(x: Tuple[Color]) -> Any: return x - assert dcargs.cli(main, args="--x.0.r 255 --x.0.g 127 --x.0.b 5".split(" ")) == ( + assert tyro.cli(main, args="--x.0.r 255 --x.0.g 127 --x.0.b 5".split(" ")) == ( Color(255, 127, 5), ) @@ -27,7 +27,7 @@ def test_nested_tuple_fixed_two(): def main(x: Tuple[Color, Color]) -> Any: return x - assert dcargs.cli( + assert tyro.cli( main, args=( "--x.0.r 255 --x.0.g 127 --x.0.b 5 --x.1.r 255 --x.1.g 127 --x.1.b 0" @@ -42,7 +42,7 @@ def test_nested_tuple_fixed_three(): def main(x: Tuple[Color, int, Color]) -> Any: return x - assert dcargs.cli( + assert tyro.cli( main, args=( "--x.0.r 255 --x.0.g 127 --x.0.b 5 --x.1 94709 --x.2.r 255 --x.2.g 127" @@ -59,7 +59,7 @@ def test_nested_tuple_recursive(): def main(x: Tuple[Color, Tuple[Color, Color]]) -> Any: return x - assert dcargs.cli( + assert tyro.cli( main, args=( "--x.0.r 255 --x.0.g 127 --x.0.b 5 --x.1.0.r 255 --x.1.0.g 127 --x.1.0.b 0" @@ -79,8 +79,8 @@ def test_tuple_bad(): def main(x: Tuple[Color, ...]) -> None: pass - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main, args=[]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=[]) def test_set_bad(): @@ -88,16 +88,16 @@ def test_set_bad(): def main(x: Set[Color]) -> None: pass - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main, args=[]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=[]) def test_set_ok(): def main(x: Set[Color] = {Color(255, 0, 0)}) -> Any: return x - assert dcargs.cli(main, args=[]) == {Color(255, 0, 0)} - assert dcargs.cli(main, args="--x.0.r 127".split(" ")) == {Color(127, 0, 0)} + assert tyro.cli(main, args=[]) == {Color(255, 0, 0)} + assert tyro.cli(main, args="--x.0.r 127".split(" ")) == {Color(127, 0, 0)} def test_list_bad(): @@ -105,48 +105,48 @@ def test_list_bad(): def main(x: List[Color]) -> None: pass - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main, args=[]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=[]) def test_list_ok(): def main(x: List[Color] = [Color(255, 0, 0)]) -> Any: return x - assert dcargs.cli(main, args=[]) == [Color(255, 0, 0)] - assert dcargs.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] + assert tyro.cli(main, args=[]) == [Color(255, 0, 0)] + assert tyro.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] def test_list_object(): def main(x: List[object] = [Color(255, 0, 0)]) -> Any: return x - assert dcargs.cli(main, args=[]) == [Color(255, 0, 0)] - assert dcargs.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] + assert tyro.cli(main, args=[]) == [Color(255, 0, 0)] + assert tyro.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] def test_list_any(): def main(x: List[Any] = [Color(255, 0, 0)]) -> Any: return x - assert dcargs.cli(main, args=[]) == [Color(255, 0, 0)] - assert dcargs.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] + assert tyro.cli(main, args=[]) == [Color(255, 0, 0)] + assert tyro.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] def test_tuple_in_list(): def main(x: List[Tuple[Color]] = [(Color(255, 0, 0),)]) -> Any: return x - assert dcargs.cli(main, args=[]) == [(Color(255, 0, 0),)] - assert dcargs.cli(main, args="--x.0.0.r 127".split(" ")) == [(Color(127, 0, 0),)] + assert tyro.cli(main, args=[]) == [(Color(255, 0, 0),)] + assert tyro.cli(main, args="--x.0.0.r 127".split(" ")) == [(Color(127, 0, 0),)] def test_tuple_variable(): def main(x: Tuple[Color, ...] = (Color(255, 0, 0), Color(255, 0, 127))) -> Any: return x - assert dcargs.cli(main, args=[]) == (Color(255, 0, 0), Color(255, 0, 127)) - assert dcargs.cli(main, args="--x.0.r 127".split(" ")) == ( + assert tyro.cli(main, args=[]) == (Color(255, 0, 0), Color(255, 0, 127)) + assert tyro.cli(main, args="--x.0.r 127".split(" ")) == ( Color(127, 0, 0), Color(255, 0, 127), ) @@ -156,8 +156,8 @@ def test_dict_bad(): def main(x: Dict[str, Color]) -> Any: return x - with pytest.raises(dcargs.UnsupportedTypeAnnotationError): - dcargs.cli(main, args=[]) + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=[]) def test_dict_ok(): @@ -170,8 +170,8 @@ def main( ) -> Any: return x - assert dcargs.cli(main, args=[])["green"] == Color(0, 255, 0) - assert dcargs.cli(main, args="--x.green.g 127".split(" "))["green"] == Color( + assert tyro.cli(main, args=[])["green"] == Color(0, 255, 0) + assert tyro.cli(main, args="--x.green.g 127".split(" "))["green"] == Color( 0, 127, 0 ) @@ -186,8 +186,8 @@ def main( ) -> Any: return x - assert dcargs.cli(main, args=[])[1] == Color(0, 255, 0) - assert dcargs.cli(main, args="--x.1.g 127".split(" "))[1] == Color(0, 127, 0) + assert tyro.cli(main, args=[])[1] == Color(0, 255, 0) + assert tyro.cli(main, args="--x.1.g 127".split(" "))[1] == Color(0, 127, 0) def test_dict_key_enum(): @@ -205,10 +205,10 @@ def main( ) -> Any: return x - assert dcargs.cli(main, args=[])[ColorType.GREEN] == Color(0, 255, 0) - assert dcargs.cli(main, args="--x.GREEN.g 127".split(" "))[ - ColorType.GREEN - ] == Color(0, 127, 0) + assert tyro.cli(main, args=[])[ColorType.GREEN] == Color(0, 255, 0) + assert tyro.cli(main, args="--x.GREEN.g 127".split(" "))[ColorType.GREEN] == Color( + 0, 127, 0 + ) def test_dict_nested(): @@ -222,8 +222,8 @@ def main( ) -> Any: return x - assert dcargs.cli(main, args=[])["green"] == (Color(0, 255, 0), 10) - assert dcargs.cli(main, args="--x.green.0.g 127 --x.green.1 2".split(" "))[ + assert tyro.cli(main, args=[])["green"] == (Color(0, 255, 0), 10) + assert tyro.cli(main, args="--x.green.0.g 127 --x.green.1 2".split(" "))[ "green" ] == (Color(0, 127, 0), 2) @@ -240,7 +240,7 @@ class GenericColor(Generic[ScalarType]): def main(x: Tuple[GenericColor[float], GenericColor[int]]) -> Any: return x - assert dcargs.cli( + assert tyro.cli( main, args=( "--x.0.r 0.5 --x.0.g 0.2 --x.0.b 0.3 --x.1.r 25 --x.1.g 2 --x.1.b 3".split( @@ -267,7 +267,7 @@ def main( ) -> Any: return x - assert dcargs.cli( + assert tyro.cli( main, args=( "--x.0.r 0.5 --x.0.g 0.2 --x.0.b 0.3 --x.1.r 25 --x.1.g 2 --x.1.b 3".split( @@ -294,7 +294,7 @@ def main( ) -> Any: return x - assert dcargs.cli( + assert tyro.cli( main, args=( "--x.0.r 0.5 --x.0.g 0.9 --x.0.b 0.3 --x.1.r 25 --x.1.g 2 --x.1.b 3".split( @@ -321,10 +321,10 @@ def main( ) -> Any: return x - assert dcargs.cli(main, args="--x.float.g 0.1".split(" "),)[ + assert tyro.cli(main, args="--x.float.g 0.1".split(" "),)[ "float" ] == GenericColor(0.5, 0.1, 0.3) - assert dcargs.cli( + assert tyro.cli( main, args="--x.int.g 0".split(" "), ) == {"float": GenericColor(0.5, 0.2, 0.3), "int": GenericColor(25, 0, 3)} @@ -349,10 +349,10 @@ def main( ) -> Any: return x - assert dcargs.cli(main, args="--x.hello.float.g 0.1".split(" "),)["hello"][ + assert tyro.cli(main, args="--x.hello.float.g 0.1".split(" "),)["hello"][ "float" ] == GenericColor(0.5, 0.1, 0.3) - assert dcargs.cli(main, args="--x.hello.int.g 0".split(" "),) == { + assert tyro.cli(main, args="--x.hello.int.g 0".split(" "),) == { "hello": {"float": GenericColor(0.5, 0.2, 0.3), "int": GenericColor(25, 0, 3)} } @@ -368,6 +368,6 @@ def main( ) -> Any: return x - assert dcargs.cli(main, args="--x.hello.a.g 1".split(" "),)["hello"][ + assert tyro.cli(main, args="--x.hello.a.g 1".split(" "),)["hello"][ "a" ] == Color(5, 1, 3) diff --git a/tests/test_new_style_annotations_only_py310.py b/tests/test_new_style_annotations_only_py310.py index 162c2064a..8b99a75e4 100644 --- a/tests/test_new_style_annotations_only_py310.py +++ b/tests/test_new_style_annotations_only_py310.py @@ -1,38 +1,37 @@ -import dataclasses from typing import Any, Literal import pytest -import dcargs +import tyro def test_union_basic(): def main(x: int | str) -> int | str: return x - assert dcargs.cli(main, args=["--x", "5"]) == 5 - assert dcargs.cli(main, args=["--x", "6"]) == 6 - assert dcargs.cli(main, args=["--x", "five"]) == "five" + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == 6 + assert tyro.cli(main, args=["--x", "five"]) == "five" def test_union_with_list(): def main(x: int | str | list[bool]) -> Any: return x - assert dcargs.cli(main, args=["--x", "5"]) == 5 - assert dcargs.cli(main, args=["--x", "6"]) == 6 - assert dcargs.cli(main, args=["--x", "five"]) == "five" - assert dcargs.cli(main, args=["--x", "True"]) == "True" - assert dcargs.cli(main, args=["--x", "True", "False"]) == [True, False] + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == 6 + assert tyro.cli(main, args=["--x", "five"]) == "five" + assert tyro.cli(main, args=["--x", "True"]) == "True" + assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] def test_union_literal(): def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: return x - assert dcargs.cli(main, args=["--x", "5"]) == 5 - assert dcargs.cli(main, args=["--x", "6"]) == "6" - assert dcargs.cli(main, args=["--x", "five"]) == "five" + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == "6" + assert tyro.cli(main, args=["--x", "five"]) == "five" def test_super_nested(): @@ -48,13 +47,13 @@ def main( ) -> Any: return x - assert dcargs.cli(main, args=[]) is None - assert dcargs.cli(main, args="--x None".split(" ")) is None - assert dcargs.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] - assert dcargs.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] - assert dcargs.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ + assert tyro.cli(main, args=[]) is None + assert tyro.cli(main, args="--x None".split(" ")) is None + assert tyro.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] + assert tyro.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] + assert tyro.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ (2, 3, ("x", "2")), (2, 3, (1, 2)), ] with pytest.raises(SystemExit): - dcargs.cli(main, args=["--help"]) + tyro.cli(main, args=["--help"]) diff --git a/tests/test_positional_ignore_py37.py b/tests/test_positional_ignore_py37.py index 0c68a7e5c..b19c6b3ea 100644 --- a/tests/test_positional_ignore_py37.py +++ b/tests/test_positional_ignore_py37.py @@ -2,7 +2,7 @@ import pytest -import dcargs +import tyro def test_positional(): @@ -26,9 +26,9 @@ def main( """ return (x, y, z[0]) - assert dcargs.cli(main, args="1 2 --z 3".split(" ")) == (1, 2, 3) + assert tyro.cli(main, args="1 2 --z 3".split(" ")) == (1, 2, 3) with pytest.raises(SystemExit): - assert dcargs.cli(main, args="--x 1 --y 2 --z 3".split(" ")) == (1, 2, 3) + assert tyro.cli(main, args="--x 1 --y 2 --z 3".split(" ")) == (1, 2, 3) def test_nested_positional(): @@ -39,12 +39,10 @@ def __init__(self, a: int, hello_world: int, /, c: int): def nest1(a: int, b: int, thing: A, /, c: int) -> A: return thing - assert isinstance(dcargs.cli(nest1, args="0 1 2 3 --thing.c 4 --c 4".split(" ")), A) - assert ( - dcargs.cli(nest1, args="0 1 2 3 --thing.c 4 --c 4".split(" ")).hello_world == 3 - ) + assert isinstance(tyro.cli(nest1, args="0 1 2 3 --thing.c 4 --c 4".split(" ")), A) + assert tyro.cli(nest1, args="0 1 2 3 --thing.c 4 --c 4".split(" ")).hello_world == 3 with pytest.raises(SystemExit): - dcargs.cli(nest1, args="0 1 2 3 4 --thing.c 4 --c 4".split(" ")) + tyro.cli(nest1, args="0 1 2 3 4 --thing.c 4 --c 4".split(" ")) def test_nested_positional_alt(): @@ -55,9 +53,9 @@ def __init__(self, a: int, b: int, /, c: int): def nest2(a: int, b: int, /, thing: B, c: int): return thing - assert isinstance(dcargs.cli(nest2, args="0 1 2 3 --thing.c 4 --c 4".split(" ")), B) + assert isinstance(tyro.cli(nest2, args="0 1 2 3 --thing.c 4 --c 4".split(" ")), B) with pytest.raises(SystemExit): - dcargs.cli(nest2, args="0 1 2 3 4 --thing.c 4 --c 4".split(" ")) + tyro.cli(nest2, args="0 1 2 3 4 --thing.c 4 --c 4".split(" ")) def test_positional_with_underscores(): @@ -66,7 +64,7 @@ def test_positional_with_underscores(): def main(a_multi_word_input: int, /) -> int: return a_multi_word_input - assert dcargs.cli(main, args=["5"]) == 5 + assert tyro.cli(main, args=["5"]) == 5 def test_positional_booleans(): @@ -80,42 +78,42 @@ def main( ) -> Tuple[bool, bool, bool]: return flag1, flag2, flag3 - assert dcargs.cli(main, args=["True"]) == (True, True, False) - assert dcargs.cli(main, args=["True", "False"]) == (True, False, False) - assert dcargs.cli(main, args=["False", "False", "True"]) == (False, False, True) + assert tyro.cli(main, args=["True"]) == (True, True, False) + assert tyro.cli(main, args=["True", "False"]) == (True, False, False) + assert tyro.cli(main, args=["False", "False", "True"]) == (False, False, True) with pytest.raises(SystemExit): - dcargs.cli(main, args=["hmm"]) + tyro.cli(main, args=["hmm"]) with pytest.raises(SystemExit): - dcargs.cli(main, args=["true"]) + tyro.cli(main, args=["true"]) with pytest.raises(SystemExit): - dcargs.cli(main, args=["True", "false"]) + tyro.cli(main, args=["True", "false"]) def test_optional_list(): def main(a: Optional[List[int]], /) -> Optional[List[int]]: return a - assert dcargs.cli(main, args=["None"]) is None - assert dcargs.cli(main, args=["1", "2"]) == [1, 2] + assert tyro.cli(main, args=["None"]) is None + assert tyro.cli(main, args=["1", "2"]) == [1, 2] with pytest.raises(SystemExit): - dcargs.cli(main, args=[]) + tyro.cli(main, args=[]) with pytest.raises(SystemExit): - dcargs.cli(main, args=["hm"]) + tyro.cli(main, args=["hm"]) def test_optional_list_with_default(): def main(a: Optional[List[int]] = None, /) -> Optional[List[int]]: return a - assert dcargs.cli(main, args=["None"]) is None - assert dcargs.cli(main, args=["5", "5"]) == [5, 5] + assert tyro.cli(main, args=["None"]) is None + assert tyro.cli(main, args=["5", "5"]) == [5, 5] with pytest.raises(SystemExit): - dcargs.cli(main, args=["None", "5"]) + tyro.cli(main, args=["None", "5"]) def test_positional_tuple(): def main(x: Tuple[int, int], y: Tuple[str, str], /): return x, y - assert dcargs.cli(main, args="1 2 3 4".split(" ")) == ((1, 2), ("3", "4")) + assert tyro.cli(main, args="1 2 3 4".split(" ")) == ((1, 2), ("3", "4")) diff --git a/tests/test_print_completion.py b/tests/test_print_completion.py index 9f4c34d95..142db6870 100644 --- a/tests/test_print_completion.py +++ b/tests/test_print_completion.py @@ -5,10 +5,10 @@ import pytest -import dcargs +import tyro -# https://github.com/brentyi/dcargs/issues/9 +# https://github.com/brentyi/tyro/issues/9 @dataclasses.dataclass(frozen=True) class Subtype: data: int = 1 @@ -32,12 +32,12 @@ class Wrapper: def test_bash(): target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): - dcargs.cli(Wrapper, args=["--dcargs-print-completion", "bash"]) + tyro.cli(Wrapper, args=["--tyro-print-completion", "bash"]) assert "# AUTOMATCALLY GENERATED by `shtab`" in target.getvalue() def test_zsh(): target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): - dcargs.cli(Wrapper, args=["--dcargs-print-completion", "zsh"]) + tyro.cli(Wrapper, args=["--tyro-print-completion", "zsh"]) assert "# AUTOMATCALLY GENERATED by `shtab`" in target.getvalue() diff --git a/tests/test_strings.py b/tests/test_strings.py index 398b84cd4..6790fece8 100644 --- a/tests/test_strings.py +++ b/tests/test_strings.py @@ -1,4 +1,4 @@ -from dcargs import _strings +from tyro import _strings def test_words_from_name(): diff --git a/tests/test_union_from_mapping.py b/tests/test_union_from_mapping.py index 7e9d35ee4..841710e78 100644 --- a/tests/test_union_from_mapping.py +++ b/tests/test_union_from_mapping.py @@ -1,7 +1,7 @@ import dataclasses from typing import Optional -import dcargs +import tyro @dataclasses.dataclass @@ -15,12 +15,12 @@ def test_union_from_mapping(): "two": A(2), "three": A(3), } - ConfigUnion = dcargs.extras.subcommand_type_from_defaults(base_configs) + ConfigUnion = tyro.extras.subcommand_type_from_defaults(base_configs) - assert dcargs.cli(ConfigUnion, args="one".split(" ")) == A(1) - assert dcargs.cli(ConfigUnion, args="two".split(" ")) == A(2) - assert dcargs.cli(ConfigUnion, args="two --x 4".split(" ")) == A(4) - assert dcargs.cli(ConfigUnion, args="three".split(" ")) == A(3) + assert tyro.cli(ConfigUnion, args="one".split(" ")) == A(1) + assert tyro.cli(ConfigUnion, args="two".split(" ")) == A(2) + assert tyro.cli(ConfigUnion, args="two --x 4".split(" ")) == A(4) + assert tyro.cli(ConfigUnion, args="three".split(" ")) == A(3) def test_union_from_mapping_in_function(): @@ -32,17 +32,17 @@ def test_union_from_mapping_in_function(): # Hack for mypy. Not needed for pyright. ConfigUnion = A - ConfigUnion = dcargs.extras.subcommand_type_from_defaults(base_configs) # type: ignore + ConfigUnion = tyro.extras.subcommand_type_from_defaults(base_configs) # type: ignore def main(config: ConfigUnion, flag: bool = False) -> Optional[A]: if flag: return config return None - assert dcargs.cli(main, args="--flag config:one".split(" ")) == A(1) - assert dcargs.cli(main, args="--flag config:one --config.x 3".split(" ")) == A(3) - assert dcargs.cli(main, args="config:one --config.x 1".split(" ")) is None + assert tyro.cli(main, args="--flag config:one".split(" ")) == A(1) + assert tyro.cli(main, args="--flag config:one --config.x 3".split(" ")) == A(3) + assert tyro.cli(main, args="config:one --config.x 1".split(" ")) is None - assert dcargs.cli(main, args="--flag config:two".split(" ")) == A(2) - assert dcargs.cli(main, args="--flag config:two --config.x 3".split(" ")) == A(3) - assert dcargs.cli(main, args="config:two --config.x 1".split(" ")) is None + assert tyro.cli(main, args="--flag config:two".split(" ")) == A(2) + assert tyro.cli(main, args="--flag config:two --config.x 3".split(" ")) == A(3) + assert tyro.cli(main, args="config:two --config.x 1".split(" ")) is None diff --git a/tests/test_unsupported_but_should_work.py b/tests/test_unsupported_but_should_work.py index 8022e7409..9045e3503 100644 --- a/tests/test_unsupported_but_should_work.py +++ b/tests/test_unsupported_but_should_work.py @@ -12,8 +12,8 @@ import omegaconf import pytest -import dcargs -import dcargs._strings +import tyro +import tyro._strings def test_omegaconf_missing(): @@ -26,19 +26,19 @@ def main( ) -> Tuple[int, int, int]: return (required_a, optional, required_b) # type: ignore - assert dcargs.cli( + assert tyro.cli( main, args="--required-a 3 --optional 4 --required-b 5".split(" ") ) == (3, 4, 5) - assert dcargs.cli(main, args="--required-a 3 --required-b 5".split(" ")) == ( + assert tyro.cli(main, args="--required-a 3 --required-b 5".split(" ")) == ( 3, 3, 5, ) with pytest.raises(SystemExit): - dcargs.cli(main, args="--required-a 3 --optional 4") + tyro.cli(main, args="--required-a 3 --optional 4") with pytest.raises(SystemExit): - dcargs.cli(main, args="--required-a 3") + tyro.cli(main, args="--required-a 3") def main2( required_a: int, @@ -47,19 +47,19 @@ def main2( ) -> Tuple[int, int, int]: return (required_a, optional, required_b) - assert dcargs.cli( + assert tyro.cli( main2, args="--required-a 3 --optional 4 --required-b 5".split(" ") ) == (3, 4, 5) - assert dcargs.cli(main2, args="--required-a 3 --required-b 5".split(" ")) == ( + assert tyro.cli(main2, args="--required-a 3 --required-b 5".split(" ")) == ( 3, 3, 5, ) with pytest.raises(SystemExit): - dcargs.cli(main2, args="--required-a 3 --optional 4") + tyro.cli(main2, args="--required-a 3 --optional 4") with pytest.raises(SystemExit): - dcargs.cli(main2, args="--required-a 3") + tyro.cli(main2, args="--required-a 3") def test_attrs_basic(): @@ -70,8 +70,8 @@ class ManyTypesA: f: float = attr.ib() p: pathlib.Path = attr.ib() - # We can directly pass a dataclass to `dcargs.cli()`: - assert dcargs.cli( + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( ManyTypesA, args=[ "--i", @@ -93,8 +93,8 @@ class ManyTypesB: s: str = attr.ib() f: float = attr.ib(default=1.0) - # We can directly pass a dataclass to `dcargs.cli()`: - assert dcargs.cli( + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( ManyTypesB, args=[ "--i", @@ -121,9 +121,9 @@ class Helptext: f = io.StringIO() with pytest.raises(SystemExit): with contextlib.redirect_stdout(f): - dcargs.cli(Helptext, args=["--help"]) + tyro.cli(Helptext, args=["--help"]) helptext = f.getvalue() - assert dcargs._strings.strip_ansi_sequences(cast(str, Helptext.__doc__)) in helptext + assert tyro._strings.strip_ansi_sequences(cast(str, Helptext.__doc__)) in helptext # Note that required detection seems to be broken here. assert "Documentation 1" in helptext diff --git a/dcargs/__init__.py b/tyro/__init__.py similarity index 100% rename from dcargs/__init__.py rename to tyro/__init__.py diff --git a/dcargs/_argparse_formatter.py b/tyro/_argparse_formatter.py similarity index 96% rename from dcargs/_argparse_formatter.py rename to tyro/_argparse_formatter.py index b9840008f..cc9dda2f4 100644 --- a/dcargs/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -6,7 +6,7 @@ - Can be themed with an accent color. This is largely built by fussing around in argparse implementation details, and is by -far the hackiest part of `dcargs`. +far the hackiest part of `tyro`. """ import argparse import contextlib @@ -194,11 +194,11 @@ def format_help(self): # For dense multi-column layouts, the fixed help position is often shorter. # For wider layouts, using the default help position settings can be more # efficient. - self._dcargs_rule = None + self._tyro_rule = None self._fixed_help_position = False help1 = super().format_help() - self._dcargs_rule = None + self._tyro_rule = None self._fixed_help_position = True help2 = super().format_help() @@ -215,15 +215,15 @@ def __init__(self, formatter, parent, heading=None): self.parent = parent self.heading = heading self.items = [] - self.formatter._dcargs_rule = None + self.formatter._tyro_rule = None def format_help(self): if self.parent is None: - return self._dcargs_format_root() + return self._tyro_format_root() else: - return self._dcargs_format_nonroot() + return self._tyro_format_nonroot() - def _dcargs_format_root(self): + def _tyro_format_root(self): console = Console(width=self.formatter._width, theme=THEME.as_rich_theme()) with console.capture() as capture: # Get rich renderables from items. @@ -366,7 +366,7 @@ def _format_action(self, action: argparse.Action): return item_parts - def _dcargs_format_nonroot(self): + def _tyro_format_nonroot(self): # Add each child item as a rich renderable. description_part = None item_parts = [] @@ -423,21 +423,21 @@ def _dcargs_format_nonroot(self): ) max_width = max(map(len, lines)) - if self.formatter._dcargs_rule is None: + if self.formatter._tyro_rule is None: # Note: we don't use rich.rule.Rule() because this will make all of # the panels expand to fill the full width of the console. (this only # impacts single-column layouts) - self.formatter._dcargs_rule = Text.from_ansi( + self.formatter._tyro_rule = Text.from_ansi( "─" * max_width, style=THEME.border, overflow="crop" ) - elif len(self.formatter._dcargs_rule._text[0]) < max_width: - self.formatter._dcargs_rule._text = ["─" * max_width] + elif len(self.formatter._tyro_rule._text[0]) < max_width: + self.formatter._tyro_rule._text = ["─" * max_width] # Add description text if needed. if description_part is not None: item_parts = [ description_part, - self.formatter._dcargs_rule, + self.formatter._tyro_rule, ] + item_parts return Panel( diff --git a/dcargs/_arguments.py b/tyro/_arguments.py similarity index 100% rename from dcargs/_arguments.py rename to tyro/_arguments.py diff --git a/dcargs/_calling.py b/tyro/_calling.py similarity index 100% rename from dcargs/_calling.py rename to tyro/_calling.py diff --git a/dcargs/_cli.py b/tyro/_cli.py similarity index 93% rename from dcargs/_cli.py rename to tyro/_cli.py index 300dd38f6..eafc68703 100644 --- a/dcargs/_cli.py +++ b/tyro/_cli.py @@ -57,7 +57,7 @@ def cli( CLI interface. `f` should have type-annotated inputs, and can be a function or type. Note that if - `f` is a type, `dcargs.cli()` returns an instance. + `f` is a type, `tyro.cli()` returns an instance. The parser is generated by populating helptext from docstrings and types from annotations; a broad range of core type annotations are supported. @@ -88,7 +88,7 @@ def cli( - Generics (including nested generics). Completion script generation for interactive shells is also provided. To print a - script that can be used for tab completion, pass in `--dcargs-print-completion + script that can be used for tab completion, pass in `--tyro-print-completion {bash/zsh/tcsh}`. Args: @@ -133,9 +133,9 @@ def get_parser( default: Optional[OutT] = None, ) -> argparse.ArgumentParser: """Get the `argparse.ArgumentParser` object generated under-the-hood by - `dcargs.cli()`. Useful for tools like `sphinx-argparse`, `argcomplete`, etc. + `tyro.cli()`. Useful for tools like `sphinx-argparse`, `argcomplete`, etc. - For tab completion, we recommend using `dcargs.cli()`'s built-in `--dcargs-print-completion` + For tab completion, we recommend using `tyro.cli()`'s built-in `--tyro-print-completion` flag.""" return cast( argparse.ArgumentParser, @@ -160,7 +160,7 @@ def _cli_impl( return_parser: bool, **deprecated_kwargs, ) -> Union[OutT, argparse.ArgumentParser]: - """Helper for stitching the `dcargs` pipeline together. + """Helper for stitching the `tyro` pipeline together. Converts `f` into a """ @@ -172,7 +172,7 @@ def _cli_impl( if deprecated_kwargs.get("avoid_subparsers", False): f = conf.AvoidSubcommands[f] # type: ignore warnings.warn( - "`avoid_subparsers=` is deprecated! use `dcargs.conf.AvoidSubparsers[]`" + "`avoid_subparsers=` is deprecated! use `tyro.conf.AvoidSubparsers[]`" " instead.", stacklevel=2, ) @@ -220,12 +220,12 @@ def fix_arg(arg: str) -> str: args = list(map(fix_arg, args)) - # If we pass in the --dcargs-print-completion flag: turn formatting tags, and get + # If we pass in the --tyro-print-completion flag: turn formatting tags, and get # the shell we want to generate a completion script for (bash/zsh/tcsh). # # Note that shtab also offers an add_argument_to() functions that fulfills a similar # goal, but manual parsing of argv is convenient for turning off formatting. - print_completion = len(args) >= 2 and args[0] == "--dcargs-print-completion" + print_completion = len(args) >= 2 and args[0] == "--tyro-print-completion" # Note: setting USE_RICH must happen before the parser specification is generated. # TODO: revisit this. Ideally we should be able to eliminate the global state @@ -275,7 +275,7 @@ def fix_arg(arg: str) -> str: shtab.complete( parser=parser, shell=completion_shell, - root_prefix=f"dcargs_{parser.prog}", + root_prefix=f"tyro_{parser.prog}", ) ) raise SystemExit() diff --git a/dcargs/_deprecated.py b/tyro/_deprecated.py similarity index 100% rename from dcargs/_deprecated.py rename to tyro/_deprecated.py diff --git a/dcargs/_docstrings.py b/tyro/_docstrings.py similarity index 100% rename from dcargs/_docstrings.py rename to tyro/_docstrings.py diff --git a/dcargs/_fields.py b/tyro/_fields.py similarity index 99% rename from dcargs/_fields.py rename to tyro/_fields.py index 915d6ef70..d5acaa50a 100644 --- a/dcargs/_fields.py +++ b/tyro/_fields.py @@ -109,7 +109,7 @@ class ExcludeFromCallType(_singleton.Singleton): # Note that our "public" missing API will always be the propagating missing sentinel. MISSING_PUBLIC: Any = MISSING_PROP """Sentinel value to mark fields as missing. Can be used to mark fields passed in as a -`default_instance` for `dcargs.cli()` as required.""" +`default_instance` for `tyro.cli()` as required.""" MISSING_SINGLETONS = [ @@ -593,7 +593,7 @@ def _get_dataclass_field_default( parent_default_instance not in MISSING_SINGLETONS and parent_default_instance is not None ): - # Populate default from some parent, eg `default_instance` in `dcargs.cli()`. + # Populate default from some parent, eg `default_instance` in `tyro.cli()`. if hasattr(parent_default_instance, field.name): return getattr(parent_default_instance, field.name) else: diff --git a/dcargs/_instantiators.py b/tyro/_instantiators.py similarity index 100% rename from dcargs/_instantiators.py rename to tyro/_instantiators.py diff --git a/dcargs/_parsers.py b/tyro/_parsers.py similarity index 99% rename from dcargs/_parsers.py rename to tyro/_parsers.py index 8dcab4e24..b923e8394 100644 --- a/dcargs/_parsers.py +++ b/tyro/_parsers.py @@ -359,7 +359,7 @@ def from_field( ) if default_name not in parser_from_name: # If we can't find the subparser by name, search by type. This is needed - # when the user renames their subcommands. (eg via dcargs.subcommand) + # when the user renames their subcommands. (eg via tyro.subcommand) # # TODO: this will display some weird behaviors if multiple subcommands # have the same type. diff --git a/dcargs/_resolver.py b/tyro/_resolver.py similarity index 100% rename from dcargs/_resolver.py rename to tyro/_resolver.py diff --git a/dcargs/_shtab/LICENCE b/tyro/_shtab/LICENCE similarity index 100% rename from dcargs/_shtab/LICENCE rename to tyro/_shtab/LICENCE diff --git a/dcargs/_shtab/README.md b/tyro/_shtab/README.md similarity index 100% rename from dcargs/_shtab/README.md rename to tyro/_shtab/README.md diff --git a/dcargs/_shtab/__init__.py b/tyro/_shtab/__init__.py similarity index 100% rename from dcargs/_shtab/__init__.py rename to tyro/_shtab/__init__.py diff --git a/dcargs/_shtab/__main__.py b/tyro/_shtab/__main__.py similarity index 100% rename from dcargs/_shtab/__main__.py rename to tyro/_shtab/__main__.py diff --git a/dcargs/_shtab/_dist_ver.py b/tyro/_shtab/_dist_ver.py similarity index 100% rename from dcargs/_shtab/_dist_ver.py rename to tyro/_shtab/_dist_ver.py diff --git a/dcargs/_shtab/main.py b/tyro/_shtab/main.py similarity index 100% rename from dcargs/_shtab/main.py rename to tyro/_shtab/main.py diff --git a/dcargs/_shtab/py.typed b/tyro/_shtab/py.typed similarity index 100% rename from dcargs/_shtab/py.typed rename to tyro/_shtab/py.typed diff --git a/dcargs/_singleton.py b/tyro/_singleton.py similarity index 100% rename from dcargs/_singleton.py rename to tyro/_singleton.py diff --git a/dcargs/_strings.py b/tyro/_strings.py similarity index 97% rename from dcargs/_strings.py rename to tyro/_strings.py index 6897bfd8d..b3e4cfcf5 100644 --- a/dcargs/_strings.py +++ b/tyro/_strings.py @@ -7,7 +7,7 @@ from . import _resolver -dummy_field_name = "__dcargs_dummy_field__" +dummy_field_name = "__tyro_dummy_field__" def _strip_dummy_field_names(parts: Iterable[str]) -> Iterable[str]: @@ -67,7 +67,7 @@ def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: cls, _subcommands._SubcommandConfiguration ) - # Subparser name from `dcargs.metadata.subcommand()`. + # Subparser name from `tyro.metadata.subcommand()`. found_name = None prefix_name = True if len(found_subcommand_configs) > 0: diff --git a/dcargs/conf/__init__.py b/tyro/conf/__init__.py similarity index 89% rename from dcargs/conf/__init__.py rename to tyro/conf/__init__.py index 1c74f72a4..1833f784d 100644 --- a/dcargs/conf/__init__.py +++ b/tyro/conf/__init__.py @@ -1,4 +1,4 @@ -"""The :mod:`dcargs.conf` submodule contains helpers for attaching parsing-specific +"""The :mod:`tyro.conf` submodule contains helpers for attaching parsing-specific configuration metadata to types via [PEP 593](https://peps.python.org/pep-0593/) runtime annotations. diff --git a/dcargs/conf/_markers.py b/tyro/conf/_markers.py similarity index 93% rename from dcargs/conf/_markers.py rename to tyro/conf/_markers.py index aca503c58..ffa4f9265 100644 --- a/dcargs/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -19,12 +19,12 @@ argument.""" Fixed = Annotated[T, None] -"""A type `T` can be annotated as `Fixed[T]` to prevent `dcargs.cli` from parsing it; a +"""A type `T` can be annotated as `Fixed[T]` to prevent `tyro.cli` from parsing it; a default value should be set instead. Note that fields with defaults that can't be parsed will also be marked as fixed automatically.""" Suppress = Annotated[T, None] -"""A type `T` can be annotated as `Suppress[T]` to prevent `dcargs.cli` from parsing it, and +"""A type `T` can be annotated as `Suppress[T]` to prevent `tyro.cli` from parsing it, and to prevent it from showing up in helptext.""" SuppressFixed = Annotated[T, None] diff --git a/dcargs/conf/_subcommands.py b/tyro/conf/_subcommands.py similarity index 91% rename from dcargs/conf/_subcommands.py rename to tyro/conf/_subcommands.py index 92f84b030..63b3d5d7e 100644 --- a/dcargs/conf/_subcommands.py +++ b/tyro/conf/_subcommands.py @@ -28,19 +28,19 @@ def subcommand( Consider the standard approach for creating subcommands: ```python - dcargs.cli( + tyro.cli( Union[NestedTypeA, NestedTypeB] ) ``` This will create two subcommands: `nested-type-a` and `nested-type-b`. - Annotating each type with `dcargs.metadata.subcommand()` allows us to override for + Annotating each type with `tyro.metadata.subcommand()` allows us to override for each subcommand the (a) name, (b) defaults, (c) helptext, and (d) whether to prefix the name or not. ```python - dcargs.cli( + tyro.cli( Union[ Annotated[ NestedTypeA, subcommand("a", ...) diff --git a/dcargs/extras/__init__.py b/tyro/extras/__init__.py similarity index 84% rename from dcargs/extras/__init__.py rename to tyro/extras/__init__.py index e70de2144..75b6c4bde 100644 --- a/dcargs/extras/__init__.py +++ b/tyro/extras/__init__.py @@ -1,4 +1,4 @@ -"""The :mod:`dcargs.extras` submodule contains helpers that complement :func:`dcargs.cli()`. +"""The :mod:`tyro.extras` submodule contains helpers that complement :func:`tyro.cli()`. Compared to the core interface, APIs here are more likely to be changed or deprecated. """ diff --git a/dcargs/extras/_base_configs.py b/tyro/extras/_base_configs.py similarity index 84% rename from dcargs/extras/_base_configs.py rename to tyro/extras/_base_configs.py index 36d06bd8b..347d50eb7 100644 --- a/dcargs/extras/_base_configs.py +++ b/tyro/extras/_base_configs.py @@ -16,7 +16,7 @@ def subcommand_type_from_defaults( """Construct a Union type for defining subcommands that choose between defaults. This can most commonly be used to create a "base configuration" pattern: - https://brentyi.github.io/dcargs/examples/10_base_configs/ + https://brentyi.github.io/tyro/examples/10_base_configs/ For example, when `defaults` is set to: @@ -33,19 +33,19 @@ def subcommand_type_from_defaults( Union[ Annotated[ Config, - dcargs.conf.subcommand("small", default=Config(...)) + tyro.conf.subcommand("small", default=Config(...)) ], Annotated[ Config, - dcargs.conf.subcommand("big", default=Config(...)) + tyro.conf.subcommand("big", default=Config(...)) ] ] ``` - The resulting type can be used directly in dcargs.cli: + The resulting type can be used directly in tyro.cli: ```python - config = dcargs.cli(subcommand_type_from_defaults(default_from_name)) + config = tyro.cli(subcommand_type_from_defaults(default_from_name)) reveal_type(config) # Should be correct! ``` @@ -60,7 +60,7 @@ def train( ) -> None: ... - dcargs.cli(train) + tyro.cli(train) ``` Note that Pyright understands the latter case, but mypy does not. If mypy support is diff --git a/dcargs/extras/_choices_type.py b/tyro/extras/_choices_type.py similarity index 100% rename from dcargs/extras/_choices_type.py rename to tyro/extras/_choices_type.py diff --git a/dcargs/extras/_serialization.py b/tyro/extras/_serialization.py similarity index 92% rename from dcargs/extras/_serialization.py rename to tyro/extras/_serialization.py index 9d23f31c2..9ae536fad 100644 --- a/dcargs/extras/_serialization.py +++ b/tyro/extras/_serialization.py @@ -173,11 +173,11 @@ def from_yaml( stream: Union[str, IO[str], bytes, IO[bytes]], ) -> DataclassType: """Re-construct a dataclass instance from a yaml-compatible string, which should be - generated from `dcargs.extras.to_yaml()`. + generated from `tyro.extras.to_yaml()`. - As a secondary feature aimed at enabling the use of :func:`dcargs.cli` for general + As a secondary feature aimed at enabling the use of :func:`tyro.cli` for general configuration use cases, we also introduce functions for human-readable dataclass - serialization: :func:`dcargs.conf.from_yaml` and :func:`dcargs.conf.to_yaml` attempt + serialization: :func:`tyro.conf.from_yaml` and :func:`tyro.conf.to_yaml` attempt to strike a balance between flexibility and robustness — in contrast to naively dumping or loading dataclass instances (via pickle, PyYAML, etc), explicit type references enable custom tags that are robust against code reorganization and @@ -185,7 +185,7 @@ def from_yaml( .. warning:: Serialization functionality is stable but deprecated. It may be removed in a - future version of :code:`dcargs`. + future version of :code:`tyro`. Args: cls: Type to reconstruct. @@ -202,11 +202,11 @@ def from_yaml( def to_yaml(instance: Any) -> str: """Serialize a dataclass; returns a yaml-compatible string that can be deserialized - via `dcargs.extras.from_yaml()`. + via `tyro.extras.from_yaml()`. - As a secondary feature aimed at enabling the use of :func:`dcargs.cli` for general + As a secondary feature aimed at enabling the use of :func:`tyro.cli` for general configuration use cases, we also introduce functions for human-readable dataclass - serialization: :func:`dcargs.conf.from_yaml` and :func:`dcargs.conf.to_yaml` attempt + serialization: :func:`tyro.conf.from_yaml` and :func:`tyro.conf.to_yaml` attempt to strike a balance between flexibility and robustness — in contrast to naively dumping or loading dataclass instances (via pickle, PyYAML, etc), explicit type references enable custom tags that are robust against code reorganization and @@ -214,7 +214,7 @@ def to_yaml(instance: Any) -> str: .. warning:: Serialization functionality is stable but deprecated. It may be removed in a - future version of :code:`dcargs`. + future version of :code:`tyro`. Args: instance: Dataclass instance to serialize. @@ -222,4 +222,4 @@ def to_yaml(instance: Any) -> str: Returns: YAML string. """ - return "# dcargs YAML.\n" + yaml.dump(instance, Dumper=_make_dumper(instance)) + return "# tyro YAML.\n" + yaml.dump(instance, Dumper=_make_dumper(instance)) diff --git a/dcargs/py.typed b/tyro/py.typed similarity index 100% rename from dcargs/py.typed rename to tyro/py.typed From 5852a944e36ab7378398b7a68b7ad1f32c759bcc Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Oct 2022 13:00:00 -0700 Subject: [PATCH 177/491] Use absolute URL for logo --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 516510c71..1df944a72 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@

-tyro logo + + tyro logo

From 4a48c9e748512c9af5a8663982f037612d4334a0 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Oct 2022 13:01:17 -0700 Subject: [PATCH 178/491] Rename `master` to `main` --- .github/workflows/build.yml | 4 ++-- .github/workflows/coverage.yml | 4 ++-- .github/workflows/docs.yml | 2 +- .github/workflows/lint.yml | 4 ++-- .github/workflows/mypy.yml | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b5b79ba08..8718ca45e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,9 +2,9 @@ name: build on: push: - branches: [master] + branches: [main] pull_request: - branches: [master] + branches: [main] jobs: build: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index c89e3dc5f..0cc69a2a4 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -2,9 +2,9 @@ name: coverage on: push: - branches: [master] + branches: [main] pull_request: - branches: [master] + branches: [main] jobs: coverage: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 43211e3e1..35401b80d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -2,7 +2,7 @@ name: docs on: push: - branches: [master] + branches: [main] jobs: docs: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5df6615b9..4f5170c1a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,9 +2,9 @@ name: lint on: push: - branches: [master] + branches: [main] pull_request: - branches: [master] + branches: [main] jobs: black-check: diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index b315c9c88..507d9046d 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -2,9 +2,9 @@ name: mypy on: push: - branches: [master] + branches: [main] pull_request: - branches: [master] + branches: [main] jobs: mypy: From df6b79e3c7cbd3382d352afdf762b08acf21dd18 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Oct 2022 13:12:29 -0700 Subject: [PATCH 179/491] Fix coverage action --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 0cc69a2a4..8eec8ecfa 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -21,7 +21,7 @@ jobs: poetry install - name: Generate coverage report run: | - poetry run pytest --cov=dcargs --cov-report=xml + poetry run pytest --cov=tyro --cov-report=xml - name: Upload to Codecov uses: codecov/codecov-action@v1 with: From eefb672e40bcdf93e266aec49fe48fc1c9e52b21 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Oct 2022 13:42:38 -0700 Subject: [PATCH 180/491] Update docs, bump version --- README.md | 4 +- docs/source/building_configuration_systems.md | 19 +++ docs/source/index.md | 129 ++++++++++++++++ docs/source/index.rst | 141 ------------------ docs/source/your_first_cli.md | 64 ++++++++ pyproject.toml | 2 +- 6 files changed, 214 insertions(+), 145 deletions(-) create mode 100644 docs/source/building_configuration_systems.md create mode 100644 docs/source/index.md delete mode 100644 docs/source/index.rst create mode 100644 docs/source/your_first_cli.md diff --git a/README.md b/README.md index 1df944a72..3e11b0bb0 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,7 @@ Python callables and types into fully-featured argument parsers and configuration objects. To get started, we recommend visiting the examples in our -[documentation](https://brentyi.github.io/tyro). If you're familiar with -alternative libraries, we also include comparisons between [tyro and argparse]() -and [tyro and hydra](). +[documentation](https://brentyi.github.io/tyro). ### Why `tyro`? diff --git a/docs/source/building_configuration_systems.md b/docs/source/building_configuration_systems.md new file mode 100644 index 000000000..9477eab06 --- /dev/null +++ b/docs/source/building_configuration_systems.md @@ -0,0 +1,19 @@ +# Building configuration systems + +Beyond building simple command-line interfaces, :func:`tyro.cli()` is designed +to scale to larger configuration systems such as those typically built with +libraries like [`hydra`](https://github.com/facebookresearch/hydra). + +For a live example of this, see +[nerfstudio](https://github.com/nerfstudio-project/nerfstudio/). Notably, +`nerfstudio`'s configuration system is implemented entirely in Python, no YAML +needed, and has full tab completion support in your terminal. + +For overriding a dynamically loaded configuration object (typically a +dataclass), the `default=` parameter of :func:`tyro.cli()` can be used. If you +have multiple default configuration objects and need to select one to pass in as +a default -- this might be a YAML file or config instance -- an environment +variable or manually parsed (via `sys.argv`) positional argument can be used. + +For exposing each of these base configurations as subcommands, see our +[base config example](./examples/10_base_configs). diff --git a/docs/source/index.md b/docs/source/index.md new file mode 100644 index 000000000..3ef09583c --- /dev/null +++ b/docs/source/index.md @@ -0,0 +1,129 @@ +# tyro + +|build| |nbsp| |mypy| |nbsp| |lint| |nbsp| |coverage| |nbsp| |versions| + +:code:`tyro` is a library for building CLI interfaces, configuration objects, +and configuration _systems_ with modern, type-annotated Python. + +Our core interface consists of just one function, :func:`tyro.cli()`, which +translates Python callables and types into fully-featured argument parsers and +configuration objects. + +To get started, we recommend browsing the examples to the left. If you're +familiar with alternative libraries, we also include comparisons between +[tyro and argparse](./first_cli) and [tyro and hydra](). + +### Why `tyro`? + +1. **Strong typing.** + + Unlike tools dependent on dictionaries, YAML, or dynamic namespaces, + arguments populated by `tyro` benefit from IDE and language server-supported + operations — think tab completion, rename, jump-to-def, docstrings on hover — + as well as static checking tools like `pyright` and `mypy`. + +2. **Minimal overhead.** + + Standard Python type annotations, docstrings, and default values are parsed + to automatically generate command-line interfaces with informative helptext. + + If you're familiar with type annotations and docstrings in Python, you + already know how to use `tyro`! If you're not, learning to use `tyro` reduces + to learning to write modern Python. + + Hate `tyro`? Just remove one line of code, and you're left with beautiful, + type-annotated, and documented vanilla Python that can be used with a range + of other configuration libraries. + +3. **Automatic helptext generation.** + + `tyro` parses docstrings, types, and defaults to generate carefully formatted + helptext. Longer messages are organized into responsive multi-column layouts. + +4. **Modularity.** + + `tyro` supports hierarchically nested configuration structures, which make it + easy to distribute definitions, defaults, and documentation of configurable + fields across modules or source files. + +5. **Tab completion.** + + By extending [shtab](https://github.com/iterative/shtab), `tyro` + automatically generates tab completion scripts for bash, zsh, and tcsh! + +### In the wild + +`tyro` is still a new library, but being stress tested in several projects! + +- [nerfstudio](https://github.com/nerfstudio-project/nerfstudio/) uses `tyro` + both to build compact command-line utilities and for YAML-free experiment + configuration. +- [obj2mjcf](https://github.com/kevinzakka/obj2mjcf) uses `tyro` to build a CLI + for processing composite Wavefront OBJ files for Mujoco. +- [tensorf-jax](https://github.com/brentyi/tensorf-jax/) implements + [Tensorial Radiance Fields](https://apchenstu.github.io/TensoRF/) in JAX, with + `tyro` for configuration. + + + +.. toctree:: + :caption: Getting started + :maxdepth: 1 + :hidden: + :titlesonly: + + installation + your_first_cli + +.. toctree:: + :caption: Notes + :maxdepth: 5 + :hidden: + :glob: + + helptext_generation + tab_completion + building_configuration_systems + goals_and_alternatives + +.. toctree:: + :caption: Examples + :maxdepth: 1 + :hidden: + :titlesonly: + :glob: + + examples/* + +.. toctree:: + :caption: API Reference + :maxdepth: 5 + :hidden: + :titlesonly: + + api/tyro/index + + + +.. |build| image:: https://github.com/brentyi/tyro/workflows/build/badge.svg + :alt: Build status icon + :target: https://github.com/brentyi/tyro +.. |mypy| image:: https://github.com/brentyi/tyro/workflows/mypy/badge.svg?branch=master + :alt: Mypy status icon + :target: https://github.com/brentyi/tyro +.. |lint| image:: https://github.com/brentyi/tyro/workflows/lint/badge.svg + :alt: Lint status icon + :target: https://github.com/brentyi/tyro +.. |coverage| image:: https://codecov.io/gh/brentyi/tyro/branch/master/graph/badge.svg + :alt: Test coverage status icon + :target: https://codecov.io/gh/brentyi/tyro +.. |downloads| image:: https://pepy.tech/badge/tyro + :alt: Download count icon + :target: https://pypi.org/project/tyro/ +.. |versions| image:: https://img.shields.io/pypi/pyversions/tyro + :alt: Version icon + :target: https://pypi.org/project/tyro/ +.. |nbsp| unicode:: 0xA0 + :trim: + + diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index 23604bab4..000000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,141 +0,0 @@ -tyro -========================================== - -|build| |nbsp| |mypy| |nbsp| |lint| |nbsp| |coverage| |nbsp| |versions| - -:code:`tyro` is a library for typed CLI interfaces and configuration objects. - -Our core interface, :func:`tyro.cli()`, generates argument parsers from -type-annotated callables: functions, classes, dataclasses, and *nested* -dataclasses and classes. - -This can be used as a replacement for :code:`argparse`: - - -.. code-block:: - - """Sum two numbers with argparse.""" - - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--a", type=int, required=True) - parser.add_argument("--b", type=int, default=3) - args = parser.parse_args() - - print(args.a + args.b) - - -.. code-block:: - - """Sum two numbers by calling a function with tyro.""" - - import tyro - - def main(a: int, b: int = 3) -> None: - print(a + b) - - tyro.cli(main) - - -.. code-block:: - - """Sum two numbers by instantiating a dataclass with tyro.""" - - from dataclasses import dataclass - - import tyro - - @dataclass - class Args: - a: int - b: int = 3 - - args = tyro.cli(Args) - print(args.a + args.b) - - -The broader goal is also a replacement for tools like :code:`hydra`, -:code:`gin-config`, and :code:`ml_collections` that's: - -- **Low effort.** Standard Python type annotations, docstrings, and default - values are parsed to automatically generate command-line interfaces with - informative helptext. - -- **Expressive.** :func:`tyro.cli` understands functions, classes, - dataclasses, and *nested* classes and dataclasses, as well as frequently used - annotations like unions, literals, and collections, which can be composed into - hierarchical configuration objects built on standard Python features. - -- **Typed.** Unlike dynamic configuration namespaces produced by libraries like - :code:`argparse`, :code:`YACS`, :code:`abseil`, :code:`hydra`, or - :code:`ml_collections`, typed outputs mean that IDE-assisted autocomplete, - rename, refactor, and go-to-definition operations work out-of-the-box, as well - as static checking tools like :code:`mypy` and :code:`pyright`. - -- **Modular.** Most approaches to configuration objects require a centralized - definition of all configurable fields. Hierarchically nesting configuration - structures, however, makes it easy to distribute definitions, defaults, and - documentation of configurable fields across modules or source files. A model - configuration dataclass, for example, can be co-located in its entirety with - the model implementation and dropped into any experiment configuration with an - import — this eliminates redundancy and makes entire modules easy to port - across codebases. - -.. toctree:: - :caption: Getting started - :maxdepth: 1 - :hidden: - :titlesonly: - - installation - -.. toctree:: - :caption: Notes - :maxdepth: 5 - :hidden: - :glob: - - helptext_generation - tab_completion - goals_and_alternatives - -.. toctree:: - :caption: API Reference - :maxdepth: 5 - :hidden: - :titlesonly: - - api/tyro/index - -.. toctree:: - :caption: Examples - :maxdepth: 1 - :hidden: - :titlesonly: - :glob: - - examples/* - - - -.. |build| image:: https://github.com/brentyi/tyro/workflows/build/badge.svg - :alt: Build status icon - :target: https://github.com/brentyi/tyro -.. |mypy| image:: https://github.com/brentyi/tyro/workflows/mypy/badge.svg?branch=master - :alt: Mypy status icon - :target: https://github.com/brentyi/tyro -.. |lint| image:: https://github.com/brentyi/tyro/workflows/lint/badge.svg - :alt: Lint status icon - :target: https://github.com/brentyi/tyro -.. |coverage| image:: https://codecov.io/gh/brentyi/tyro/branch/master/graph/badge.svg - :alt: Test coverage status icon - :target: https://codecov.io/gh/brentyi/tyro -.. |downloads| image:: https://pepy.tech/badge/tyro - :alt: Download count icon - :target: https://pypi.org/project/tyro/ -.. |versions| image:: https://img.shields.io/pypi/pyversions/tyro - :alt: Version icon - :target: https://pypi.org/project/tyro/ -.. |nbsp| unicode:: 0xA0 - :trim: diff --git a/docs/source/your_first_cli.md b/docs/source/your_first_cli.md new file mode 100644 index 000000000..cf46b3243 --- /dev/null +++ b/docs/source/your_first_cli.md @@ -0,0 +1,64 @@ +# Your first CLI + +For getting started with `tyro`, consider the simple `argparse`-based +command-line interface: + +```python +"""Sum two numbers from argparse.""" + +import argparse +parser = argparse.ArgumentParser() +parser.add_argument( + "--a", + type=int, + required=True, +) +parser.add_argument( + "--b", + type=int, + default=3, +) +args = parser.parse_args() + +print(args.a + args.b) +``` + +The basic feature of :func:`tyro.cli()` is to enable the removal of +parsing-specific boilerplate. + +We can specify the same logic with a function signature: + +```python +"""Sum two numbers by calling a function with tyro.""" + +import tyro + +def main(a: int, b: int = 3) -> None: + print(a + b) + +tyro.cli(main) +``` + +Particularly when interfaces grow in complexity or require hierarchical +structures, dataclasses can also be helpful: + +```python +"""Sum two numbers by instantiating a dataclass with tyro.""" + +from dataclasses import dataclass + +import tyro + +@dataclass +class Args: + a: int + b: int = 3 + +args = tyro.cli(Args) +print(args.a + args.b) +``` + +And that's it for the core API! By incorporating more advanced type annotations +from the standard library, we can specify a broad range of more advanced +behaviors: variable-length inputs, unions over types, subcommands, and more. Our +examples walk through a selection of these features. diff --git a/pyproject.toml b/pyproject.toml index 0b4882f61..48dfa6007 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.21" +version = "0.3.22" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] From 067bd3c7ba08d412da93cbf97a9787049150a782 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Oct 2022 14:11:36 -0700 Subject: [PATCH 181/491] Fix docs --- docs/source/index.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/index.md b/docs/source/index.md index 3ef09583c..150fcfcb3 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -9,9 +9,7 @@ Our core interface consists of just one function, :func:`tyro.cli()`, which translates Python callables and types into fully-featured argument parsers and configuration objects. -To get started, we recommend browsing the examples to the left. If you're -familiar with alternative libraries, we also include comparisons between -[tyro and argparse](./first_cli) and [tyro and hydra](). +To get started, we recommend browsing the examples to the left. ### Why `tyro`? From 32d140f4939a78ddd034597aebecc3b63dd671b8 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Oct 2022 14:12:16 -0700 Subject: [PATCH 182/491] Remove nested backticks, which are not supported by rst --- docs/source/building_configuration_systems.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/building_configuration_systems.md b/docs/source/building_configuration_systems.md index 9477eab06..83a7005bd 100644 --- a/docs/source/building_configuration_systems.md +++ b/docs/source/building_configuration_systems.md @@ -2,7 +2,7 @@ Beyond building simple command-line interfaces, :func:`tyro.cli()` is designed to scale to larger configuration systems such as those typically built with -libraries like [`hydra`](https://github.com/facebookresearch/hydra). +libraries like [hydra](https://github.com/facebookresearch/hydra). For a live example of this, see [nerfstudio](https://github.com/nerfstudio-project/nerfstudio/). Notably, From fcf7ea76c29449ff089e8175eb25787db57ccfef Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Oct 2022 14:36:02 -0700 Subject: [PATCH 183/491] Docs nit --- README.md | 6 +++--- docs/source/index.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3e11b0bb0..094748edb 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,6 @@ For more examples, see our [documentation](https://brentyi.github.io/tyro). configuration. - [obj2mjcf](https://github.com/kevinzakka/obj2mjcf) uses `tyro` to build a CLI for processing composite Wavefront OBJ files for Mujoco. -- [tensorf-jax](https://github.com/brentyi/tensorf-jax/) implements - [Tensorial Radiance Fields](https://apchenstu.github.io/TensoRF/) in JAX, with - `tyro` for configuration. +- [tensorf-jax](https://github.com/brentyi/tensorf-jax/) (unofficially) + implements [Tensorial Radiance Fields](https://apchenstu.github.io/TensoRF/) + in JAX, with `tyro` for configuration. diff --git a/docs/source/index.md b/docs/source/index.md index 150fcfcb3..154cedaa5 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -58,9 +58,9 @@ To get started, we recommend browsing the examples to the left. configuration. - [obj2mjcf](https://github.com/kevinzakka/obj2mjcf) uses `tyro` to build a CLI for processing composite Wavefront OBJ files for Mujoco. -- [tensorf-jax](https://github.com/brentyi/tensorf-jax/) implements - [Tensorial Radiance Fields](https://apchenstu.github.io/TensoRF/) in JAX, with - `tyro` for configuration. +- [tensorf-jax](https://github.com/brentyi/tensorf-jax/) (unofficially) + implements [Tensorial Radiance Fields](https://apchenstu.github.io/TensoRF/) + in JAX, with `tyro` for configuration. From d4f331b9422bdc033b969d2f707d953f5663058a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Oct 2022 21:29:29 -0700 Subject: [PATCH 184/491] Dark mode logo for README --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 094748edb..046af15bd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,10 @@

- tyro logo -

+ + + tyro logo + +

Documentation From dd34b9ea57d029d0599b670ff284a5e105ce0b05 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Oct 2022 21:44:02 -0700 Subject: [PATCH 185/491] Shrink logo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 046af15bd..55ff3f369 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ - tyro logo + tyro logo From 2ea7f60e989ddf4ca135f3ef87ba6fb962e50205 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Oct 2022 23:29:19 -0700 Subject: [PATCH 186/491] Fix link in docs --- docs/source/building_configuration_systems.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/building_configuration_systems.md b/docs/source/building_configuration_systems.md index 83a7005bd..a79322d5b 100644 --- a/docs/source/building_configuration_systems.md +++ b/docs/source/building_configuration_systems.md @@ -16,4 +16,4 @@ a default -- this might be a YAML file or config instance -- an environment variable or manually parsed (via `sys.argv`) positional argument can be used. For exposing each of these base configurations as subcommands, see our -[base config example](./examples/10_base_configs). +[base config example](/examples/10_base_configs). From ec8b83a31d8d6beba3c6c25218357040fe71ded3 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 6 Oct 2022 02:01:47 -0700 Subject: [PATCH 187/491] More aggressive caching for docstring parser --- pyproject.toml | 2 +- tyro/_arguments.py | 2 +- tyro/_docstrings.py | 22 ++++++++++++++++++++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 48dfa6007..3191839ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.22" +version = "0.3.23" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] diff --git a/tyro/_arguments.py b/tyro/_arguments.py index b005a6810..199683d2c 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -285,7 +285,7 @@ def as_str(x: Any) -> Tuple[str, ...]: USE_RICH = True -def _rich_tag_if_enabled(x: str, tag: str): +def _rich_tag_if_enabled(x: str, tag: str) -> str: x = rich.markup.escape(x) return ( x diff --git a/tyro/_docstrings.py b/tyro/_docstrings.py index 85d310c32..523a3764c 100644 --- a/tyro/_docstrings.py +++ b/tyro/_docstrings.py @@ -7,13 +7,28 @@ import io import itertools import tokenize -from typing import Callable, Dict, Generic, Hashable, List, Optional, Type +from typing import ( + Callable, + Dict, + Generic, + Hashable, + List, + Optional, + Type, + TypeVar, + cast, +) import docstring_parser from typing_extensions import get_origin, is_typeddict from . import _resolver, _strings +T = TypeVar("T", bound=Callable) + +# Cast for making types more lenient. +_cache = cast(Callable[[int], Callable[[T], T]], functools.lru_cache) + @dataclasses.dataclass(frozen=True) class _Token: @@ -39,7 +54,7 @@ class _ClassTokenization: field_data_from_name: Dict[str, _FieldData] @staticmethod - @functools.lru_cache(maxsize=8) + @_cache(64) def make(clz) -> "_ClassTokenization": """Parse the source code of a class, and cache some tokenization information.""" readline = io.BytesIO(inspect.getsource(clz).encode("utf-8")).readline @@ -109,6 +124,7 @@ def make(clz) -> "_ClassTokenization": ) +@_cache(256) def get_class_tokenization_with_field( cls: Type, field_name: str ) -> Optional[_ClassTokenization]: @@ -147,6 +163,7 @@ def get_class_tokenization_with_field( return tokenization +@_cache(256) def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: """Get docstring for a field in a class.""" @@ -258,6 +275,7 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: ) +@_cache(256) def get_callable_description(f: Callable) -> str: """Get description associated with a callable via docstring parsing. From afddac4812a9946ac12659d7b921cb010ec10792 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 6 Oct 2022 09:10:24 -0700 Subject: [PATCH 188/491] README tweaks --- README.md | 19 +++++++------------ docs/source/index.md | 19 +++++++------------ 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 55ff3f369..8e18a2c64 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ configuration objects, and configuration _systems_ with modern, type-annotated Python. -Our core interface consists of just one function, `tyro.cli()`, which translates +Our core interface consists of just one function, `tyro.cli()`, that translates Python callables and types into fully-featured argument parsers and configuration objects. @@ -59,21 +59,16 @@ To get started, we recommend visiting the examples in our type-annotated, and documented vanilla Python that can be used with a range of other configuration libraries. -3. **Automatic helptext generation.** +3. **Modularity.** - `tyro` parses docstrings, types, and defaults to generate carefully formatted - helptext. Longer messages are organized into responsive multi-column layouts. + `tyro` supports hierarchical configuration structures, which make it easy to + distribute definitions, defaults, and documentation of configurable fields + across modules or source files. -4. **Modularity.** - - `tyro` supports hierarchically nested configuration structures, which make it - easy to distribute definitions, defaults, and documentation of configurable - fields across modules or source files. - -5. **Tab completion.** +4. **Tab completion.** By extending [shtab](https://github.com/iterative/shtab), `tyro` - automatically generates tab completion scripts for bash, zsh, and tcsh! + automatically generates tab completion scripts for bash, zsh, and tcsh. ### A minimal example diff --git a/docs/source/index.md b/docs/source/index.md index 154cedaa5..eb52860dc 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -5,7 +5,7 @@ :code:`tyro` is a library for building CLI interfaces, configuration objects, and configuration _systems_ with modern, type-annotated Python. -Our core interface consists of just one function, :func:`tyro.cli()`, which +Our core interface consists of just one function, :func:`tyro.cli()`, that translates Python callables and types into fully-featured argument parsers and configuration objects. @@ -33,21 +33,16 @@ To get started, we recommend browsing the examples to the left. type-annotated, and documented vanilla Python that can be used with a range of other configuration libraries. -3. **Automatic helptext generation.** +3. **Modularity.** - `tyro` parses docstrings, types, and defaults to generate carefully formatted - helptext. Longer messages are organized into responsive multi-column layouts. + `tyro` supports hierarchical configuration structures, which make it easy to + distribute definitions, defaults, and documentation of configurable fields + across modules or source files. -4. **Modularity.** - - `tyro` supports hierarchically nested configuration structures, which make it - easy to distribute definitions, defaults, and documentation of configurable - fields across modules or source files. - -5. **Tab completion.** +4. **Tab completion.** By extending [shtab](https://github.com/iterative/shtab), `tyro` - automatically generates tab completion scripts for bash, zsh, and tcsh! + automatically generates tab completion scripts for bash, zsh, and tcsh. ### In the wild From 109a4245eaffde55406f3e4a6f0c4c4e8a56fe71 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 6 Oct 2022 10:29:03 -0700 Subject: [PATCH 189/491] Reduce slow string manipulation --- docs/update_example_docs.py | 6 ++---- tyro/_argparse_formatter.py | 19 ++++++++++++++----- tyro/_arguments.py | 9 +++------ tyro/_parsers.py | 9 ++++++++- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/docs/update_example_docs.py b/docs/update_example_docs.py index 05e06b929..fc40bffee 100644 --- a/docs/update_example_docs.py +++ b/docs/update_example_docs.py @@ -110,10 +110,8 @@ def main( ).write_text( "\n".join( [ - ( - ".. Comment: this file is automatically generated by" - " `update_example_docs.py`." - ), + ".. Comment: this file is automatically generated by" + " `update_example_docs.py`.", " It should not be modified manually.", "", f"{ex.index}. {ex.title}", diff --git a/tyro/_argparse_formatter.py b/tyro/_argparse_formatter.py index cc9dda2f4..67ab33172 100644 --- a/tyro/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -122,8 +122,6 @@ def inner() -> Generator[None, None, None]: def str_from_rich( renderable: RenderableType, width: Optional[int] = None, soft_wrap: bool = False ) -> str: - # TODO: everywhere that this function is used is a little bit sketchy. Could use - # re-thinking. console = Console(width=width, theme=THEME.as_rich_theme()) with console.capture() as out: console.print(renderable, soft_wrap=soft_wrap) @@ -314,6 +312,18 @@ def _format_action(self, action: argparse.Action): action.help = str_from_rich( Text.from_markup("[helptext]" + action.help + "[/helptext]") ) + + # Unescape % signs, which need special handling in argparse. + if action.help is not None: + assert isinstance(action.help, str) + helptext = ( + Text.from_ansi(action.help) + if _strings.strip_ansi_sequences(action.help) != action.help + else Text.from_markup(action.help) + ) + else: + helptext = Text("") + if ( action.help and len(_strings.strip_ansi_sequences(invocation)) < help_position - 1 @@ -327,8 +337,7 @@ def _format_action(self, action: argparse.Action): invocation, style=THEME.invocation, ), - # Unescape % signs, which need special handling in argparse. - Text.from_ansi(action.help.replace("%%", "%")), + helptext, ) item_parts.append(table) @@ -344,7 +353,7 @@ def _format_action(self, action: argparse.Action): item_parts.append( Padding( # Unescape % signs, which need special handling in argparse. - Text.from_ansi(action.help.replace("%%", "%")), + helptext, pad=(0, 0, 0, help_position), ) ) diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 199683d2c..fb57c46d7 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -285,13 +285,10 @@ def as_str(x: Any) -> Tuple[str, ...]: USE_RICH = True +# TODO: this function is also called outside of _arguments.py. Should be revisited. def _rich_tag_if_enabled(x: str, tag: str) -> str: - x = rich.markup.escape(x) - return ( - x - if not USE_RICH - else _argparse_formatter.str_from_rich(Text.from_markup(f"[{tag}]{x}[/{tag}]")) - ) + x = rich.markup.escape(_strings.strip_ansi_sequences(x)) + return x if not USE_RICH else f"[{tag}]{x}[/{tag}]" def _rule_generate_helptext( diff --git a/tyro/_parsers.py b/tyro/_parsers.py index b923e8394..7da4d6f0a 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -457,10 +457,17 @@ def apply( subparser_tree_nodes.append(subparser) for name, subparser_def in self.parser_from_name.items(): + helptext = subparser_def.description + if len(helptext) > 0: + # TODO: calling a private function here. + helptext = _arguments._rich_tag_if_enabled( + helptext.strip(), "helptext" + ) + subparser = argparse_subparsers.add_parser( name, formatter_class=_argparse_formatter.DcargsArgparseHelpFormatter, - help=rich.markup.escape(subparser_def.description), + help=helptext, ) subparser_def.apply(subparser) From ada4301395bac9b1f7198b479f08a88579d94371 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 6 Oct 2022 10:47:26 -0700 Subject: [PATCH 190/491] Fixes + tests for special character handling --- README.md | 2 +- docs/source/index.md | 2 +- docs/update_example_docs.py | 6 ++++-- tests/test_helptext.py | 9 +++++++-- tyro/_argparse_formatter.py | 4 ++-- tyro/_arguments.py | 3 +-- tyro/_parsers.py | 3 +-- 7 files changed, 17 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8e18a2c64..07696c3a9 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@
tyro is a library for building CLI interfaces, -configuration objects, and configuration _systems_ with modern, type-annotated +configuration objects, and configuration systems with modern, type-annotated Python. Our core interface consists of just one function, `tyro.cli()`, that translates diff --git a/docs/source/index.md b/docs/source/index.md index eb52860dc..b31ed7d4c 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -3,7 +3,7 @@ |build| |nbsp| |mypy| |nbsp| |lint| |nbsp| |coverage| |nbsp| |versions| :code:`tyro` is a library for building CLI interfaces, configuration objects, -and configuration _systems_ with modern, type-annotated Python. +and configuration systems with modern, type-annotated Python. Our core interface consists of just one function, :func:`tyro.cli()`, that translates Python callables and types into fully-featured argument parsers and diff --git a/docs/update_example_docs.py b/docs/update_example_docs.py index fc40bffee..05e06b929 100644 --- a/docs/update_example_docs.py +++ b/docs/update_example_docs.py @@ -110,8 +110,10 @@ def main( ).write_text( "\n".join( [ - ".. Comment: this file is automatically generated by" - " `update_example_docs.py`.", + ( + ".. Comment: this file is automatically generated by" + " `update_example_docs.py`." + ), " It should not be modified manually.", "", f"{ex.index}. {ex.title}", diff --git a/tests/test_helptext.py b/tests/test_helptext.py index c50815899..3c5930c88 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -434,6 +434,8 @@ class OptionalLiteralHelptext: def test_multiple_subparsers_helptext(): @dataclasses.dataclass class Subcommand1: + """2% milk.""" # % symbol is prone to bugs in argparse. + x: int = 0 @dataclasses.dataclass @@ -457,6 +459,7 @@ class MultipleSubparsers: helptext = _get_helptext(MultipleSubparsers) + assert "2% milk." in helptext assert "Field a description." in helptext assert "Field b description." not in helptext assert "Field c description." not in helptext @@ -465,6 +468,7 @@ class MultipleSubparsers: MultipleSubparsers, args=["a:subcommand1", "b:subcommand1", "--help"] ) + assert "2% milk." in helptext assert "Field a description." not in helptext assert "Field b description." not in helptext assert "Field c description." in helptext @@ -474,7 +478,7 @@ class MultipleSubparsers: def test_optional_helptext(): @dataclasses.dataclass class OptionalHelptext: - """This docstring should be printed as a description.""" + """This docstring should be printed as a description. 2% milk.""" x: Optional[int] # Documentation 1 @@ -485,7 +489,8 @@ class OptionalHelptext: """Documentation 3""" helptext = _get_helptext(OptionalHelptext) - assert cast(str, OptionalHelptext.__doc__) in helptext + assert cast(str, OptionalHelptext.__doc__[:20]) in helptext + assert "2% milk" in helptext assert "--x {None}|INT" in helptext assert "--y {None}|INT [{None}|INT ...]" in helptext assert "[--z {None}|INT]" in helptext diff --git a/tyro/_argparse_formatter.py b/tyro/_argparse_formatter.py index 67ab33172..a29a38554 100644 --- a/tyro/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -317,9 +317,9 @@ def _format_action(self, action: argparse.Action): if action.help is not None: assert isinstance(action.help, str) helptext = ( - Text.from_ansi(action.help) + Text.from_ansi(action.help.replace("%%", "%")) if _strings.strip_ansi_sequences(action.help) != action.help - else Text.from_markup(action.help) + else Text.from_markup(action.help.replace("%%", "%")) ) else: helptext = Text("") diff --git a/tyro/_arguments.py b/tyro/_arguments.py index fb57c46d7..41e12f13d 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -22,9 +22,8 @@ ) import rich.markup -from rich.text import Text -from . import _argparse_formatter, _fields, _instantiators, _resolver +from . import _fields, _instantiators, _resolver from . import _shtab as shtab from . import _strings from .conf import _markers diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 7da4d6f0a..2292d2c2d 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -7,7 +7,6 @@ import itertools from typing import Any, Callable, Dict, List, Optional, Set, Type, TypeVar, Union, cast -import rich.markup from typing_extensions import Annotated, get_args, get_origin from . import ( @@ -457,7 +456,7 @@ def apply( subparser_tree_nodes.append(subparser) for name, subparser_def in self.parser_from_name.items(): - helptext = subparser_def.description + helptext = subparser_def.description.replace("%", "%%") if len(helptext) > 0: # TODO: calling a private function here. helptext = _arguments._rich_tag_if_enabled( From c2939d8dd55c1b19b5b178ef78d5bc2e2fb7d5a0 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 6 Oct 2022 12:11:14 -0700 Subject: [PATCH 191/491] Improve line break handling for descriptions --- docs/update_example_docs.py | 6 ++---- tests/test_strings.py | 26 +++++++++++++++++++------- tyro/_docstrings.py | 8 ++++---- tyro/_parsers.py | 8 +++++--- tyro/_strings.py | 3 ++- 5 files changed, 32 insertions(+), 19 deletions(-) diff --git a/docs/update_example_docs.py b/docs/update_example_docs.py index 05e06b929..fc40bffee 100644 --- a/docs/update_example_docs.py +++ b/docs/update_example_docs.py @@ -110,10 +110,8 @@ def main( ).write_text( "\n".join( [ - ( - ".. Comment: this file is automatically generated by" - " `update_example_docs.py`." - ), + ".. Comment: this file is automatically generated by" + " `update_example_docs.py`.", " It should not be modified manually.", "", f"{ex.index}. {ex.title}", diff --git a/tests/test_strings.py b/tests/test_strings.py index 6790fece8..ff7669a75 100644 --- a/tests/test_strings.py +++ b/tests/test_strings.py @@ -22,18 +22,18 @@ def test_make_field_name(): def test_postprocess_helptext(): - assert _strings.postprocess_helptext("hello world") == "hello world" - assert _strings.postprocess_helptext("hello\nworld") == "hello world" - assert _strings.postprocess_helptext("hello \nworld") == "hello world" - assert _strings.postprocess_helptext("hello\n\nworld") == "hello\n\nworld" + assert _strings.remove_single_line_breaks("hello world") == "hello world" + assert _strings.remove_single_line_breaks("hello\nworld") == "hello world" + assert _strings.remove_single_line_breaks("hello \nworld") == "hello world" + assert _strings.remove_single_line_breaks("hello\n\nworld") == "hello\n\nworld" assert ( - _strings.postprocess_helptext( + _strings.remove_single_line_breaks( "a paragraph:\nSentence one.\nSentence two.\nSentence three.\n" ) == "a paragraph: Sentence one. Sentence two. Sentence three." ) assert ( - _strings.postprocess_helptext( + _strings.remove_single_line_breaks( "a bulleted list:\n" "- The first problem.\n" "- The second problem.\n" @@ -45,7 +45,19 @@ def test_postprocess_helptext(): "- The third problem." ) assert ( - _strings.postprocess_helptext( + _strings.remove_single_line_breaks( + "an indented list:\n" + " The first problem.\n" + " The second problem.\n" + " The third problem.\n" + ) + == "an indented list:\n" + " The first problem.\n" + " The second problem.\n" + " The third problem." + ) + assert ( + _strings.remove_single_line_breaks( "a numbered list:\n" "1. The first problem.\n" "2. The second problem.\n" diff --git a/tyro/_docstrings.py b/tyro/_docstrings.py index 523a3764c..4b50dd26c 100644 --- a/tyro/_docstrings.py +++ b/tyro/_docstrings.py @@ -172,7 +172,7 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: for param_doc in docstring_parser.parse(docstring).params: if param_doc.arg_name == field_name: return ( - _strings.postprocess_helptext(param_doc.description) + _strings.remove_single_line_breaks(param_doc.description) if param_doc.description is not None else None ) @@ -198,7 +198,7 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: and first_token_content.startswith('"""') and first_token_content.endswith('"""') ): - return _strings.postprocess_helptext( + return _strings.remove_single_line_breaks( _strings.dedent(first_token_content[3:-3]) ) @@ -209,7 +209,7 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: if final_token_on_line.token_type == tokenize.COMMENT: comment: str = final_token_on_line.content assert comment.startswith("#") - return _strings.postprocess_helptext(comment[1:].strip()) + return _strings.remove_single_line_breaks(comment[1:].strip()) # Check for comments that come before the field. This is intentionally written to # support comments covering multiple (grouped) fields, for example: @@ -262,7 +262,7 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: current_actual_line -= 1 if len(comments) > 0: - return _strings.postprocess_helptext("\n".join(reversed(comments))) + return _strings.remove_single_line_breaks("\n".join(reversed(comments))) return None diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 2292d2c2d..6c4652906 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -185,9 +185,11 @@ def from_callable_or_type( return ParserSpecification( f=f, - description=description - if description is not None - else _docstrings.get_callable_description(f), + description=_strings.remove_single_line_breaks( + description + if description is not None + else _docstrings.get_callable_description(f) + ), args=args, helptext_from_nested_class_field_name=helptext_from_nested_class_field_name, subparsers_from_name=subparsers_from_name, diff --git a/tyro/_strings.py b/tyro/_strings.py index b3e4cfcf5..7d94eb1f1 100644 --- a/tyro/_strings.py +++ b/tyro/_strings.py @@ -120,7 +120,7 @@ def multi_metavar_from_single(single: str) -> str: return f"{single} [{single} ...]" -def postprocess_helptext(helptext: str) -> str: +def remove_single_line_breaks(helptext: str) -> str: lines = helptext.split("\n") output_parts: List[str] = [] for line in lines: @@ -133,6 +133,7 @@ def postprocess_helptext(helptext: str) -> str: if not prev_is_break: output_parts.append("\n") output_parts.append("\n") + # Empty line. else: if not line[0].isalpha(): From c9e2ce834261bc011069bf36f751285c714f2eee Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 10 Oct 2022 10:36:56 -0700 Subject: [PATCH 192/491] Improve default matching for subcommands (closes #15) --- tests/helptext_utils.py | 54 ++++++++++++++ tests/test_helptext.py | 125 ++++++++++---------------------- tests/test_metadata.py | 42 +++++++++++ tyro/_parsers.py | 154 +++++++++++++++++++++++++--------------- tyro/_resolver.py | 4 +- 5 files changed, 231 insertions(+), 148 deletions(-) create mode 100644 tests/helptext_utils.py diff --git a/tests/helptext_utils.py b/tests/helptext_utils.py new file mode 100644 index 000000000..44b94e147 --- /dev/null +++ b/tests/helptext_utils.py @@ -0,0 +1,54 @@ +import argparse +import contextlib +import io +import os +from typing import Callable, List + +import pytest + +import tyro +import tyro._arguments +import tyro._strings + + +def get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(f, args=args) + + # Check tyro.extras.get_parser(). + parser = tyro.extras.get_parser(f) + assert isinstance(parser, argparse.ArgumentParser) + + # Returned parser should have formatting information stripped. External tools rarely + # support ANSI sequences. + unformatted_helptext = parser.format_help() + assert ( + tyro._strings.strip_ansi_sequences(unformatted_helptext) == unformatted_helptext + ) + unformatted_usage = parser.format_usage() + assert tyro._strings.strip_ansi_sequences(unformatted_usage) == unformatted_usage + + # Completion scripts; just smoke test for now. + with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): + tyro.cli(f, args=["--tyro-print-completion", "bash"]) + with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): + tyro.cli(f, args=["--tyro-print-completion", "zsh"]) + + # Check helptext with vs without formatting. This can help catch text wrapping bugs + # caused by ANSI sequences. + target2 = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): + tyro._arguments.USE_RICH = False + tyro.cli(f, args=args) + tyro._arguments.USE_RICH = True + + if target2.getvalue() != tyro._strings.strip_ansi_sequences(target.getvalue()): + raise AssertionError( + "Potential wrapping bug! These two strings should match:\n" + + target2.getvalue() + + "\n\n" + + tyro._strings.strip_ansi_sequences(target.getvalue()) + ) + + return target2.getvalue() diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 3c5930c88..20ef6daf1 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -1,63 +1,14 @@ -import argparse -import contextlib import dataclasses import enum -import io -import os import pathlib from collections.abc import Callable from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar, Union, cast -import pytest import torch.nn as nn +from helptext_utils import get_helptext from typing_extensions import Annotated, Literal import tyro -import tyro._arguments -import tyro._strings - - -def _get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: - target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): - tyro.cli(f, args=args) - - # Check tyro.extras.get_parser(). - parser = tyro.extras.get_parser(f) - assert isinstance(parser, argparse.ArgumentParser) - - # Returned parser should have formatting information stripped. External tools rarely - # support ANSI sequences. - unformatted_helptext = parser.format_help() - assert ( - tyro._strings.strip_ansi_sequences(unformatted_helptext) == unformatted_helptext - ) - unformatted_usage = parser.format_usage() - assert tyro._strings.strip_ansi_sequences(unformatted_usage) == unformatted_usage - - # Completion scripts; just smoke test for now. - with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): - tyro.cli(f, args=["--tyro-print-completion", "bash"]) - with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): - tyro.cli(f, args=["--tyro-print-completion", "zsh"]) - - # Check helptext with vs without formatting. This can help catch text wrapping bugs - # caused by ANSI sequences. - target2 = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): - tyro._arguments.USE_RICH = False - tyro.cli(f, args=args) - tyro._arguments.USE_RICH = True - - if target2.getvalue() != tyro._strings.strip_ansi_sequences(target.getvalue()): - raise AssertionError( - "Potential wrapping bug! These two strings should match:\n" - + target2.getvalue() - + "\n\n" - + tyro._strings.strip_ansi_sequences(target.getvalue()) - ) - - return target2.getvalue() def test_helptext(): @@ -73,7 +24,7 @@ class Helptext: z: int = 3 """Documentation 3""" - helptext = _get_helptext(Helptext) + helptext = get_helptext(Helptext) assert cast(str, helptext) in helptext assert "x INT" in helptext assert "y INT" in helptext @@ -98,7 +49,7 @@ class Helptext2: y: Annotated[int, "ignored"] z: int = 3 - helptext = _get_helptext(Helptext2) + helptext = get_helptext(Helptext2) assert "This docstring should be printed as a description" in helptext assert "Attributes" not in helptext assert "x INT" in helptext @@ -124,7 +75,7 @@ class Helptext3: y: Annotated[int, "ignored"] z: int = 3 - helptext = _get_helptext(Helptext3) + helptext = get_helptext(Helptext3) assert "This docstring should be printed as a description" in helptext assert "Args" not in helptext assert "x INT" in helptext @@ -159,7 +110,7 @@ def some_method(self) -> None: # noqa class ChildClass(UnrelatedParentClass, ActualParentClass): pass - helptext = _get_helptext(ChildClass) + helptext = get_helptext(ChildClass) assert "Documentation 1" in helptext assert "Documentation 2" in helptext @@ -188,7 +139,7 @@ class ChildClass(ParentClass): def main(x: ParentClass = ChildClass(x=5, y=5)) -> Any: return x - helptext = _get_helptext(main) + helptext = get_helptext(main) assert "Documentation 1" in helptext assert "Documentation 2" in helptext assert "__not__" not in helptext @@ -217,12 +168,12 @@ def main_with_docstring(a: Inner) -> None: def main_no_docstring(a: Inner) -> None: """main_no_docstring.""" - helptext = _get_helptext(main_with_docstring) + helptext = get_helptext(main_with_docstring) assert "Documented in function" in helptext and str(Inner.__doc__) not in helptext assert "Args:" not in helptext assert "Hello world!" in helptext - helptext = _get_helptext(main_no_docstring) + helptext = get_helptext(main_no_docstring) assert "Something" in helptext assert "Args:" not in helptext assert "Hello world!" in helptext @@ -239,7 +190,7 @@ class HelptextWithVariousDefaults: x: pathlib.Path = pathlib.Path("/some/path/to/a/file") y: Color = Color.RED - helptext = _get_helptext(HelptextWithVariousDefaults) + helptext = get_helptext(HelptextWithVariousDefaults) assert "show this help message and exit" in helptext assert "--x PATH" in helptext assert "(default: /some/path/to/a/file)" in helptext @@ -262,7 +213,7 @@ class HelptextMultiline: """Documentation 3 Next line of documentation 3""" - helptext = _get_helptext(HelptextMultiline) + helptext = get_helptext(HelptextMultiline) assert "Documentation 1 (required)" in helptext assert "Documentation 2" in helptext assert "documentation 2" in helptext @@ -278,7 +229,7 @@ class HelptextGrouped: y: int z: int = 3 - helptext = _get_helptext(HelptextGrouped) + helptext = get_helptext(HelptextGrouped) assert "Documentation 1 (required)" in helptext assert "Description of both y and z. (required)" in helptext assert "Description of both y and z. (default: 3)" in helptext @@ -290,7 +241,7 @@ class Config: x: Optional[int] = None """An optional variable.""" - helptext = _get_helptext(Config) + helptext = get_helptext(Config) assert "--x {None}|INT" in helptext assert "An optional variable. (default: None)" in helptext @@ -308,7 +259,7 @@ class HelptextHardString: # Note that the percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string - helptext = _get_helptext(HelptextHardString) + helptext = get_helptext(HelptextHardString) assert "--x" in helptext assert "2% milk." in helptext @@ -327,7 +278,7 @@ class Parent: class Child(Parent): pass - helptext = _get_helptext(Child) + helptext = get_helptext(Child) assert "--x STR" in helptext assert "Helptext." in helptext assert "(default: 'This docstring" in helptext @@ -352,7 +303,7 @@ class Child2(Parent2): """Helptext!""" # fmt: on - helptext = _get_helptext(Child2) + helptext = get_helptext(Child2) assert "--x STR" in helptext assert "Helptext! (default: 'This" in helptext @@ -362,7 +313,7 @@ def test_tuple_helptext(): class TupleHelptext: x: Tuple[int, str, float] - helptext = _get_helptext(TupleHelptext) + helptext = get_helptext(TupleHelptext) assert "--x INT STR FLOAT" in helptext @@ -371,7 +322,7 @@ def test_tuple_helptext_defaults(): class TupleHelptextDefaults: x: Tuple[int, str, str] = (5, "hello world", "hello") - helptext = _get_helptext(TupleHelptextDefaults) + helptext = get_helptext(TupleHelptextDefaults) assert "--x INT STR STR" in helptext assert "(default: 5 'hello world' hello)" in helptext @@ -383,7 +334,7 @@ def test_generic_helptext(): class GenericTupleHelptext(Generic[T]): x: T - helptext = _get_helptext(GenericTupleHelptext[int]) + helptext = get_helptext(GenericTupleHelptext[int]) assert "--x INT" in helptext @@ -394,7 +345,7 @@ def test_generic_tuple_helptext(): class GenericTupleHelptext(Generic[T]): x: Tuple[T, T, T] - helptext = _get_helptext(GenericTupleHelptext[int]) + helptext = get_helptext(GenericTupleHelptext[int]) assert "--x INT INT INT" in helptext @@ -405,7 +356,7 @@ def test_generic_list_helptext(): class GenericTupleHelptext(Generic[T]): x: List[T] - helptext = _get_helptext(GenericTupleHelptext[int]) + helptext = get_helptext(GenericTupleHelptext[int]) assert "--x INT [INT ...]" in helptext @@ -415,7 +366,7 @@ class LiteralHelptext: x: Literal[1, 2, 3] """A number.""" - helptext = _get_helptext(LiteralHelptext) + helptext = get_helptext(LiteralHelptext) assert "--x {1,2,3}" in helptext assert "A number. (required)" in helptext @@ -426,7 +377,7 @@ class OptionalLiteralHelptext: x: Optional[Literal[1, 2, 3]] = None """A number.""" - helptext = _get_helptext(OptionalLiteralHelptext) + helptext = get_helptext(OptionalLiteralHelptext) assert "--x {None,1,2,3}" in helptext assert "A number. (default: None)" in helptext @@ -457,14 +408,14 @@ class MultipleSubparsers: default_factory=Subcommand3 ) - helptext = _get_helptext(MultipleSubparsers) + helptext = get_helptext(MultipleSubparsers) assert "2% milk." in helptext assert "Field a description." in helptext assert "Field b description." not in helptext assert "Field c description." not in helptext - helptext = _get_helptext( + helptext = get_helptext( MultipleSubparsers, args=["a:subcommand1", "b:subcommand1", "--help"] ) @@ -488,7 +439,7 @@ class OptionalHelptext: z: Optional[int] = 3 """Documentation 3""" - helptext = _get_helptext(OptionalHelptext) + helptext = get_helptext(OptionalHelptext) assert cast(str, OptionalHelptext.__doc__[:20]) in helptext assert "2% milk" in helptext assert "--x {None}|INT" in helptext @@ -500,7 +451,7 @@ def test_metavar_0(): def main(x: Union[Literal[0, 1, 2, 3], Tuple[int, int]]) -> None: pass - helptext = _get_helptext(main) + helptext = get_helptext(main) assert "--x {0,1,2,3}|{INT INT}" in helptext @@ -515,7 +466,7 @@ def main( pass # The comma formatting is unfortunate, but matches argparse's default behavior. - helptext = _get_helptext(main) + helptext = get_helptext(main) assert "--x {0,1,2,3,hey,there,hello}|{INT [INT ...]}" in helptext @@ -528,7 +479,7 @@ def main( ) -> None: pass - helptext = _get_helptext(main) + helptext = get_helptext(main) assert "--x {0,1,2,3} INT|STR" in helptext @@ -541,7 +492,7 @@ def main( ) -> None: pass - helptext = _get_helptext(main) + helptext = get_helptext(main) assert "--x {0,1,2,3}|{INT INT}|STR" in helptext @@ -555,7 +506,7 @@ def main( ) -> None: pass - helptext = _get_helptext(main) + helptext = get_helptext(main) assert "--x {0,1,2,3}|{INT INT}|{STR STR STR}|{True}" in helptext @@ -567,7 +518,7 @@ def main( ) -> None: pass - helptext = _get_helptext(main) + helptext = get_helptext(main) assert "[--x {INT INT}|{STR STR} [{INT INT}|{STR STR} ...]]" in helptext @@ -575,7 +526,7 @@ def test_metavar_6(): def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> dict: return x - helptext = _get_helptext(main) + helptext = get_helptext(main) assert ( "--x {INT INT}|{STR STR} INT INT [{INT INT}|{STR STR} INT INT ...]" in helptext ) @@ -592,7 +543,7 @@ class Something( # But this text should! b: int - helptext = _get_helptext(Something) + helptext = get_helptext(Something) assert "This text should not" not in helptext assert "But this text should!" in helptext @@ -605,13 +556,13 @@ class Struct: def main(x: Any = Struct()): pass - helptext = _get_helptext(main) + helptext = get_helptext(main) assert "--x {fixed}" in helptext def main2(x: Callable = nn.ReLU): pass - helptext = _get_helptext(main2) + helptext = get_helptext(main2) assert "--x {fixed}" in helptext assert "(fixed to:" in helptext assert "torch" in helptext @@ -626,7 +577,7 @@ class Struct: def main(x: Any = Struct()): pass - helptext = _get_helptext(main) + helptext = get_helptext(main) assert "--x.a" in helptext assert "--x.b" not in helptext @@ -640,7 +591,7 @@ class Struct: def main(x: Any = Struct()): pass - helptext = _get_helptext(main) + helptext = get_helptext(main) assert "--x.a" in helptext assert "--x.b" not in helptext @@ -654,6 +605,6 @@ class Struct: def main(x: tyro.conf.SuppressFixed[Any] = Struct()): pass - helptext = _get_helptext(main) + helptext = get_helptext(main) assert "--x.a" in helptext assert "--x.b" not in helptext diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 3b58da893..4573c8ae7 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -2,6 +2,7 @@ from typing import Generic, TypeVar, Union import pytest +from helptext_utils import get_helptext from typing_extensions import Annotated import tyro @@ -299,6 +300,47 @@ class Parent: ) == Parent(Nested1(Nested2(B(7)))) +def test_subparser_in_nested_with_metadata_default_matching(): + @dataclasses.dataclass(frozen=True) + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + default_one = B(3) + default_two = B(9) + + @dataclasses.dataclass + class Nested: + subcommand: Union[ + # Annotated[A, tyro.conf.subcommand("zero")], + Annotated[B, tyro.conf.subcommand("one", default=default_one)], + Annotated[B, tyro.conf.subcommand("two", default=default_two)], + Annotated[B, tyro.conf.subcommand("three")], + ] + + # Match by hash. + def main_one(x: Nested = Nested(default_one)) -> None: + pass + + assert "default: x.subcommand:one" in get_helptext(main_one) + + # Match by value. + def main_two(x: Nested = Nested(B(9))) -> None: + pass + + assert "default: x.subcommand:two" in get_helptext(main_two) + + # Match by type. + def main_three(x: Nested = Nested(B(15))) -> None: + pass + + assert "default: x.subcommand:three" in get_helptext(main_three) + + def test_flag(): """When boolean flags have no default value, they must be explicitly specified.""" diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 6c4652906..5879f0e2c 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -290,27 +290,97 @@ def from_field( ): return None - # Add subparser for each option. - parser_from_name: Dict[str, ParserSpecification] = {} + # Get subcommand configurations from `tyro.conf.subcommand()`. + subcommand_config_from_name: Dict[ + str, _subcommands._SubcommandConfiguration + ] = {} + subcommand_name_from_default_hash: Dict[int, str] = {} + subcommand_name_from_type: Dict[Type, str] = {} # Used for default matching. for option in options_no_none: - name = _strings.subparser_name_from_type(prefix, option) + subcommand_name = _strings.subparser_name_from_type(prefix, option) option, found_subcommand_configs = _resolver.unwrap_annotated( option, _subcommands._SubcommandConfiguration ) - if len(found_subcommand_configs) == 0: - # Make a dummy subcommand config. - found_subcommand_configs = ( - _subcommands._SubcommandConfiguration( - "unused", - description=None, - default=( - field.default - if type(field.default) - is _resolver.unwrap_origin_strip_extras(option) - else _fields.MISSING_NONPROP - ), - prefix_name=True, - ), + default_hash = None + if len(found_subcommand_configs) != 0: + # Explicitly annotated default. + assert len(found_subcommand_configs) == 1, ( + f"Expected only one subcommand config, but {subcommand_name} has" + f" {len(found_subcommand_configs)}." + ) + subcommand_config_from_name[subcommand_name] = found_subcommand_configs[ + 0 + ] + + if ( + found_subcommand_configs[0].default + not in _fields.MISSING_SINGLETONS + ): + default_hash = object.__hash__(found_subcommand_configs[0].default) + subcommand_name_from_default_hash[default_hash] = subcommand_name + + # Use subcommand types for default matching if no default is explicitly + # annotated. + if default_hash is None: + subcommand_name_from_type[option] = subcommand_name + + # If there are any required arguments in the default subparser, we should mark + # the subparser group as a whole as required. + default_name = None + if ( + field.default is not None + and field.default not in _fields.MISSING_SINGLETONS + ): + # It's really hard to concretize a generic type at runtime, so we just... + # don't. :-) + if hasattr(type(field.default), "__parameters__"): + raise _instantiators.UnsupportedTypeAnnotationError( + "Default values for generic subparsers are not supported." + ) + + # Get default subcommand name: by default hash. + default_hash = object.__hash__(field.default) + default_name = subcommand_name_from_default_hash.get(default_hash, None) + + # Get default subcommand name: by default value. + if default_name is None: + for ( + subcommand_name, + subcommand_config, + ) in subcommand_config_from_name.items(): + equal = field.default == subcommand_config.default + if isinstance(equal, bool) and equal: + default_name = subcommand_name + break + + # Get default subcommand name: by default type. + if default_name is None: + default_name = subcommand_name_from_type.get(type(field.default), None) + + assert default_name is not None + + # Add subcommands for each option. + parser_from_name: Dict[str, ParserSpecification] = {} + for option in options_no_none: + subcommand_name = _strings.subparser_name_from_type(prefix, option) + option, _ = _resolver.unwrap_annotated(option) + + # Get a subcommand config: either pulled from the type annotations or the + # field default. + if subcommand_name in subcommand_config_from_name: + subcommand_config = subcommand_config_from_name[subcommand_name] + else: + subcommand_config = _subcommands._SubcommandConfiguration( + "unused", + description=None, + default=_fields.MISSING_NONPROP, + prefix_name=True, + ) + + # If names match, borrow subcommand default from field default. + if default_name == subcommand_name: + subcommand_config = dataclasses.replace( + subcommand_config, default=field.default ) subparser = ParserSpecification.from_callable_or_type( @@ -318,10 +388,10 @@ def from_field( Annotated.__class_getitem__((option,) + tuple(field.markers)) # type: ignore if len(field.markers) > 0 else option, - description=found_subcommand_configs[0].description, + description=subcommand_config.description, parent_classes=parent_classes, parent_type_from_typevar=type_from_typevar, - default_instance=found_subcommand_configs[0].default, + default_instance=subcommand_config.default, prefix=prefix, subcommand_prefix=prefix, ) @@ -334,48 +404,14 @@ def from_field( for k, v in subparser.helptext_from_nested_class_field_name.items() }, ) - parser_from_name[name] = subparser + parser_from_name[subcommand_name] = subparser - # Optional if: type hint is Optional[], or a default instance is provided. - required = True - if field.default not in _fields.MISSING_SINGLETONS: - required = False + # Required if a default is missing. + required = field.default in _fields.MISSING_SINGLETONS - # If there are any required arguments in the default subparser, we should mark - # the subparser group as a whole as required. - default_name = None - if ( - field.default is not None - and field.default not in _fields.MISSING_SINGLETONS - ): - # It's really hard to concretize a generic type at runtime, so we just... - # don't. :-) - if hasattr(type(field.default), "__parameters__"): - raise _instantiators.UnsupportedTypeAnnotationError( - "Default values for generic subparsers are not supported." - ) - - default_name = _strings.subparser_name_from_type( - prefix, type(field.default) - ) - if default_name not in parser_from_name: - # If we can't find the subparser by name, search by type. This is needed - # when the user renames their subcommands. (eg via tyro.subcommand) - # - # TODO: this will display some weird behaviors if multiple subcommands - # have the same type. - default_name = None - - for name, parser in parser_from_name.items(): - if type(field.default) is _resolver.unwrap_origin_strip_extras( - parser.f - ): - default_name = name - break - assert default_name is not None, ( - f"Default with type {type(field.default)} was passed in, but no" - " matching subparser." - ) + # Required if a default is passed in, but the default value has missing + # parameters. + if default_name is not None: default_parser = parser_from_name[default_name] if any(map(lambda arg: arg.lowered.required, default_parser.args)): required = True diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 1abcaf3fb..54bac4be8 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -142,8 +142,8 @@ def unwrap_annotated( Examples: - int, int => (int, ()) - - Annotated[int, 1], int => (int, 1) - - Annotated[int, "1"], int => (int, None) + - Annotated[int, 1], int => (int, (1,)) + - Annotated[int, "1"], int => (int, ()) """ if not hasattr(typ, "__metadata__"): return typ, () From 8d199cfa49a4d1b791b4ec3c52ac98ca4245f8a8 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 10 Oct 2022 11:02:29 -0700 Subject: [PATCH 193/491] README updates --- README.md | 18 ++++++++------- docs/source/building_configuration_systems.md | 6 ++--- docs/source/goals_and_alternatives.md | 18 +++++++-------- docs/source/index.md | 18 ++++++++------- docs/source/your_first_cli.md | 23 +++++++++---------- 5 files changed, 43 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 07696c3a9..037172d21 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +

@@ -5,7 +6,6 @@ tyro logo -

Documentation   •   @@ -146,11 +146,13 @@ For more examples, see our [documentation](https://brentyi.github.io/tyro). `tyro` is still a new library, but being stress tested in several projects! -- [nerfstudio](https://github.com/nerfstudio-project/nerfstudio/) uses `tyro` - both to build compact command-line utilities and for YAML-free experiment - configuration. -- [obj2mjcf](https://github.com/kevinzakka/obj2mjcf) uses `tyro` to build a CLI +- [nerfstudio-project/nerfstudio](https://github.com/nerfstudio-project/nerfstudio/) + provides a set of tools for end-to-end training, testing, and rendering of + neural radiance fields. +- [Sea-Snell/JAXSeq](https://github.com/Sea-Snell/JAXSeq/) is a library for + distributed training of large language models in JAX. +- [kevinzakka/obj2mjcf](https://github.com/kevinzakka/obj2mjcf) is an interface for processing composite Wavefront OBJ files for Mujoco. -- [tensorf-jax](https://github.com/brentyi/tensorf-jax/) (unofficially) - implements [Tensorial Radiance Fields](https://apchenstu.github.io/TensoRF/) - in JAX, with `tyro` for configuration. +- [brentyi/tensorf-jax](https://github.com/brentyi/tensorf-jax/) is an + unofficial implementation of + [Tensorial Radiance Fields](https://apchenstu.github.io/TensoRF/) in JAX. diff --git a/docs/source/building_configuration_systems.md b/docs/source/building_configuration_systems.md index a79322d5b..1e6941018 100644 --- a/docs/source/building_configuration_systems.md +++ b/docs/source/building_configuration_systems.md @@ -4,10 +4,10 @@ Beyond building simple command-line interfaces, :func:`tyro.cli()` is designed to scale to larger configuration systems such as those typically built with libraries like [hydra](https://github.com/facebookresearch/hydra). -For a live example of this, see +For a live example, see [nerfstudio](https://github.com/nerfstudio-project/nerfstudio/). Notably, -`nerfstudio`'s configuration system is implemented entirely in Python, no YAML -needed, and has full tab completion support in your terminal. +`nerfstudio`'s configuration system is implemented entirely in Python, without +YAML, and has full tab completion support in your terminal. For overriding a dynamically loaded configuration object (typically a dataclass), the `default=` parameter of :func:`tyro.cli()` can be used. If you diff --git a/docs/source/goals_and_alternatives.md b/docs/source/goals_and_alternatives.md index 900172b9a..e5f36e0ff 100644 --- a/docs/source/goals_and_alternatives.md +++ b/docs/source/goals_and_alternatives.md @@ -7,9 +7,9 @@ annotations — overlaps significantly with features offered by other libraries. Usage distinctions are the result of two API goals: - **One uninvasive function.** For all core functionality, learning to use - `tyro` should reduce to learning to write (type-annotated) Python. For - example, types are specified using standard annotations, helptext using - docstrings, choices using the standard `typing.Literal` type, subcommands with + `tyro` should reduce to learning to write type-annotated Python. For example, + types are specified using standard annotations, helptext using docstrings, + choices using the standard `typing.Literal` type, subcommands with `typing.Union` of nested types, and positional arguments with `/`. - In contrast, similar libraries have more expansive APIs , and require more library-specific structures, decorators, or metadata formats for configuring @@ -37,7 +37,7 @@ More concretely, we can also compare specific features. A noncomprehensive set: | [pyrallis][pyrallis] | ✓ | | | ✓ | ✓ | | | ✓ | | | | [yahp][yahp] | ✓ | | | ~[^yahp_docstrings] | ✓ | ✓ | ~[^yahp_unions_nested] | ✓ | | | | [omegaconf][omegaconf] | ✓ | | | | ✓ | | | ✓ | ✓ | | -| **tyro** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| **tyro** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | @@ -66,13 +66,13 @@ More concretely, we can also compare specific features. A noncomprehensive set: -Note that most of these other libraries are generally aimed specifically at only -one of dataclasses (`datargs`, `simple-parsing`, `argparse-dataclass`, +Note that other libraries are generally aimed specifically at only one of +dataclasses (`datargs`, `simple-parsing`, `argparse-dataclass`, `argparse-dataclasses`, `dataclass-cli`, `clout`, `hf_argparser`, `pyrallis`, `yahp`), custom structures (`tap`), or functions (`typer`) rather than general -typed callables, but offer other features that you might find useful, such as -registration for custom types (`pyrallis`), built-in approaches for -serialization and config files (`tap`, `pyrallis`, `yahp`) and, opportunities +types and callables, but offer other features that you might find critical, such +as registration for custom types (`pyrallis`), built-in approaches for +serialization and config files (`tap`, `pyrallis`, `yahp`), and opportunities for integration with fields generated using standard argparse definitions (`simple-parsing`). diff --git a/docs/source/index.md b/docs/source/index.md index b31ed7d4c..4505cf013 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -48,14 +48,16 @@ To get started, we recommend browsing the examples to the left. `tyro` is still a new library, but being stress tested in several projects! -- [nerfstudio](https://github.com/nerfstudio-project/nerfstudio/) uses `tyro` - both to build compact command-line utilities and for YAML-free experiment - configuration. -- [obj2mjcf](https://github.com/kevinzakka/obj2mjcf) uses `tyro` to build a CLI +- [nerfstudio-project/nerfstudio](https://github.com/nerfstudio-project/nerfstudio/) + provides a set of tools for end-to-end training, testing, and rendering of + neural radiance fields. +- [Sea-Snell/JAXSeq](https://github.com/Sea-Snell/JAXSeq/) is a library for + distributed training of large language models in JAX. +- [kevinzakka/obj2mjcf](https://github.com/kevinzakka/obj2mjcf) is an interface for processing composite Wavefront OBJ files for Mujoco. -- [tensorf-jax](https://github.com/brentyi/tensorf-jax/) (unofficially) - implements [Tensorial Radiance Fields](https://apchenstu.github.io/TensoRF/) - in JAX, with `tyro` for configuration. +- [brentyi/tensorf-jax](https://github.com/brentyi/tensorf-jax/) is an + unofficial implementation of + [Tensorial Radiance Fields](https://apchenstu.github.io/TensoRF/) in JAX. @@ -74,10 +76,10 @@ To get started, we recommend browsing the examples to the left. :hidden: :glob: + goals_and_alternatives helptext_generation tab_completion building_configuration_systems - goals_and_alternatives .. toctree:: :caption: Examples diff --git a/docs/source/your_first_cli.md b/docs/source/your_first_cli.md index cf46b3243..57bf1185c 100644 --- a/docs/source/your_first_cli.md +++ b/docs/source/your_first_cli.md @@ -7,24 +7,23 @@ command-line interface: """Sum two numbers from argparse.""" import argparse + parser = argparse.ArgumentParser() -parser.add_argument( - "--a", - type=int, - required=True, -) -parser.add_argument( - "--b", - type=int, - default=3, -) +parser.add_argument("--a", type=int, required=True) +parser.add_argument("--b", type=int, default=3) args = parser.parse_args() print(args.a + args.b) ``` -The basic feature of :func:`tyro.cli()` is to enable the removal of -parsing-specific boilerplate. +This is dramatically cleaner than manually parsing `sys.argv`, but has several +issues: it lacks type checking and IDE support (consider: jumping to +definitions, finding references, docstrings, refactoring and renaming tools), +requires a significant amount of boilerplate, and generally becomes difficult to +manage as interfaces grow. + +The basic feature of :func:`tyro.cli()` is to provide a wrapper for `argparse` +that solves these issues. We can specify the same logic with a function signature: From 8634789560802f85cb0a24e585216b3d652d0200 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 13 Oct 2022 18:25:15 -0700 Subject: [PATCH 194/491] Fix nested dictionaries, bump version --- pyproject.toml | 2 +- tests/test_dict_namedtuple.py | 104 ++++++++++++++++++++++++++++++++++ tyro/_parsers.py | 2 +- 3 files changed, 106 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3191839ac..243532738 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.23" +version = "0.3.24" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index f7741e1ea..c6a2855b6 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -1,4 +1,5 @@ import contextlib +import copy import dataclasses import io import pathlib @@ -291,3 +292,106 @@ class HelptextNamedTuple(NamedTuple): assert "Documentation 2 (required)" in helptext assert "Documentation 3" in helptext assert "(default: 3)" in helptext + + +def test_nested_dict(): + loaded_config = { + "batch_size": 32, + "optimizer": { + "learning_rate": 1e-4, + "epsilon": 1e-8, + "scheduler": {"schedule_type": "constant"}, + }, + } + backup_config = copy.deepcopy(loaded_config) + overrided_config = tyro.cli( + dict, + default=loaded_config, + args=[ + "--batch-size", + "16", + "--optimizer.scheduler.schedule-type", + "exponential", + ], + ) + + # Overridden config should be different from loaded config. + assert overrided_config != loaded_config + assert overrided_config["batch_size"] == 16 + assert overrided_config["optimizer"]["scheduler"]["schedule_type"] == "exponential" + + # Original loaded config should not be mutated. + assert loaded_config == backup_config + + +def test_nested_dict_hyphen(): + # We do a lot of underscore <=> conversion in the code; this is just to make sure it + # doesn't break anything! + loaded_config = { + "batch-size": 32, + "optimizer": { + "learning-rate": 1e-4, + "epsilon": 1e-8, + "scheduler": {"schedule-type": "constant"}, + }, + } + backup_config = copy.deepcopy(loaded_config) + overrided_config = tyro.cli( + dict, + default=loaded_config, + args=[ + "--batch-size", + "16", + "--optimizer.scheduler.schedule-type", + "exponential", + ], + ) + + # Overridden config should be different from loaded config. + assert overrided_config != loaded_config + assert overrided_config["batch-size"] == 16 + assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" + + # Original loaded config should not be mutated. + assert loaded_config == backup_config + + +def test_nested_dict_annotations(): + loaded_config = { + "optimizer": { + "scheduler": {"schedule-type": "constant"}, + }, + } + + overrided_config = tyro.cli( + dict, + default=loaded_config, + args=[ + "--optimizer.scheduler.schedule-type", + "exponential", + ], + ) + assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" + del overrided_config + + overrided_config = tyro.cli( + Dict[str, Dict], + default=loaded_config, + args=[ + "--optimizer.scheduler.schedule-type", + "exponential", + ], + ) + assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" + del overrided_config + + overrided_config = tyro.cli( + Dict[str, Dict[str, str]], + default=loaded_config, + args=[ + "--optimizer.scheduler.schedule-type", + "exponential", + ], + ) + assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" + del overrided_config diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 5879f0e2c..638f58604 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -61,7 +61,7 @@ def from_callable_or_type( # # Note that 'parent' here refers to in the nesting hierarchy, not the # superclass. - if f in parent_classes: + if f in parent_classes and f is not dict: raise _instantiators.UnsupportedTypeAnnotationError( f"Found a cyclic dataclass dependency with type {f}." ) From b98b7583541c00df9e515d79e3a014a5310a417e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 13 Oct 2022 18:41:34 -0700 Subject: [PATCH 195/491] Add overloads for get_parser() --- tyro/_cli.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tyro/_cli.py b/tyro/_cli.py index eafc68703..0338f0e1c 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -123,6 +123,28 @@ def cli( ) +@overload +def get_parser( + f: Type[OutT], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + default: Optional[OutT] = None, +) -> argparse.ArgumentParser: + ... + + +@overload +def get_parser( + f: Callable[..., OutT], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + default: Optional[OutT] = None, +) -> argparse.ArgumentParser: + ... + + def get_parser( f: Union[Type[OutT], Callable[..., OutT]], *, From 5fc10f0369339db670d042c008a222c4369974ef Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 13 Oct 2022 18:44:56 -0700 Subject: [PATCH 196/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 243532738..3ea20eb7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.24" +version = "0.3.25" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] From 5c3bc9ab6df08cade2b7695170670436540643a6 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 17 Oct 2022 13:47:07 -0700 Subject: [PATCH 197/491] Improve type narrowing for containers --- pyproject.toml | 2 +- tests/test_collections.py | 21 +++++++++++++++++++++ tyro/_parsers.py | 13 ++++++++----- tyro/_resolver.py | 10 +++++++++- 4 files changed, 39 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3ea20eb7a..a35f7a24b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.25" +version = "0.3.26" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] diff --git a/tests/test_collections.py b/tests/test_collections.py index 4e670e607..e4eab8637 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -450,3 +450,24 @@ def main( "str": "7", } } + + +def test_list_narrowing(): + def main(x: list = [0, 1, 2, "hello"]) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", 5] + + +def test_set_narrowing(): + def main(x: set = {0, 1, 2, "hello"}) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", 5} + + +def test_tuple_narrowing(): + def main(x: tuple = (0, 1, 2, "hello")) -> Any: + return x + + assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == (0, 1, 2, "3") diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 638f58604..ad5aa97ba 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -83,11 +83,14 @@ def from_callable_or_type( field = dataclasses.replace( field, # Resolve generic types. - typ=_resolver.type_from_typevar_constraints( - _resolver.apply_type_from_typevar( - field.typ, - type_from_typevar, - ) + typ=_resolver.narrow_type( + _resolver.type_from_typevar_constraints( # type: ignore + _resolver.apply_type_from_typevar( + field.typ, + type_from_typevar, + ) + ), + default_instance=field.default, ), ) diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 54bac4be8..3ed9daf16 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -126,9 +126,17 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: return Annotated.__class_getitem__( # type: ignore (potential_subclass,) + get_args(typ)[1:] ) - return cast(TypeT, potential_subclass) + typ = cast(TypeT, potential_subclass) except TypeError: pass + + if typ is list and isinstance(default_instance, list): + typ = List.__getitem__(Union.__getitem__(tuple(map(type, default_instance)))) # type: ignore + elif typ is set and isinstance(default_instance, set): + typ = Set.__getitem__(Union.__getitem__(tuple(map(type, default_instance)))) # type: ignore + elif typ is tuple and isinstance(default_instance, tuple): + typ = Tuple.__getitem__(tuple(map(type, default_instance))) # type: ignore + return typ From fbefca6b471b74e62c30c85d5408b346bed98f99 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 18 Oct 2022 17:20:30 -0700 Subject: [PATCH 198/491] Documentation rework --- README.md | 7 +- docs/source/building_configuration_systems.md | 3 +- .../examples/{ => 01_basics}/01_functions.rst | 14 ++-- .../{ => 01_basics}/02_dataclasses.rst | 21 +++--- .../03_containers.rst} | 27 +++----- docs/source/examples/01_basics/04_enums.rst | 65 +++++++++++++++++++ .../{04_flags.rst => 01_basics/05_flags.rst} | 18 ++--- .../source/examples/01_basics/06_literals.rst | 49 ++++++++++++++ .../07_unions.rst} | 22 ++----- .../01_nesting.rst} | 14 ++-- .../02_subcommands.rst} | 22 +++---- .../03_multiple_subcommands.rst} | 18 ++--- .../04_nesting_in_containers.rst} | 6 +- .../01_base_configs.rst} | 22 +++---- .../03_config_systems/02_overriding_yaml.rst | 62 ++++++++++++++++++ .../01_positional_args.rst} | 10 +-- .../02_dictionaries.rst} | 18 ++--- .../03_tuples.rst} | 14 ++-- .../04_classes.rst} | 10 +-- .../05_generics.rst} | 6 +- .../06_conf.rst} | 6 +- .../07_flax.rst} | 12 ++-- docs/source/index.md | 41 ++++++++++-- docs/update_example_docs.py | 31 +++++---- examples/{ => 01_basics}/01_functions.py | 4 +- examples/{ => 01_basics}/02_dataclasses.py | 11 +++- .../03_containers.py} | 21 +++--- examples/01_basics/04_enums.py | 36 ++++++++++ .../{04_flags.py => 01_basics/05_flags.py} | 12 ++-- examples/01_basics/06_literals.py | 34 ++++++++++ .../07_unions.py} | 20 ++---- .../01_nesting.py} | 10 +-- .../02_subcommands.py} | 14 ++-- .../03_multiple_subcommands.py} | 12 ++-- .../04_nesting_in_containers.py} | 6 +- .../01_base_configs.py} | 4 +- .../03_config_systems/02_overriding_yaml.py | 40 ++++++++++++ .../01_positional_args.py} | 8 ++- .../02_dictionaries.py} | 12 ++-- .../03_tuples.py} | 10 +-- .../04_classes.py} | 8 ++- .../05_generics.py} | 6 +- .../06_conf.py} | 6 +- .../07_flax.py} | 8 ++- examples/_rename_example.py | 3 + 45 files changed, 568 insertions(+), 235 deletions(-) rename docs/source/examples/{ => 01_basics}/01_functions.rst (64%) rename docs/source/examples/{ => 01_basics}/02_dataclasses.rst (53%) rename docs/source/examples/{03_enums_and_containers.rst => 01_basics/03_containers.rst} (57%) create mode 100644 docs/source/examples/01_basics/04_enums.rst rename docs/source/examples/{04_flags.rst => 01_basics/05_flags.rst} (65%) create mode 100644 docs/source/examples/01_basics/06_literals.rst rename docs/source/examples/{06_literals_and_unions.rst => 01_basics/07_unions.rst} (65%) rename docs/source/examples/{05_hierarchical_configs.rst => 02_nesting/01_nesting.rst} (79%) rename docs/source/examples/{08_subcommands.rst => 02_nesting/02_subcommands.rst} (60%) rename docs/source/examples/{09_multiple_subcommands.rst => 02_nesting/03_multiple_subcommands.rst} (68%) rename docs/source/examples/{15_nesting_in_containers.rst => 02_nesting/04_nesting_in_containers.rst} (90%) rename docs/source/examples/{10_base_configs.rst => 03_config_systems/01_base_configs.rst} (83%) create mode 100644 docs/source/examples/03_config_systems/02_overriding_yaml.rst rename docs/source/examples/{07_positional_args.rst => 04_additional/01_positional_args.rst} (83%) rename docs/source/examples/{11_dictionaries.rst => 04_additional/02_dictionaries.rst} (71%) rename docs/source/examples/{12_tuples.rst => 04_additional/03_tuples.rst} (66%) rename docs/source/examples/{13_standard_classes.rst => 04_additional/04_classes.rst} (73%) rename docs/source/examples/{14_generics.rst => 04_additional/05_generics.rst} (87%) rename docs/source/examples/{16_advanced_configuration.rst => 04_additional/06_conf.rst} (92%) rename docs/source/examples/{17_flax_modules.rst => 04_additional/07_flax.rst} (79%) rename examples/{ => 01_basics}/01_functions.py (83%) rename examples/{ => 01_basics}/02_dataclasses.py (66%) rename examples/{03_enums_and_containers.py => 01_basics/03_containers.py} (54%) create mode 100644 examples/01_basics/04_enums.py rename examples/{04_flags.py => 01_basics/05_flags.py} (69%) create mode 100644 examples/01_basics/06_literals.py rename examples/{06_literals_and_unions.py => 01_basics/07_unions.py} (62%) rename examples/{05_hierarchical_configs.py => 02_nesting/01_nesting.py} (83%) rename examples/{08_subcommands.py => 02_nesting/02_subcommands.py} (67%) rename examples/{09_multiple_subcommands.py => 02_nesting/03_multiple_subcommands.py} (77%) rename examples/{15_nesting_in_containers.py => 02_nesting/04_nesting_in_containers.py} (89%) rename examples/{10_base_configs.py => 03_config_systems/01_base_configs.py} (96%) create mode 100644 examples/03_config_systems/02_overriding_yaml.py rename examples/{07_positional_args.py => 04_additional/01_positional_args.py} (87%) rename examples/{11_dictionaries.py => 04_additional/02_dictionaries.py} (75%) rename examples/{12_tuples.py => 04_additional/03_tuples.py} (74%) rename examples/{13_standard_classes.py => 04_additional/04_classes.py} (71%) rename examples/{14_generics.py => 04_additional/05_generics.py} (86%) rename examples/{16_advanced_configuration.py => 04_additional/06_conf.py} (91%) rename examples/{17_flax_modules.py => 04_additional/07_flax.py} (84%) mode change 100644 => 100755 examples/_rename_example.py diff --git a/README.md b/README.md index 037172d21..538428c1b 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,10 @@

- - tyro logo - + + tyro logo + +

Documentation diff --git a/docs/source/building_configuration_systems.md b/docs/source/building_configuration_systems.md index 1e6941018..6c0cfc441 100644 --- a/docs/source/building_configuration_systems.md +++ b/docs/source/building_configuration_systems.md @@ -15,5 +15,4 @@ have multiple default configuration objects and need to select one to pass in as a default -- this might be a YAML file or config instance -- an environment variable or manually parsed (via `sys.argv`) positional argument can be used. -For exposing each of these base configurations as subcommands, see our -[base config example](/examples/10_base_configs). +For concrete examples, see our "Config Management" documentation. diff --git a/docs/source/examples/01_functions.rst b/docs/source/examples/01_basics/01_functions.rst similarity index 64% rename from docs/source/examples/01_functions.rst rename to docs/source/examples/01_basics/01_functions.rst index 3ac1a2e67..edf75b904 100644 --- a/docs/source/examples/01_functions.rst +++ b/docs/source/examples/01_basics/01_functions.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -1. Functions +Functions ========================================== @@ -37,22 +37,22 @@ populated from the CLI. .. raw:: html - python 01_functions.py --help + python 01_basics/01_functions.py --help -.. program-output:: python ../../examples/01_functions.py --help +.. program-output:: python ../../examples/01_basics/01_functions.py --help ------------ .. raw:: html - python 01_functions.py --field1 hello + python 01_basics/01_functions.py --field1 hello -.. program-output:: python ../../examples/01_functions.py --field1 hello +.. program-output:: python ../../examples/01_basics/01_functions.py --field1 hello ------------ .. raw:: html - python 01_functions.py --field1 hello --field2 10 + python 01_basics/01_functions.py --field1 hello --field2 10 -.. program-output:: python ../../examples/01_functions.py --field1 hello --field2 10 +.. program-output:: python ../../examples/01_basics/01_functions.py --field1 hello --field2 10 diff --git a/docs/source/examples/02_dataclasses.rst b/docs/source/examples/01_basics/02_dataclasses.rst similarity index 53% rename from docs/source/examples/02_dataclasses.rst rename to docs/source/examples/01_basics/02_dataclasses.rst index ded2a0523..79b64aa35 100644 --- a/docs/source/examples/02_dataclasses.rst +++ b/docs/source/examples/01_basics/02_dataclasses.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -2. Dataclasses +Dataclasses ========================================== @@ -24,8 +24,11 @@ can be used as a typed alternative for an argparse namespace. """Description. This should show up in the helptext!""" - field1: str # A string field. - field2: int = 3 # A numeric field, with a default value. + field1: str + """A string field.""" + + field2: int = 3 + """A numeric field, with a default value.""" if __name__ == "__main__": @@ -36,22 +39,22 @@ can be used as a typed alternative for an argparse namespace. .. raw:: html - python 02_dataclasses.py --help + python 01_basics/02_dataclasses.py --help -.. program-output:: python ../../examples/02_dataclasses.py --help +.. program-output:: python ../../examples/01_basics/02_dataclasses.py --help ------------ .. raw:: html - python 02_dataclasses.py --field1 hello + python 01_basics/02_dataclasses.py --field1 hello -.. program-output:: python ../../examples/02_dataclasses.py --field1 hello +.. program-output:: python ../../examples/01_basics/02_dataclasses.py --field1 hello ------------ .. raw:: html - python 02_dataclasses.py --field1 hello --field2 5 + python 01_basics/02_dataclasses.py --field1 hello --field2 5 -.. program-output:: python ../../examples/02_dataclasses.py --field1 hello --field2 5 +.. program-output:: python ../../examples/01_basics/02_dataclasses.py --field1 hello --field2 5 diff --git a/docs/source/examples/03_enums_and_containers.rst b/docs/source/examples/01_basics/03_containers.rst similarity index 57% rename from docs/source/examples/03_enums_and_containers.rst rename to docs/source/examples/01_basics/03_containers.rst index 1a4b56c6d..1e618c8a6 100644 --- a/docs/source/examples/03_enums_and_containers.rst +++ b/docs/source/examples/01_basics/03_containers.rst @@ -1,12 +1,13 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -3. Enums And Containers +Containers ========================================== -We can generate argument parsers from more advanced type annotations, like enums and -container types. +Arguments of both fixed and variable lengths can be annotated with standard Python +container types; such as ``typing.List[T]`` or ``typing.Tuple[T1, T2]``. In Python 3.9, +``list[T]`` and ``tuple[T]`` are also directly supported. @@ -38,14 +39,6 @@ container types. image_dimensions: Tuple[int, int] = (32, 32) """Height and width of some image data.""" - # Enums are handled seamlessly. - optimizer_type: OptimizerType = OptimizerType.ADAM - """Gradient-based optimizer to use.""" - - # We can also explicitly mark arguments as optional. - checkpoint_interval: Optional[int] = None - """Interval to save checkpoints at.""" - if __name__ == "__main__": config = tyro.cli(TrainConfig) @@ -55,22 +48,22 @@ container types. .. raw:: html - python 03_enums_and_containers.py --help + python 01_basics/03_containers.py --help -.. program-output:: python ../../examples/03_enums_and_containers.py --help +.. program-output:: python ../../examples/01_basics/03_containers.py --help ------------ .. raw:: html - python 03_enums_and_containers.py --dataset-sources ./data --image-dimensions 16 16 + python 01_basics/03_containers.py --dataset-sources ./data --image-dimensions 16 16 -.. program-output:: python ../../examples/03_enums_and_containers.py --dataset-sources ./data --image-dimensions 16 16 +.. program-output:: python ../../examples/01_basics/03_containers.py --dataset-sources ./data --image-dimensions 16 16 ------------ .. raw:: html - python 03_enums_and_containers.py --dataset-sources ./data --optimizer-type SGD + python 01_basics/03_containers.py --dataset-sources ./data -.. program-output:: python ../../examples/03_enums_and_containers.py --dataset-sources ./data --optimizer-type SGD +.. program-output:: python ../../examples/01_basics/03_containers.py --dataset-sources ./data diff --git a/docs/source/examples/01_basics/04_enums.rst b/docs/source/examples/01_basics/04_enums.rst new file mode 100644 index 000000000..b1f8cb19b --- /dev/null +++ b/docs/source/examples/01_basics/04_enums.rst @@ -0,0 +1,65 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Enums +========================================== + + +We can generate argument parsers from more advanced type annotations, like enums. + + + +.. code-block:: python + :linenos: + + + import dataclasses + import enum + import pathlib + from typing import Optional, Tuple + + import tyro + + + class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + + @dataclasses.dataclass(frozen=True) + class TrainConfig: + # Enums are handled seamlessly. + optimizer_type: OptimizerType = OptimizerType.ADAM + """Gradient-based optimizer to use.""" + + learning_rate: float = 1e-4 + """Learning rate for optimizer.""" + + + if __name__ == "__main__": + config = tyro.cli(TrainConfig) + print(config) + +------------ + +.. raw:: html + + python 01_basics/04_enums.py --help + +.. program-output:: python ../../examples/01_basics/04_enums.py --help + +------------ + +.. raw:: html + + python 01_basics/04_enums.py --optimizer-type SGD + +.. program-output:: python ../../examples/01_basics/04_enums.py --optimizer-type SGD + +------------ + +.. raw:: html + + python 01_basics/04_enums.py --optimizer-type ADAM --learning-rate 3e-4 + +.. program-output:: python ../../examples/01_basics/04_enums.py --optimizer-type ADAM --learning-rate 3e-4 diff --git a/docs/source/examples/04_flags.rst b/docs/source/examples/01_basics/05_flags.rst similarity index 65% rename from docs/source/examples/04_flags.rst rename to docs/source/examples/01_basics/05_flags.rst index 2dc23d0bb..66c807af9 100644 --- a/docs/source/examples/04_flags.rst +++ b/docs/source/examples/01_basics/05_flags.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -4. Flags +Booleans and Flags ========================================== @@ -45,30 +45,30 @@ To turn off conversion, see :class:`tyro.conf.FlagConversionOff`. .. raw:: html - python 04_flags.py --help + python 01_basics/05_flags.py --help -.. program-output:: python ../../examples/04_flags.py --help +.. program-output:: python ../../examples/01_basics/05_flags.py --help ------------ .. raw:: html - python 04_flags.py --boolean True + python 01_basics/05_flags.py --boolean True -.. program-output:: python ../../examples/04_flags.py --boolean True +.. program-output:: python ../../examples/01_basics/05_flags.py --boolean True ------------ .. raw:: html - python 04_flags.py --boolean False --flag-a + python 01_basics/05_flags.py --boolean False --flag-a -.. program-output:: python ../../examples/04_flags.py --boolean False --flag-a +.. program-output:: python ../../examples/01_basics/05_flags.py --boolean False --flag-a ------------ .. raw:: html - python 04_flags.py --boolean False --no-flag-b + python 01_basics/05_flags.py --boolean False --no-flag-b -.. program-output:: python ../../examples/04_flags.py --boolean False --no-flag-b +.. program-output:: python ../../examples/01_basics/05_flags.py --boolean False --no-flag-b diff --git a/docs/source/examples/01_basics/06_literals.rst b/docs/source/examples/01_basics/06_literals.rst new file mode 100644 index 000000000..91ea0a901 --- /dev/null +++ b/docs/source/examples/01_basics/06_literals.rst @@ -0,0 +1,49 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Choices +========================================== + + +``typing.Literal[]`` can be used to restrict inputs to a fixed set of literal choices. + + + +.. code-block:: python + :linenos: + + + import dataclasses + import enum + from typing import Literal, Optional, Tuple, Union + + import tyro + + + class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + + @dataclasses.dataclass(frozen=True) + class Args: + # We can use Literal[] to restrict the set of allowable inputs, for example, over + # enums. + restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED + + # Or mix them with other types! + mixed: Literal[Color.RED, Color.GREEN, "blue"] = "blue" + + + if __name__ == "__main__": + args = tyro.cli(Args) + print(args) + +------------ + +.. raw:: html + + python 01_basics/06_literals.py --help + +.. program-output:: python ../../examples/01_basics/06_literals.py --help diff --git a/docs/source/examples/06_literals_and_unions.rst b/docs/source/examples/01_basics/07_unions.rst similarity index 65% rename from docs/source/examples/06_literals_and_unions.rst rename to docs/source/examples/01_basics/07_unions.rst index 9031a1c7e..67438832e 100644 --- a/docs/source/examples/06_literals_and_unions.rst +++ b/docs/source/examples/01_basics/07_unions.rst @@ -1,12 +1,11 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -6. Literals And Unions +Unions ========================================== -``typing.Literal[]`` can be used to restrict inputs to a fixed set of literal choices; -``typing.Union[]`` can be used to restrict inputs to a fixed set of types. +``typing.Union[]`` can be used to expand inputs to multiple types. @@ -29,16 +28,6 @@ @dataclasses.dataclass(frozen=True) class Args: - # We can use Literal[] to restrict the set of allowable inputs, for example, over - # enums. - restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED - - # Or mix them with other types! - mixed: Literal[Color.RED, Color.GREEN, "blue"] = "blue" - - # Literals can also be marked Optional. - integer: Optional[Literal[0, 1, 2, 3]] = None - # Unions can be used to specify multiple allowable types. union_over_types: Union[int, str] = 0 string_or_enum: Union[Literal["red", "green"], Color] = "red" @@ -52,6 +41,9 @@ Color.RED, ) + # Optional[T] is equivalent to Union[T, None]. + integer: Optional[Literal[0, 1, 2, 3]] = None + if __name__ == "__main__": args = tyro.cli(Args) @@ -61,6 +53,6 @@ .. raw:: html - python 06_literals_and_unions.py --help + python 01_basics/07_unions.py --help -.. program-output:: python ../../examples/06_literals_and_unions.py --help +.. program-output:: python ../../examples/01_basics/07_unions.py --help diff --git a/docs/source/examples/05_hierarchical_configs.rst b/docs/source/examples/02_nesting/01_nesting.rst similarity index 79% rename from docs/source/examples/05_hierarchical_configs.rst rename to docs/source/examples/02_nesting/01_nesting.rst index 775fe458f..e5daf3a97 100644 --- a/docs/source/examples/05_hierarchical_configs.rst +++ b/docs/source/examples/02_nesting/01_nesting.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -5. Hierarchical Configs +Hierarchical Configs ========================================== @@ -82,22 +82,22 @@ objects. This helps with modularity and grouping in larger projects. .. raw:: html - python 05_hierarchical_configs.py --help + python 02_nesting/01_nesting.py --help -.. program-output:: python ../../examples/05_hierarchical_configs.py --help +.. program-output:: python ../../examples/02_nesting/01_nesting.py --help ------------ .. raw:: html - python 05_hierarchical_configs.py --out-dir . --config.optimizer.algorithm SGD + python 02_nesting/01_nesting.py --out-dir . --config.optimizer.algorithm SGD -.. program-output:: python ../../examples/05_hierarchical_configs.py --out-dir . --config.optimizer.algorithm SGD +.. program-output:: python ../../examples/02_nesting/01_nesting.py --out-dir . --config.optimizer.algorithm SGD ------------ .. raw:: html - python 05_hierarchical_configs.py --out-dir . --restore-checkpoint + python 02_nesting/01_nesting.py --out-dir . --restore-checkpoint -.. program-output:: python ../../examples/05_hierarchical_configs.py --out-dir . --restore-checkpoint +.. program-output:: python ../../examples/02_nesting/01_nesting.py --out-dir . --restore-checkpoint diff --git a/docs/source/examples/08_subcommands.rst b/docs/source/examples/02_nesting/02_subcommands.rst similarity index 60% rename from docs/source/examples/08_subcommands.rst rename to docs/source/examples/02_nesting/02_subcommands.rst index 77bc6a108..38b434363 100644 --- a/docs/source/examples/08_subcommands.rst +++ b/docs/source/examples/02_nesting/02_subcommands.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -8. Subcommands +Subcommands ========================================== @@ -53,38 +53,38 @@ For configuring subcommands beyond what can be expressed with type annotations, .. raw:: html - python 08_subcommands.py --help + python 02_nesting/02_subcommands.py --help -.. program-output:: python ../../examples/08_subcommands.py --help +.. program-output:: python ../../examples/02_nesting/02_subcommands.py --help ------------ .. raw:: html - python 08_subcommands.py cmd:commit --help + python 02_nesting/02_subcommands.py cmd:commit --help -.. program-output:: python ../../examples/08_subcommands.py cmd:commit --help +.. program-output:: python ../../examples/02_nesting/02_subcommands.py cmd:commit --help ------------ .. raw:: html - python 08_subcommands.py cmd:commit --cmd.message hello --cmd.all + python 02_nesting/02_subcommands.py cmd:commit --cmd.message hello --cmd.all -.. program-output:: python ../../examples/08_subcommands.py cmd:commit --cmd.message hello --cmd.all +.. program-output:: python ../../examples/02_nesting/02_subcommands.py cmd:commit --cmd.message hello --cmd.all ------------ .. raw:: html - python 08_subcommands.py cmd:checkout --help + python 02_nesting/02_subcommands.py cmd:checkout --help -.. program-output:: python ../../examples/08_subcommands.py cmd:checkout --help +.. program-output:: python ../../examples/02_nesting/02_subcommands.py cmd:checkout --help ------------ .. raw:: html - python 08_subcommands.py cmd:checkout --cmd.branch main + python 02_nesting/02_subcommands.py cmd:checkout --cmd.branch main -.. program-output:: python ../../examples/08_subcommands.py cmd:checkout --cmd.branch main +.. program-output:: python ../../examples/02_nesting/02_subcommands.py cmd:checkout --cmd.branch main diff --git a/docs/source/examples/09_multiple_subcommands.rst b/docs/source/examples/02_nesting/03_multiple_subcommands.rst similarity index 68% rename from docs/source/examples/09_multiple_subcommands.rst rename to docs/source/examples/02_nesting/03_multiple_subcommands.rst index 41e4d416c..85590a40f 100644 --- a/docs/source/examples/09_multiple_subcommands.rst +++ b/docs/source/examples/02_nesting/03_multiple_subcommands.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -9. Multiple Subcommands +Sequenced Subcommands ========================================== @@ -76,30 +76,30 @@ Multiple unions over nested types are populated using a series of subcommands. .. raw:: html - python 09_multiple_subcommands.py + python 02_nesting/03_multiple_subcommands.py -.. program-output:: python ../../examples/09_multiple_subcommands.py +.. program-output:: python ../../examples/02_nesting/03_multiple_subcommands.py ------------ .. raw:: html - python 09_multiple_subcommands.py --help + python 02_nesting/03_multiple_subcommands.py --help -.. program-output:: python ../../examples/09_multiple_subcommands.py --help +.. program-output:: python ../../examples/02_nesting/03_multiple_subcommands.py --help ------------ .. raw:: html - python 09_multiple_subcommands.py dataset:mnist --help + python 02_nesting/03_multiple_subcommands.py dataset:mnist --help -.. program-output:: python ../../examples/09_multiple_subcommands.py dataset:mnist --help +.. program-output:: python ../../examples/02_nesting/03_multiple_subcommands.py dataset:mnist --help ------------ .. raw:: html - python 09_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 + python 02_nesting/03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 -.. program-output:: python ../../examples/09_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 +.. program-output:: python ../../examples/02_nesting/03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 diff --git a/docs/source/examples/15_nesting_in_containers.rst b/docs/source/examples/02_nesting/04_nesting_in_containers.rst similarity index 90% rename from docs/source/examples/15_nesting_in_containers.rst rename to docs/source/examples/02_nesting/04_nesting_in_containers.rst index 13c2a2e09..da896fb95 100644 --- a/docs/source/examples/15_nesting_in_containers.rst +++ b/docs/source/examples/02_nesting/04_nesting_in_containers.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -15. Nesting In Containers +Nesting in Containers ========================================== @@ -70,6 +70,6 @@ parsing default values. .. raw:: html - python 15_nesting_in_containers.py --help + python 02_nesting/04_nesting_in_containers.py --help -.. program-output:: python ../../examples/15_nesting_in_containers.py --help +.. program-output:: python ../../examples/02_nesting/04_nesting_in_containers.py --help diff --git a/docs/source/examples/10_base_configs.rst b/docs/source/examples/03_config_systems/01_base_configs.rst similarity index 83% rename from docs/source/examples/10_base_configs.rst rename to docs/source/examples/03_config_systems/01_base_configs.rst index d1b905bf7..724641f40 100644 --- a/docs/source/examples/10_base_configs.rst +++ b/docs/source/examples/03_config_systems/01_base_configs.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -10. Base Configs +Base Configurations ========================================== @@ -132,38 +132,38 @@ avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. .. raw:: html - python 10_base_configs.py --help + python 03_config_systems/01_base_configs.py --help -.. program-output:: python ../../examples/10_base_configs.py --help +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py --help ------------ .. raw:: html - python 10_base_configs.py small --help + python 03_config_systems/01_base_configs.py small --help -.. program-output:: python ../../examples/10_base_configs.py small --help +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py small --help ------------ .. raw:: html - python 10_base_configs.py small --seed 94720 + python 03_config_systems/01_base_configs.py small --seed 94720 -.. program-output:: python ../../examples/10_base_configs.py small --seed 94720 +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py small --seed 94720 ------------ .. raw:: html - python 10_base_configs.py big --help + python 03_config_systems/01_base_configs.py big --help -.. program-output:: python ../../examples/10_base_configs.py big --help +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py big --help ------------ .. raw:: html - python 10_base_configs.py big --seed 94720 + python 03_config_systems/01_base_configs.py big --seed 94720 -.. program-output:: python ../../examples/10_base_configs.py big --seed 94720 +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py big --seed 94720 diff --git a/docs/source/examples/03_config_systems/02_overriding_yaml.rst b/docs/source/examples/03_config_systems/02_overriding_yaml.rst new file mode 100644 index 000000000..035b32910 --- /dev/null +++ b/docs/source/examples/03_config_systems/02_overriding_yaml.rst @@ -0,0 +1,62 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Overriding YAML Configs +========================================== + + +If you have a library of existing YAML files that you want to use, ``tyro`` can be used to +override values in them. + + + +.. code-block:: python + :linenos: + + + import yaml + + import tyro + + # Our YAML configuration. Note that this could also be loaded from a file! + # Environment variables are an easy way to select between different YAML files. + default_yaml = r""" + exp_name: test + optimizer: + learning_rate: 0.0001 + type: adam + training: + batch_size: 32 + num_steps: 10000 + checkpoint_steps: + - 500 + - 1000 + - 1500 + """.strip() + + if __name__ == "__main__": + # Convert out YAML config into a nested dictionary. + default_config = yaml.safe_load(default_yaml) + + # Override fields in the dictionary. + overridden_config = tyro.cli(dict, default=default_config) + + # Print the overridden config. + overridden_yaml = yaml.safe_dump(overridden_config) + print(overridden_yaml) + +------------ + +.. raw:: html + + python 03_config_systems/02_overriding_yaml.py --help + +.. program-output:: python ../../examples/03_config_systems/02_overriding_yaml.py --help + +------------ + +.. raw:: html + + python 03_config_systems/02_overriding_yaml.py --training.checkpoint-steps 300 1000 9000 + +.. program-output:: python ../../examples/03_config_systems/02_overriding_yaml.py --training.checkpoint-steps 300 1000 9000 diff --git a/docs/source/examples/07_positional_args.rst b/docs/source/examples/04_additional/01_positional_args.rst similarity index 83% rename from docs/source/examples/07_positional_args.rst rename to docs/source/examples/04_additional/01_positional_args.rst index b618756dd..ba9059bd4 100644 --- a/docs/source/examples/07_positional_args.rst +++ b/docs/source/examples/04_additional/01_positional_args.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -7. Positional Args +Positional Arguments ========================================== @@ -72,14 +72,14 @@ For more general positional arguments, see :func:`tyro.conf.Positional`. .. raw:: html - python 07_positional_args.py --help + python 04_additional/01_positional_args.py --help -.. program-output:: python ../../examples/07_positional_args.py --help +.. program-output:: python ../../examples/04_additional/01_positional_args.py --help ------------ .. raw:: html - python 07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 + python 04_additional/01_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 -.. program-output:: python ../../examples/07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 +.. program-output:: python ../../examples/04_additional/01_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 diff --git a/docs/source/examples/11_dictionaries.rst b/docs/source/examples/04_additional/02_dictionaries.rst similarity index 71% rename from docs/source/examples/11_dictionaries.rst rename to docs/source/examples/04_additional/02_dictionaries.rst index d063b2b5c..2c52b34ac 100644 --- a/docs/source/examples/11_dictionaries.rst +++ b/docs/source/examples/04_additional/02_dictionaries.rst @@ -1,12 +1,12 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -11. Dictionaries +Overriding Dictionaries ========================================== -Dictionary inputs can be specified using either a standard ``Dict[K, V]`` annotation, -or a ``TypedDict`` subclass. +Dictionary inputs can be specified using either a standard ``Dict[K, V]`` annotation, or a +``TypedDict`` subclass. @@ -59,22 +59,22 @@ or a ``TypedDict`` subclass. .. raw:: html - python 11_dictionaries.py --help + python 04_additional/02_dictionaries.py --help -.. program-output:: python ../../examples/11_dictionaries.py --help +.. program-output:: python ../../examples/04_additional/02_dictionaries.py --help ------------ .. raw:: html - python 11_dictionaries.py --typed-dict.learning-rate 3e-4 + python 04_additional/02_dictionaries.py --typed-dict.learning-rate 3e-4 -.. program-output:: python ../../examples/11_dictionaries.py --typed-dict.learning-rate 3e-4 +.. program-output:: python ../../examples/04_additional/02_dictionaries.py --typed-dict.learning-rate 3e-4 ------------ .. raw:: html - python 11_dictionaries.py --typed-dict.betas 0.9 0.999 + python 04_additional/02_dictionaries.py --typed-dict.betas 0.9 0.999 -.. program-output:: python ../../examples/11_dictionaries.py --typed-dict.betas 0.9 0.999 +.. program-output:: python ../../examples/04_additional/02_dictionaries.py --typed-dict.betas 0.9 0.999 diff --git a/docs/source/examples/12_tuples.rst b/docs/source/examples/04_additional/03_tuples.rst similarity index 66% rename from docs/source/examples/12_tuples.rst rename to docs/source/examples/04_additional/03_tuples.rst index 719e699f1..2cfc03f8a 100644 --- a/docs/source/examples/12_tuples.rst +++ b/docs/source/examples/04_additional/03_tuples.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -12. Tuples +Tuples ========================================== @@ -45,22 +45,22 @@ Example using ``tyro.cli()`` to instantiate tuple types. .. raw:: html - python 12_tuples.py --help + python 04_additional/03_tuples.py --help -.. program-output:: python ../../examples/12_tuples.py --help +.. program-output:: python ../../examples/04_additional/03_tuples.py --help ------------ .. raw:: html - python 12_tuples.py --color 127 127 127 + python 04_additional/03_tuples.py --color 127 127 127 -.. program-output:: python ../../examples/12_tuples.py --color 127 127 127 +.. program-output:: python ../../examples/04_additional/03_tuples.py --color 127 127 127 ------------ .. raw:: html - python 12_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0 + python 04_additional/03_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0 -.. program-output:: python ../../examples/12_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0 +.. program-output:: python ../../examples/04_additional/03_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0 diff --git a/docs/source/examples/13_standard_classes.rst b/docs/source/examples/04_additional/04_classes.rst similarity index 73% rename from docs/source/examples/13_standard_classes.rst rename to docs/source/examples/04_additional/04_classes.rst index d0e350c39..526ae71b1 100644 --- a/docs/source/examples/13_standard_classes.rst +++ b/docs/source/examples/04_additional/04_classes.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -13. Standard Classes +Instantiating Classes ========================================== @@ -42,14 +42,14 @@ constructors of) standard Python classes. .. raw:: html - python 13_standard_classes.py --help + python 04_additional/04_classes.py --help -.. program-output:: python ../../examples/13_standard_classes.py --help +.. program-output:: python ../../examples/04_additional/04_classes.py --help ------------ .. raw:: html - python 13_standard_classes.py --field1 hello --field2 7 + python 04_additional/04_classes.py --field1 hello --field2 7 -.. program-output:: python ../../examples/13_standard_classes.py --field1 hello --field2 7 +.. program-output:: python ../../examples/04_additional/04_classes.py --field1 hello --field2 7 diff --git a/docs/source/examples/14_generics.rst b/docs/source/examples/04_additional/05_generics.rst similarity index 87% rename from docs/source/examples/14_generics.rst rename to docs/source/examples/04_additional/05_generics.rst index 627b4a656..175dddef6 100644 --- a/docs/source/examples/14_generics.rst +++ b/docs/source/examples/04_additional/05_generics.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -14. Generics +Generic Types ========================================== @@ -50,6 +50,6 @@ Example of parsing for generic dataclasses. .. raw:: html - python 14_generics.py --help + python 04_additional/05_generics.py --help -.. program-output:: python ../../examples/14_generics.py --help +.. program-output:: python ../../examples/04_additional/05_generics.py --help diff --git a/docs/source/examples/16_advanced_configuration.rst b/docs/source/examples/04_additional/06_conf.rst similarity index 92% rename from docs/source/examples/16_advanced_configuration.rst rename to docs/source/examples/04_additional/06_conf.rst index f5ce74d22..1e99a8a11 100644 --- a/docs/source/examples/16_advanced_configuration.rst +++ b/docs/source/examples/04_additional/06_conf.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -16. Advanced Configuration +Configuration via typing.Annotated[] ========================================== @@ -73,6 +73,6 @@ Features here are supported, but generally unnecessary and should be used sparin .. raw:: html - python 16_advanced_configuration.py --help + python 04_additional/06_conf.py --help -.. program-output:: python ../../examples/16_advanced_configuration.py --help +.. program-output:: python ../../examples/04_additional/06_conf.py --help diff --git a/docs/source/examples/17_flax_modules.rst b/docs/source/examples/04_additional/07_flax.rst similarity index 79% rename from docs/source/examples/17_flax_modules.rst rename to docs/source/examples/04_additional/07_flax.rst index c93ffd3b8..40b3209df 100644 --- a/docs/source/examples/17_flax_modules.rst +++ b/docs/source/examples/04_additional/07_flax.rst @@ -1,11 +1,11 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -17. Flax Modules +Directly Instantiating Flax Modules ========================================== -If you use `Flax `_\ , modules can be instantiated +If you use `flax.linen `_\ , modules can be instantiated directly from ``tyro.cli``. @@ -63,14 +63,14 @@ directly from ``tyro.cli``. .. raw:: html - python 17_flax_modules.py --help + python 04_additional/07_flax.py --help -.. program-output:: python ../../examples/17_flax_modules.py --help +.. program-output:: python ../../examples/04_additional/07_flax.py --help ------------ .. raw:: html - python 17_flax_modules.py --model.layers 4 + python 04_additional/07_flax.py --model.layers 4 -.. program-output:: python ../../examples/17_flax_modules.py --model.layers 4 +.. program-output:: python ../../examples/04_additional/07_flax.py --model.layers 4 diff --git a/docs/source/index.md b/docs/source/index.md index 4505cf013..e1cdd7e47 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -63,8 +63,8 @@ To get started, we recommend browsing the examples to the left. .. toctree:: :caption: Getting started - :maxdepth: 1 :hidden: + :maxdepth: 1 :titlesonly: installation @@ -72,8 +72,8 @@ To get started, we recommend browsing the examples to the left. .. toctree:: :caption: Notes - :maxdepth: 5 :hidden: + :maxdepth: 5 :glob: goals_and_alternatives @@ -82,18 +82,49 @@ To get started, we recommend browsing the examples to the left. building_configuration_systems .. toctree:: - :caption: Examples + :caption: Basics + :hidden: :maxdepth: 1 + :titlesonly: + :glob: + + examples/01_basics/* + + +.. toctree:: + :caption: Hierarchies :hidden: + :maxdepth: 1 :titlesonly: :glob: - examples/* + examples/02_nesting/* + + +.. toctree:: + :caption: Config Management + :hidden: + :maxdepth: 1 + :titlesonly: + :glob: + + examples/03_config_systems/* + + +.. toctree:: + :caption: Additional Features + :hidden: + :maxdepth: 1 + :titlesonly: + :glob: + + examples/04_additional/* + .. toctree:: :caption: API Reference - :maxdepth: 5 :hidden: + :maxdepth: 5 :titlesonly: api/tyro/index diff --git a/docs/update_example_docs.py b/docs/update_example_docs.py index fc40bffee..093280ac3 100644 --- a/docs/update_example_docs.py +++ b/docs/update_example_docs.py @@ -25,20 +25,20 @@ class ExampleMetadata: @staticmethod def from_path(path: pathlib.Path) -> ExampleMetadata: # 01_functions -> 01, _, functions. - index, _, title = path.stem.partition("_") + index, _, _ = path.stem.partition("_") # 01 -> 1. index_with_zero = index index = str(int(index)) - # functions -> Functions. - title = title.replace("_", " ").title() - source = path.read_text().strip() docstring = source.split('"""')[1].strip() assert "Usage:" in docstring - description, _, usage_text = docstring.partition("Usage:") + + title, _, description = docstring.partition("\n") + description, _, usage_text = description.partition("Usage:") + example_usages = map( lambda x: x[1:-1], filter( @@ -52,13 +52,13 @@ def from_path(path: pathlib.Path) -> ExampleMetadata: source=source[3:].partition('"""')[2].strip(), title=title, usages=example_usages, - description=description, + description=description.strip(), ) def get_example_paths(examples_dir: pathlib.Path) -> Iterable[pathlib.Path]: return filter( - lambda p: not p.name.startswith("_"), sorted(examples_dir.glob("*.py")) + lambda p: not p.name.startswith("_"), sorted(examples_dir.glob("**/*.py")) ) @@ -104,17 +104,20 @@ def main( "", ] - ( - example_doc_dir - / f"{ex.index_with_zero}_{ex.title.lower().replace(' ', '_')}.rst" - ).write_text( + relative_dir = path.parent.relative_to(examples_dir) + target_dir = example_doc_dir / relative_dir + target_dir.mkdir(exist_ok=True, parents=True) + + (target_dir / f"{path.stem}.rst").write_text( "\n".join( [ - ".. Comment: this file is automatically generated by" - " `update_example_docs.py`.", + ( + ".. Comment: this file is automatically generated by" + " `update_example_docs.py`." + ), " It should not be modified manually.", "", - f"{ex.index}. {ex.title}", + f"{ex.title}", "==========================================", "", m2r2.convert(ex.description), diff --git a/examples/01_functions.py b/examples/01_basics/01_functions.py similarity index 83% rename from examples/01_functions.py rename to examples/01_basics/01_functions.py index 9d4d7048b..52c7640e8 100755 --- a/examples/01_functions.py +++ b/examples/01_basics/01_functions.py @@ -1,4 +1,6 @@ -"""In the simplest case, `tyro.cli()` can be used to run a function with arguments +"""Functions + +In the simplest case, `tyro.cli()` can be used to run a function with arguments populated from the CLI. Usage: diff --git a/examples/02_dataclasses.py b/examples/01_basics/02_dataclasses.py similarity index 66% rename from examples/02_dataclasses.py rename to examples/01_basics/02_dataclasses.py index 2958fa7ee..30424fead 100644 --- a/examples/02_dataclasses.py +++ b/examples/01_basics/02_dataclasses.py @@ -1,4 +1,6 @@ -"""Common pattern: use `tyro.cli()` to instantiate a dataclass. The outputted instance +"""Dataclasses + +Common pattern: use `tyro.cli()` to instantiate a dataclass. The outputted instance can be used as a typed alternative for an argparse namespace. Usage: @@ -17,8 +19,11 @@ class Args: """Description. This should show up in the helptext!""" - field1: str # A string field. - field2: int = 3 # A numeric field, with a default value. + field1: str + """A string field.""" + + field2: int = 3 + """A numeric field, with a default value.""" if __name__ == "__main__": diff --git a/examples/03_enums_and_containers.py b/examples/01_basics/03_containers.py similarity index 54% rename from examples/03_enums_and_containers.py rename to examples/01_basics/03_containers.py index d7c7a3a5b..ac16e0b2b 100644 --- a/examples/03_enums_and_containers.py +++ b/examples/01_basics/03_containers.py @@ -1,10 +1,13 @@ -"""We can generate argument parsers from more advanced type annotations, like enums and -container types. +"""Containers + +Arguments of both fixed and variable lengths can be annotated with standard Python +container types; such as `typing.List[T]` or `typing.Tuple[T1, T2]`. In Python 3.9, +`list[T]` and `tuple[T]` are also directly supported. Usage: -`python ./03_enums_and_containers.py --help` -`python ./03_enums_and_containers.py --dataset-sources ./data --image-dimensions 16 16` -`python ./03_enums_and_containers.py --dataset-sources ./data --optimizer-type SGD` +`python ./03_containers.py --help` +`python ./03_containers.py --dataset-sources ./data --image-dimensions 16 16` +`python ./03_containers.py --dataset-sources ./data` """ import dataclasses @@ -31,14 +34,6 @@ class TrainConfig: image_dimensions: Tuple[int, int] = (32, 32) """Height and width of some image data.""" - # Enums are handled seamlessly. - optimizer_type: OptimizerType = OptimizerType.ADAM - """Gradient-based optimizer to use.""" - - # We can also explicitly mark arguments as optional. - checkpoint_interval: Optional[int] = None - """Interval to save checkpoints at.""" - if __name__ == "__main__": config = tyro.cli(TrainConfig) diff --git a/examples/01_basics/04_enums.py b/examples/01_basics/04_enums.py new file mode 100644 index 000000000..a1ad0ff90 --- /dev/null +++ b/examples/01_basics/04_enums.py @@ -0,0 +1,36 @@ +"""Enums + +We can generate argument parsers from more advanced type annotations, like enums. + +Usage: +`python ./04_enums.py --help` +`python ./04_enums.py --optimizer-type SGD` +`python ./04_enums.py --optimizer-type ADAM --learning-rate 3e-4` +""" + +import dataclasses +import enum +import pathlib +from typing import Optional, Tuple + +import tyro + + +class OptimizerType(enum.Enum): + ADAM = enum.auto() + SGD = enum.auto() + + +@dataclasses.dataclass(frozen=True) +class TrainConfig: + # Enums are handled seamlessly. + optimizer_type: OptimizerType = OptimizerType.ADAM + """Gradient-based optimizer to use.""" + + learning_rate: float = 1e-4 + """Learning rate for optimizer.""" + + +if __name__ == "__main__": + config = tyro.cli(TrainConfig) + print(config) diff --git a/examples/04_flags.py b/examples/01_basics/05_flags.py similarity index 69% rename from examples/04_flags.py rename to examples/01_basics/05_flags.py index a19244acc..5059ed8b5 100644 --- a/examples/04_flags.py +++ b/examples/01_basics/05_flags.py @@ -1,13 +1,15 @@ -"""Booleans can either be expected to be explicitly passed in, or, if given a default +"""Booleans and Flags + +Booleans can either be expected to be explicitly passed in, or, if given a default value, automatically converted to flags. To turn off conversion, see :class:`tyro.conf.FlagConversionOff`. Usage: -`python ./04_flags.py --help` -`python ./04_flags.py --boolean True` -`python ./04_flags.py --boolean False --flag-a` -`python ./04_flags.py --boolean False --no-flag-b` +`python ./05_flags.py --help` +`python ./05_flags.py --boolean True` +`python ./05_flags.py --boolean False --flag-a` +`python ./05_flags.py --boolean False --no-flag-b` """ import dataclasses diff --git a/examples/01_basics/06_literals.py b/examples/01_basics/06_literals.py new file mode 100644 index 000000000..e9f6a3efe --- /dev/null +++ b/examples/01_basics/06_literals.py @@ -0,0 +1,34 @@ +"""Choices + +`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices. + +Usage: +`python ./06_literals.py --help` +""" + +import dataclasses +import enum +from typing import Literal, Optional, Tuple, Union + +import tyro + + +class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + +@dataclasses.dataclass(frozen=True) +class Args: + # We can use Literal[] to restrict the set of allowable inputs, for example, over + # enums. + restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED + + # Or mix them with other types! + mixed: Literal[Color.RED, Color.GREEN, "blue"] = "blue" + + +if __name__ == "__main__": + args = tyro.cli(Args) + print(args) diff --git a/examples/06_literals_and_unions.py b/examples/01_basics/07_unions.py similarity index 62% rename from examples/06_literals_and_unions.py rename to examples/01_basics/07_unions.py index d3fc32c1a..a94c472b8 100644 --- a/examples/06_literals_and_unions.py +++ b/examples/01_basics/07_unions.py @@ -1,8 +1,9 @@ -"""`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices; -`typing.Union[]` can be used to restrict inputs to a fixed set of types. +"""Unions + +`typing.Union[]` can be used to expand inputs to multiple types. Usage: -`python ./06_literals_and_unions.py --help` +`python ./07_unions.py --help` """ import dataclasses @@ -20,16 +21,6 @@ class Color(enum.Enum): @dataclasses.dataclass(frozen=True) class Args: - # We can use Literal[] to restrict the set of allowable inputs, for example, over - # enums. - restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED - - # Or mix them with other types! - mixed: Literal[Color.RED, Color.GREEN, "blue"] = "blue" - - # Literals can also be marked Optional. - integer: Optional[Literal[0, 1, 2, 3]] = None - # Unions can be used to specify multiple allowable types. union_over_types: Union[int, str] = 0 string_or_enum: Union[Literal["red", "green"], Color] = "red" @@ -43,6 +34,9 @@ class Args: Color.RED, ) + # Optional[T] is equivalent to Union[T, None]. + integer: Optional[Literal[0, 1, 2, 3]] = None + if __name__ == "__main__": args = tyro.cli(Args) diff --git a/examples/05_hierarchical_configs.py b/examples/02_nesting/01_nesting.py similarity index 83% rename from examples/05_hierarchical_configs.py rename to examples/02_nesting/01_nesting.py index 69d4429a5..dd176fec3 100644 --- a/examples/05_hierarchical_configs.py +++ b/examples/02_nesting/01_nesting.py @@ -1,10 +1,12 @@ -"""Structures (typically dataclasses) can be nested to build hierarchical configuration +"""Hierarchical Configs + +Structures (typically dataclasses) can be nested to build hierarchical configuration objects. This helps with modularity and grouping in larger projects. Usage: -`python ./05_hierarchical_configs.py --help` -`python ./05_hierarchical_configs.py --out-dir . --config.optimizer.algorithm SGD` -`python ./05_hierarchical_configs.py --out-dir . --restore-checkpoint` +`python ./01_nesting.py --help` +`python ./01_nesting.py --out-dir . --config.optimizer.algorithm SGD` +`python ./01_nesting.py --out-dir . --restore-checkpoint` """ import dataclasses diff --git a/examples/08_subcommands.py b/examples/02_nesting/02_subcommands.py similarity index 67% rename from examples/08_subcommands.py rename to examples/02_nesting/02_subcommands.py index bf8bb9616..7948b2ab1 100644 --- a/examples/08_subcommands.py +++ b/examples/02_nesting/02_subcommands.py @@ -1,14 +1,16 @@ -"""Unions over nested types (classes or dataclasses) are populated using subcommands. +"""Subcommands + +Unions over nested types (classes or dataclasses) are populated using subcommands. For configuring subcommands beyond what can be expressed with type annotations, see :func:`tyro.conf.subcommand()`. Usage: -`python ./08_subcommands.py --help` -`python ./08_subcommands.py cmd:commit --help` -`python ./08_subcommands.py cmd:commit --cmd.message hello --cmd.all` -`python ./08_subcommands.py cmd:checkout --help` -`python ./08_subcommands.py cmd:checkout --cmd.branch main` +`python ./02_subcommands.py --help` +`python ./02_subcommands.py cmd:commit --help` +`python ./02_subcommands.py cmd:commit --cmd.message hello --cmd.all` +`python ./02_subcommands.py cmd:checkout --help` +`python ./02_subcommands.py cmd:checkout --cmd.branch main` """ from __future__ import annotations diff --git a/examples/09_multiple_subcommands.py b/examples/02_nesting/03_multiple_subcommands.py similarity index 77% rename from examples/09_multiple_subcommands.py rename to examples/02_nesting/03_multiple_subcommands.py index 3b689559f..7c31066a1 100644 --- a/examples/09_multiple_subcommands.py +++ b/examples/02_nesting/03_multiple_subcommands.py @@ -1,10 +1,12 @@ -"""Multiple unions over nested types are populated using a series of subcommands. +"""Sequenced Subcommands + +Multiple unions over nested types are populated using a series of subcommands. Usage: -`python ./09_multiple_subcommands.py` -`python ./09_multiple_subcommands.py --help` -`python ./09_multiple_subcommands.py dataset:mnist --help` -`python ./09_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4` +`python ./03_multiple_subcommands.py` +`python ./03_multiple_subcommands.py --help` +`python ./03_multiple_subcommands.py dataset:mnist --help` +`python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4` """ from __future__ import annotations diff --git a/examples/15_nesting_in_containers.py b/examples/02_nesting/04_nesting_in_containers.py similarity index 89% rename from examples/15_nesting_in_containers.py rename to examples/02_nesting/04_nesting_in_containers.py index d412059eb..5b699d28c 100644 --- a/examples/15_nesting_in_containers.py +++ b/examples/02_nesting/04_nesting_in_containers.py @@ -1,11 +1,13 @@ -"""Structures can be nested inside of standard containers. +"""Nesting in Containers + +Structures can be nested inside of standard containers. Note that lengths must be inferrable, either via a fixed-length tuple annotation or by parsing default values. Usage: -`python ./15_nesting_in_containers.py.py --help` +`python ./04_nesting_in_containers.py.py --help` """ import dataclasses from typing import Dict, Tuple diff --git a/examples/10_base_configs.py b/examples/03_config_systems/01_base_configs.py similarity index 96% rename from examples/10_base_configs.py rename to examples/03_config_systems/01_base_configs.py index 2b2273a09..7323bf9da 100644 --- a/examples/10_base_configs.py +++ b/examples/03_config_systems/01_base_configs.py @@ -1,4 +1,6 @@ -"""We can integrate `tyro.cli()` into common configuration patterns: here, we select +"""Base Configurations + +We can integrate `tyro.cli()` into common configuration patterns: here, we select one of multiple possible base configurations, create a subcommand for each one, and then use the CLI to either override (existing) or fill in (missing) values. diff --git a/examples/03_config_systems/02_overriding_yaml.py b/examples/03_config_systems/02_overriding_yaml.py new file mode 100644 index 000000000..9860fe587 --- /dev/null +++ b/examples/03_config_systems/02_overriding_yaml.py @@ -0,0 +1,40 @@ +"""Overriding YAML Configs + +If you have a library of existing YAML files that you want to use, `tyro` can be used to +override values in them. + +Usage: +`python ./02_overriding_yaml.py --help` +`python ./02_overriding_yaml.py --training.checkpoint-steps 300 1000 9000` +""" + +import yaml + +import tyro + +# Our YAML configuration. Note that this could also be loaded from a file! +# Environment variables are an easy way to select between different YAML files. +default_yaml = r""" +exp_name: test +optimizer: + learning_rate: 0.0001 + type: adam +training: + batch_size: 32 + num_steps: 10000 + checkpoint_steps: + - 500 + - 1000 + - 1500 +""".strip() + +if __name__ == "__main__": + # Convert out YAML config into a nested dictionary. + default_config = yaml.safe_load(default_yaml) + + # Override fields in the dictionary. + overridden_config = tyro.cli(dict, default=default_config) + + # Print the overridden config. + overridden_yaml = yaml.safe_dump(overridden_config) + print(overridden_yaml) diff --git a/examples/07_positional_args.py b/examples/04_additional/01_positional_args.py similarity index 87% rename from examples/07_positional_args.py rename to examples/04_additional/01_positional_args.py index 63902e1f7..132c30d87 100644 --- a/examples/07_positional_args.py +++ b/examples/04_additional/01_positional_args.py @@ -1,10 +1,12 @@ -"""Positional-only arguments in functions are converted to positional CLI arguments. +"""Positional Arguments + +Positional-only arguments in functions are converted to positional CLI arguments. For more general positional arguments, see :func:`tyro.conf.Positional`. Usage: -`python ./07_positional_args.py --help` -`python ./07_positional_args.py ./a ./b --optimizer.learning-rate 1e-5` +`python ./01_positional_args.py --help` +`python ./01_positional_args.py ./a ./b --optimizer.learning-rate 1e-5` """ from __future__ import annotations diff --git a/examples/11_dictionaries.py b/examples/04_additional/02_dictionaries.py similarity index 75% rename from examples/11_dictionaries.py rename to examples/04_additional/02_dictionaries.py index 3534c2a9e..a5894fa5d 100644 --- a/examples/11_dictionaries.py +++ b/examples/04_additional/02_dictionaries.py @@ -1,10 +1,12 @@ -"""Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation, -or a `TypedDict` subclass. +"""Overriding Dictionaries + +Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation, or a +`TypedDict` subclass. Usage: -`python ./11_dictionaries.py --help` -`python ./11_dictionaries.py --typed-dict.learning-rate 3e-4` -`python ./11_dictionaries.py --typed-dict.betas 0.9 0.999` +`python ./02_dictionaries.py --help` +`python ./02_dictionaries.py --typed-dict.learning-rate 3e-4` +`python ./02_dictionaries.py --typed-dict.betas 0.9 0.999` """ from typing import Dict, Mapping, Tuple, TypedDict diff --git a/examples/12_tuples.py b/examples/04_additional/03_tuples.py similarity index 74% rename from examples/12_tuples.py rename to examples/04_additional/03_tuples.py index 214e18303..0d48453b1 100644 --- a/examples/12_tuples.py +++ b/examples/04_additional/03_tuples.py @@ -1,9 +1,11 @@ -"""Example using `tyro.cli()` to instantiate tuple types. +"""Tuples + +Example using `tyro.cli()` to instantiate tuple types. Usage: -`python ./12_tuples.py --help` -`python ./12_tuples.py --color 127 127 127` -`python ./12_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0` +`python ./03_tuples.py --help` +`python ./03_tuples.py --color 127 127 127` +`python ./03_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0` """ from typing import NamedTuple, Tuple diff --git a/examples/13_standard_classes.py b/examples/04_additional/04_classes.py similarity index 71% rename from examples/13_standard_classes.py rename to examples/04_additional/04_classes.py index b276337a9..9780649a7 100644 --- a/examples/13_standard_classes.py +++ b/examples/04_additional/04_classes.py @@ -1,9 +1,11 @@ -"""In addition to functions and dataclasses, we can also generate CLIs from (the +"""Instantiating Classes + +In addition to functions and dataclasses, we can also generate CLIs from (the constructors of) standard Python classes. Usage: -`python ./13_standard_classes.py --help` -`python ./13_standard_classes.py --field1 hello --field2 7` +`python ./04_classes.py --help` +`python ./04_classes.py --field1 hello --field2 7` """ import tyro diff --git a/examples/14_generics.py b/examples/04_additional/05_generics.py similarity index 86% rename from examples/14_generics.py rename to examples/04_additional/05_generics.py index 9e2b342cb..89f96b242 100644 --- a/examples/14_generics.py +++ b/examples/04_additional/05_generics.py @@ -1,7 +1,9 @@ -"""Example of parsing for generic dataclasses. +"""Generic Types + +Example of parsing for generic dataclasses. Usage: -`python ./14_generics.py --help` +`python ./05_generics.py --help` """ import dataclasses diff --git a/examples/16_advanced_configuration.py b/examples/04_additional/06_conf.py similarity index 91% rename from examples/16_advanced_configuration.py rename to examples/04_additional/06_conf.py index 6d76e8272..7d7ce518c 100644 --- a/examples/16_advanced_configuration.py +++ b/examples/04_additional/06_conf.py @@ -1,10 +1,12 @@ -"""The :mod:`tyro.conf` module contains utilities that can be used to configure +"""Configuration via typing.Annotated[] + +The :mod:`tyro.conf` module contains utilities that can be used to configure command-line interfaces beyond what is expressible via static type annotations. Features here are supported, but generally unnecessary and should be used sparingly. Usage: -`python ./16_advanced_configuration.py --help` +`python ./06_conf.py --help` """ import dataclasses diff --git a/examples/17_flax_modules.py b/examples/04_additional/07_flax.py similarity index 84% rename from examples/17_flax_modules.py rename to examples/04_additional/07_flax.py index 4ef9e80a4..323a1a355 100644 --- a/examples/17_flax_modules.py +++ b/examples/04_additional/07_flax.py @@ -1,9 +1,11 @@ -"""If you use [Flax](https://github.com/google/flax), modules can be instantiated +"""Directly Instantiating Flax Modules + +If you use [flax.linen](https://github.com/google/flax), modules can be instantiated directly from `tyro.cli`. Usage: -`python ./17_flax_modules.py --help` -`python ./17_flax_modules.py --model.layers 4` +`python ./07_flax.py --help` +`python ./07_flax.py --model.layers 4` """ from flax import linen as nn diff --git a/examples/_rename_example.py b/examples/_rename_example.py old mode 100644 new mode 100755 index 0c741a514..503831c8d --- a/examples/_rename_example.py +++ b/examples/_rename_example.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + import pathlib import tyro @@ -12,4 +14,5 @@ def main(source: pathlib.Path, dest: pathlib.Path, /) -> None: if __name__ == "__main__": + print("starting") tyro.cli(main) From de7c57b65adfe8475473d21b22a11817d097a82c Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 18 Oct 2022 17:37:00 -0700 Subject: [PATCH 199/491] Test fixes, examples cleanup --- docs/source/examples/01_basics/03_containers.rst | 2 +- docs/source/examples/01_basics/04_enums.rst | 2 -- docs/source/examples/01_basics/06_literals.rst | 2 +- examples/01_basics/03_containers.py | 2 +- examples/01_basics/04_enums.py | 2 -- examples/01_basics/06_literals.py | 2 +- tests/test_dcargs.py | 8 +++----- tyro/_calling.py | 10 ++++++++-- tyro/_resolver.py | 14 ++++++++++++++ 9 files changed, 29 insertions(+), 15 deletions(-) diff --git a/docs/source/examples/01_basics/03_containers.rst b/docs/source/examples/01_basics/03_containers.rst index 1e618c8a6..8db582107 100644 --- a/docs/source/examples/01_basics/03_containers.rst +++ b/docs/source/examples/01_basics/03_containers.rst @@ -18,7 +18,7 @@ container types; such as ``typing.List[T]`` or ``typing.Tuple[T1, T2]``. In Pyth import dataclasses import enum import pathlib - from typing import Optional, Tuple + from typing import Tuple import tyro diff --git a/docs/source/examples/01_basics/04_enums.rst b/docs/source/examples/01_basics/04_enums.rst index b1f8cb19b..7b8296b62 100644 --- a/docs/source/examples/01_basics/04_enums.rst +++ b/docs/source/examples/01_basics/04_enums.rst @@ -15,8 +15,6 @@ We can generate argument parsers from more advanced type annotations, like enums import dataclasses import enum - import pathlib - from typing import Optional, Tuple import tyro diff --git a/docs/source/examples/01_basics/06_literals.rst b/docs/source/examples/01_basics/06_literals.rst index 91ea0a901..22699fc1e 100644 --- a/docs/source/examples/01_basics/06_literals.rst +++ b/docs/source/examples/01_basics/06_literals.rst @@ -15,7 +15,7 @@ Choices import dataclasses import enum - from typing import Literal, Optional, Tuple, Union + from typing import Literal import tyro diff --git a/examples/01_basics/03_containers.py b/examples/01_basics/03_containers.py index ac16e0b2b..ba85c5f34 100644 --- a/examples/01_basics/03_containers.py +++ b/examples/01_basics/03_containers.py @@ -13,7 +13,7 @@ import dataclasses import enum import pathlib -from typing import Optional, Tuple +from typing import Tuple import tyro diff --git a/examples/01_basics/04_enums.py b/examples/01_basics/04_enums.py index a1ad0ff90..d8679d1d7 100644 --- a/examples/01_basics/04_enums.py +++ b/examples/01_basics/04_enums.py @@ -10,8 +10,6 @@ import dataclasses import enum -import pathlib -from typing import Optional, Tuple import tyro diff --git a/examples/01_basics/06_literals.py b/examples/01_basics/06_literals.py index e9f6a3efe..64a2f1958 100644 --- a/examples/01_basics/06_literals.py +++ b/examples/01_basics/06_literals.py @@ -8,7 +8,7 @@ import dataclasses import enum -from typing import Literal, Optional, Tuple, Union +from typing import Literal import tyro diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index c589e5059..82aece9a2 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -491,14 +491,12 @@ def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]: def test_fixed_dataclass_type(): - @dataclasses.dataclass - class Dummy: - pass + dummy = lambda: 5 # noqa - def main(x: Callable = Dummy) -> Callable: + def main(x: Callable = dummy) -> Callable: return x - assert tyro.cli(main, args=[]) is Dummy + assert tyro.cli(main, args=[]) is dummy with pytest.raises(SystemExit): tyro.cli(main, args=["--x", "something"]) diff --git a/tyro/_calling.py b/tyro/_calling.py index 1c979ecae..8bfba395f 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -170,9 +170,15 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: field.name if field.name_override is None else field.name_override ] = value - unwrapped_f = _resolver.unwrap_origin_strip_extras(f) - unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore + # Note: we unwrap types both before and after narrowing. This is because narrowing + # sometimes produces types like `Tuple[T1, T2, ...]`, where we actually want just + # `tuple`. + unwrapped_f = f + unwrapped_f = _resolver.unwrap_origin_strip_extras(unwrapped_f) unwrapped_f = _resolver.narrow_type(unwrapped_f, default_instance) + unwrapped_f = _resolver.unwrap_origin_strip_extras(unwrapped_f) + unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore + if unwrapped_f in (tuple, list, set): if len(args) == 0: # When tuples are used as nested structures (eg Tuple[SomeDataclass]), we diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 3ed9daf16..ec46f1061 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -2,6 +2,7 @@ import collections.abc import copy import dataclasses +import pathlib import sys from typing import ( Any, @@ -118,8 +119,21 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: should parse as Cat. Note that Union types are intentionally excluded here.""" + + # Don't apply narrowing for pathlib.PosixPath, pathlib.WindowsPath, etc. + # This is mostly an aesthetic decision, and is needed because pathlib.Path() hacks + # __new__ to dynamically choose between path types. + if typ is pathlib.Path: + return typ + try: potential_subclass = type(default_instance) + + if potential_subclass is type: + # Don't narrow to `type`. This happens when the default instance is a class; + # it doesn't really make sense to parse this case. + return typ + superclass = unwrap_annotated(typ)[0] if superclass is Any or issubclass(potential_subclass, superclass): # type: ignore if get_origin(typ) is Annotated: From 865e1f6f38e5be9340d57a2397ff1d7dfede4def Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 18 Oct 2022 17:55:58 -0700 Subject: [PATCH 200/491] Strip unsupported HTML tags for PyPI readme --- .github/workflows/publish.yml | 3 +++ README.md | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b98736e41..3ec10a53e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -22,6 +22,9 @@ jobs: run: | curl -sSL https://install.python-poetry.org | python3 - poetry install + - name: Strip unsupported tags in README + run: | + sed -i '//,//d' README.md - name: Build and publish env: PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} diff --git a/README.md b/README.md index 538428c1b..a4eee0abc 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,20 @@

- + + + tyro logo + + +

From 60fb25aac27379589b49f0aca7fb6812e87d12d4 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 18 Oct 2022 18:05:32 -0700 Subject: [PATCH 201/491] Comment nit in publish action --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3ec10a53e..c3d39b48b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,4 @@ -# This workflows will upload a Python Package using Twine when a release is created +# This workflows will upload a Python Package using Poetry when a release is created # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries name: Upload Python Package From 9b83f9555288d92de0acafff9cd9427f192ecdd9 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 18 Oct 2022 18:58:44 -0700 Subject: [PATCH 202/491] Fix type narrowing for Python 3.10 --- pyproject.toml | 2 +- tyro/_parsers.py | 2 +- tyro/_resolver.py | 24 ++++++++++++++---------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a35f7a24b..dc8989ceb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.26" +version = "0.3.27" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] diff --git a/tyro/_parsers.py b/tyro/_parsers.py index ad5aa97ba..62cb0a182 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -83,7 +83,7 @@ def from_callable_or_type( field = dataclasses.replace( field, # Resolve generic types. - typ=_resolver.narrow_type( + typ=_resolver.narrow_container_types( _resolver.type_from_typevar_constraints( # type: ignore _resolver.apply_type_from_typevar( field.typ, diff --git a/tyro/_resolver.py b/tyro/_resolver.py index ec46f1061..4c3c2ea13 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -2,7 +2,6 @@ import collections.abc import copy import dataclasses -import pathlib import sys from typing import ( Any, @@ -118,14 +117,10 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: """Type narrowing: if we annotate as Animal but specify a default instance of Cat, we should parse as Cat. - Note that Union types are intentionally excluded here.""" - - # Don't apply narrowing for pathlib.PosixPath, pathlib.WindowsPath, etc. - # This is mostly an aesthetic decision, and is needed because pathlib.Path() hacks - # __new__ to dynamically choose between path types. - if typ is pathlib.Path: - return typ - + This should generally only be applied to fields used as nested structures, not + individual arguments/fields. (if a field is annotated as Union[int, str], and a + string default is passed in, we don't want to narrow the type to always be + strings!)""" try: potential_subclass = type(default_instance) @@ -135,6 +130,11 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: return typ superclass = unwrap_annotated(typ)[0] + + # For Python 3.10: don't narrow union types. + if get_origin(superclass) is Union: + return typ + if superclass is Any or issubclass(potential_subclass, superclass): # type: ignore if get_origin(typ) is Annotated: return Annotated.__class_getitem__( # type: ignore @@ -144,13 +144,17 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: except TypeError: pass + return typ + + +def narrow_container_types(typ: TypeT, default_instance: Any) -> TypeT: + """Type narrowing for containers. Infers types of container contents.""" if typ is list and isinstance(default_instance, list): typ = List.__getitem__(Union.__getitem__(tuple(map(type, default_instance)))) # type: ignore elif typ is set and isinstance(default_instance, set): typ = Set.__getitem__(Union.__getitem__(tuple(map(type, default_instance)))) # type: ignore elif typ is tuple and isinstance(default_instance, tuple): typ = Tuple.__getitem__(tuple(map(type, default_instance))) # type: ignore - return typ From 12a3e5d78340aee080159bbcdf59e06dd35e82e0 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 28 Oct 2022 05:50:37 +0800 Subject: [PATCH 203/491] Consolidated docs tweaks --- README.md | 10 ++++------ .../examples/03_config_systems/02_overriding_yaml.rst | 6 +++--- .../examples/04_additional/01_positional_args.rst | 2 +- docs/source/index.md | 9 ++++----- examples/03_config_systems/02_overriding_yaml.py | 6 +++--- examples/04_additional/01_positional_args.py | 2 +- tyro/_cli.py | 10 +++++----- tyro/extras/_serialization.py | 4 ++-- 8 files changed, 23 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index a4eee0abc..f070c1587 100644 --- a/README.md +++ b/README.md @@ -37,13 +37,11 @@
-tyro is a library for building CLI interfaces, -configuration objects, and configuration systems with modern, type-annotated -Python. +tyro is a library for building CLI interfaces and +configuration objects with type-annotated Python. -Our core interface consists of just one function, `tyro.cli()`, that translates -Python callables and types into fully-featured argument parsers and -configuration objects. +Our core interface consists of one function, `tyro.cli()`, that generates +argument parsers from Python callables and types. To get started, we recommend visiting the examples in our [documentation](https://brentyi.github.io/tyro). diff --git a/docs/source/examples/03_config_systems/02_overriding_yaml.rst b/docs/source/examples/03_config_systems/02_overriding_yaml.rst index 035b32910..1ce864bd2 100644 --- a/docs/source/examples/03_config_systems/02_overriding_yaml.rst +++ b/docs/source/examples/03_config_systems/02_overriding_yaml.rst @@ -18,8 +18,8 @@ override values in them. import tyro - # Our YAML configuration. Note that this could also be loaded from a file! - # Environment variables are an easy way to select between different YAML files. + # YAML configuration. Note that this could also be loaded from a file! Environment + # variables are an easy way to select between different YAML files. default_yaml = r""" exp_name: test optimizer: @@ -35,7 +35,7 @@ override values in them. """.strip() if __name__ == "__main__": - # Convert out YAML config into a nested dictionary. + # Convert our YAML config into a nested dictionary. default_config = yaml.safe_load(default_yaml) # Override fields in the dictionary. diff --git a/docs/source/examples/04_additional/01_positional_args.rst b/docs/source/examples/04_additional/01_positional_args.rst index ba9059bd4..5c0182a08 100644 --- a/docs/source/examples/04_additional/01_positional_args.rst +++ b/docs/source/examples/04_additional/01_positional_args.rst @@ -7,7 +7,7 @@ Positional Arguments Positional-only arguments in functions are converted to positional CLI arguments. -For more general positional arguments, see :func:`tyro.conf.Positional`. +For more general positional arguments, see :class:`tyro.conf.Positional`. diff --git a/docs/source/index.md b/docs/source/index.md index e1cdd7e47..853a17b74 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -2,12 +2,11 @@ |build| |nbsp| |mypy| |nbsp| |lint| |nbsp| |coverage| |nbsp| |versions| -:code:`tyro` is a library for building CLI interfaces, configuration objects, -and configuration systems with modern, type-annotated Python. +:code:`tyro` is a library for building CLI interfaces and configuration objects +with type-annotated Python. -Our core interface consists of just one function, :func:`tyro.cli()`, that -translates Python callables and types into fully-featured argument parsers and -configuration objects. +Our core interface consists of one function, :func:`tyro.cli()`, that generates +argument parsers from Python callables and types. To get started, we recommend browsing the examples to the left. diff --git a/examples/03_config_systems/02_overriding_yaml.py b/examples/03_config_systems/02_overriding_yaml.py index 9860fe587..a1e83646c 100644 --- a/examples/03_config_systems/02_overriding_yaml.py +++ b/examples/03_config_systems/02_overriding_yaml.py @@ -12,8 +12,8 @@ import tyro -# Our YAML configuration. Note that this could also be loaded from a file! -# Environment variables are an easy way to select between different YAML files. +# YAML configuration. Note that this could also be loaded from a file! Environment +# variables are an easy way to select between different YAML files. default_yaml = r""" exp_name: test optimizer: @@ -29,7 +29,7 @@ """.strip() if __name__ == "__main__": - # Convert out YAML config into a nested dictionary. + # Convert our YAML config into a nested dictionary. default_config = yaml.safe_load(default_yaml) # Override fields in the dictionary. diff --git a/examples/04_additional/01_positional_args.py b/examples/04_additional/01_positional_args.py index 132c30d87..66048cd53 100644 --- a/examples/04_additional/01_positional_args.py +++ b/examples/04_additional/01_positional_args.py @@ -2,7 +2,7 @@ Positional-only arguments in functions are converted to positional CLI arguments. -For more general positional arguments, see :func:`tyro.conf.Positional`. +For more general positional arguments, see :class:`tyro.conf.Positional`. Usage: `python ./01_positional_args.py --help` diff --git a/tyro/_cli.py b/tyro/_cli.py index 0338f0e1c..1b8f20124 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -100,13 +100,13 @@ def cli( `argparse.ArgumentParser()`. args: If set, parse arguments from a sequence of strings instead of the commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. - default: An instance of `T` to use for default values; only supported - if `T` is a dataclass, TypedDict, or NamedTuple. Helpful for merging CLI - arguments with values loaded from elsewhere. (for example, a config object - loaded from a yaml file) + default: An instance of `T` to use for default values; supported for nested + structures like dataclasses and dictionaries, but not if `f` is a function + or standard class. Helpful for merging CLI arguments with values loaded from + elsewhere. (for example, a config object loaded from a yaml file) Returns: - The output of `f(...)`, or an instance `f`. If `f` is a class, the two are + The output of `f(...)` or an instance `f`. If `f` is a class, the two are typically equivalent. """ return cast( diff --git a/tyro/extras/_serialization.py b/tyro/extras/_serialization.py index 9ae536fad..f4befc8f5 100644 --- a/tyro/extras/_serialization.py +++ b/tyro/extras/_serialization.py @@ -177,7 +177,7 @@ def from_yaml( As a secondary feature aimed at enabling the use of :func:`tyro.cli` for general configuration use cases, we also introduce functions for human-readable dataclass - serialization: :func:`tyro.conf.from_yaml` and :func:`tyro.conf.to_yaml` attempt + serialization: :func:`tyro.extras.from_yaml` and :func:`tyro.extras.to_yaml` attempt to strike a balance between flexibility and robustness — in contrast to naively dumping or loading dataclass instances (via pickle, PyYAML, etc), explicit type references enable custom tags that are robust against code reorganization and @@ -206,7 +206,7 @@ def to_yaml(instance: Any) -> str: As a secondary feature aimed at enabling the use of :func:`tyro.cli` for general configuration use cases, we also introduce functions for human-readable dataclass - serialization: :func:`tyro.conf.from_yaml` and :func:`tyro.conf.to_yaml` attempt + serialization: :func:`tyro.extras.from_yaml` and :func:`tyro.extras.to_yaml` attempt to strike a balance between flexibility and robustness — in contrast to naively dumping or loading dataclass instances (via pickle, PyYAML, etc), explicit type references enable custom tags that are robust against code reorganization and From c7f5bdc555d256a03e3a69002c3342c7ee86dfc4 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 30 Oct 2022 02:04:54 +0800 Subject: [PATCH 204/491] Remove outdated example docs --- docs/source/examples/03_config_systems/01_base_configs.rst | 3 +-- examples/03_config_systems/01_base_configs.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/source/examples/03_config_systems/01_base_configs.rst b/docs/source/examples/03_config_systems/01_base_configs.rst index 724641f40..a5ad93898 100644 --- a/docs/source/examples/03_config_systems/01_base_configs.rst +++ b/docs/source/examples/03_config_systems/01_base_configs.rst @@ -12,8 +12,7 @@ use the CLI to either override (existing) or fill in (missing) values. Note that our interfaces don't prescribe any of the mechanics used for storing base configurations. A Hydra-style YAML approach could just as easily be used for the config libary (although we generally prefer to avoid YAMLs; staying in -Python is convenient for autocompletion and type checking). For selection, we could also -avoid fussing with ``sys.argv`` by using a ``BASE_CONFIG`` environment variable. +Python is convenient for autocompletion and type checking). diff --git a/examples/03_config_systems/01_base_configs.py b/examples/03_config_systems/01_base_configs.py index 7323bf9da..4a0830afe 100644 --- a/examples/03_config_systems/01_base_configs.py +++ b/examples/03_config_systems/01_base_configs.py @@ -7,8 +7,7 @@ Note that our interfaces don't prescribe any of the mechanics used for storing base configurations. A Hydra-style YAML approach could just as easily be used for the config libary (although we generally prefer to avoid YAMLs; staying in -Python is convenient for autocompletion and type checking). For selection, we could also -avoid fussing with `sys.argv` by using a `BASE_CONFIG` environment variable. +Python is convenient for autocompletion and type checking). Usage: `python ./10_base_configs.py --help` From 17c68390032154fb1bbd077c61fcc339ccee7626 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 30 Oct 2022 18:25:31 +0800 Subject: [PATCH 205/491] Use upstream version of `shtab` --- poetry.lock | 14 +- pyproject.toml | 6 +- tests/test_print_completion.py | 4 +- tyro/_arguments.py | 5 +- tyro/_cli.py | 14 +- tyro/_shtab/LICENCE | 13 - tyro/_shtab/README.md | 7 - tyro/_shtab/__init__.py | 962 --------------------------------- tyro/_shtab/__main__.py | 10 - tyro/_shtab/_dist_ver.py | 1 - tyro/_shtab/main.py | 77 --- tyro/_shtab/py.typed | 2 - 12 files changed, 29 insertions(+), 1086 deletions(-) delete mode 100644 tyro/_shtab/LICENCE delete mode 100644 tyro/_shtab/README.md delete mode 100644 tyro/_shtab/__init__.py delete mode 100644 tyro/_shtab/__main__.py delete mode 100644 tyro/_shtab/_dist_ver.py delete mode 100644 tyro/_shtab/main.py delete mode 100644 tyro/_shtab/py.typed diff --git a/poetry.lock b/poetry.lock index 2d40220f0..acba0d084 100644 --- a/poetry.lock +++ b/poetry.lock @@ -592,6 +592,14 @@ tomli = ">=1.0.0" test = ["pytest (>=6.2)", "virtualenv (>20)"] toml = ["setuptools (>=42)"] +[[package]] +name = "shtab" +version = "1.5.6" +description = "Automagic shell tab completion for Python CLI applications" +category = "main" +optional = false +python-versions = ">=3.2" + [[package]] name = "six" version = "1.16.0" @@ -658,7 +666,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "efb24b1ca37536bb4af0e0c2d80937cc1880a0cd1cb5af5dab562f6205db9696" +content-hash = "f273885b6e4daaa758c2bd6e777e974aab9c320d63417dd5476c0e4c46211277" [metadata.files] absl-py = [ @@ -1238,6 +1246,10 @@ setuptools-scm = [ {file = "setuptools_scm-6.4.2-py3-none-any.whl", hash = "sha256:acea13255093849de7ccb11af9e1fb8bde7067783450cee9ef7a93139bddf6d4"}, {file = "setuptools_scm-6.4.2.tar.gz", hash = "sha256:6833ac65c6ed9711a4d5d2266f8024cfa07c533a0e55f4c12f6eff280a5a9e30"}, ] +shtab = [ + {file = "shtab-1.5.6-py2.py3-none-any.whl", hash = "sha256:59cc8b291ce43cea24f9c66a633d054967893bd6a55dc8bfcc65849073ee01d0"}, + {file = "shtab-1.5.6.tar.gz", hash = "sha256:0b394e4625cb0632ad3b26a51c16ee35628b707b137108502a71473abb2be4c3"}, +] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, diff --git a/pyproject.toml b/pyproject.toml index dc8989ceb..de342f5e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ PyYAML = "^6.0" colorama = {version = "^0.4.0", platform = "win32"} frozendict = "^2.3.4" rich = ">=11.1.0" +shtab = "^1.5.6" [tool.poetry.dev-dependencies] pytest = "^7.1.2" @@ -45,11 +46,6 @@ python_version = "3.8" ignore_missing_imports = true warn_unused_configs = true -[tool.coverage.run] -omit = [ - "tyro/_shtab/*", -] - [tool.coverage.report] exclude_lines = [ # Have to re-enable the standard pragma diff --git a/tests/test_print_completion.py b/tests/test_print_completion.py index 142db6870..e1d6a9419 100644 --- a/tests/test_print_completion.py +++ b/tests/test_print_completion.py @@ -33,11 +33,11 @@ def test_bash(): target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): tyro.cli(Wrapper, args=["--tyro-print-completion", "bash"]) - assert "# AUTOMATCALLY GENERATED by `shtab`" in target.getvalue() + assert "# AUTOMATICALLY GENERATED by `shtab`" in target.getvalue() def test_zsh(): target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): tyro.cli(Wrapper, args=["--tyro-print-completion", "zsh"]) - assert "# AUTOMATCALLY GENERATED by `shtab`" in target.getvalue() + assert "# AUTOMATICALLY GENERATED by `shtab`" in target.getvalue() diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 41e12f13d..cc48d42d9 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -22,10 +22,9 @@ ) import rich.markup +import shtab -from . import _fields, _instantiators, _resolver -from . import _shtab as shtab -from . import _strings +from . import _fields, _instantiators, _resolver, _strings from .conf import _markers try: diff --git a/tyro/_cli.py b/tyro/_cli.py index 1b8f20124..c4c9a5fca 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -5,9 +5,17 @@ import warnings from typing import Callable, Optional, Sequence, Type, TypeVar, Union, cast, overload -from . import _argparse_formatter, _arguments, _calling, _fields, _parsers -from . import _shtab as shtab -from . import _strings, conf +import shtab + +from . import ( + _argparse_formatter, + _arguments, + _calling, + _fields, + _parsers, + _strings, + conf, +) OutT = TypeVar("OutT") diff --git a/tyro/_shtab/LICENCE b/tyro/_shtab/LICENCE deleted file mode 100644 index aa47a9215..000000000 --- a/tyro/_shtab/LICENCE +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2020-2021 Casper da Costa-Luis - -Licensed under the Apache Licence, Version 2.0 (the "Licence"); -you may not use this project except in compliance with the Licence. -You may obtain a copy of the Licence at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the Licence is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the Licence for the specific language governing permissions and -limitations under the Licence. diff --git a/tyro/_shtab/README.md b/tyro/_shtab/README.md deleted file mode 100644 index 2a7c38f7b..000000000 --- a/tyro/_shtab/README.md +++ /dev/null @@ -1,7 +0,0 @@ -Temporary copy of `shtab` that incorporates two PRs: - -- https://github.com/iterative/shtab/pull/106 -- https://github.com/iterative/shtab/pull/108 -- https://github.com/iterative/shtab/pull/110 - -Can be removed when these are merged. diff --git a/tyro/_shtab/__init__.py b/tyro/_shtab/__init__.py deleted file mode 100644 index 482162e59..000000000 --- a/tyro/_shtab/__init__.py +++ /dev/null @@ -1,962 +0,0 @@ -from __future__ import print_function - -import logging -import re -import sys -from argparse import ( - SUPPRESS, - Action, - ArgumentParser, - _AppendAction, - _AppendConstAction, - _CountAction, - _HelpAction, - _StoreConstAction, - _VersionAction, -) -from collections import defaultdict -from functools import total_ordering -from itertools import starmap -from string import Template -from typing import Any, Dict, List -from typing import Optional as Opt -from typing import Union - -# version detector. Precedence: installed dist, git, 'UNKNOWN' -try: - from ._dist_ver import __version__ -except ImportError: - try: - from setuptools_scm import get_version - - __version__ = get_version(root="..", relative_to=__file__) - except (ImportError, LookupError): - __version__ = "UNKNOWN" -__all__ = [ - "complete", - "add_argument_to", - "SUPPORTED_SHELLS", - "FILE", - "DIRECTORY", - "DIR", -] -log = logging.getLogger(__name__) - -SUPPORTED_SHELLS: List[str] = [] -_SUPPORTED_COMPLETERS = {} -CHOICE_FUNCTIONS: Dict[str, Dict[str, str]] = { - "file": {"bash": "_shtab_compgen_files", "zsh": "_files", "tcsh": "f"}, - "directory": {"bash": "_shtab_compgen_dirs", "zsh": "_files -/", "tcsh": "d"}, -} -FILE = CHOICE_FUNCTIONS["file"] -DIRECTORY = DIR = CHOICE_FUNCTIONS["directory"] -FLAG_OPTION = ( - _StoreConstAction, - _HelpAction, - _VersionAction, - _AppendConstAction, - _CountAction, -) -OPTION_END = _HelpAction, _VersionAction -OPTION_MULTI = _AppendAction, _AppendConstAction, _CountAction -RE_ZSH_SPECIAL_CHARS = re.compile(r"([^\w\s.,()-])") # excessive but safe - - -def mark_completer(shell): - def wrapper(func): - if shell not in SUPPORTED_SHELLS: - SUPPORTED_SHELLS.append(shell) - _SUPPORTED_COMPLETERS[shell] = func - return func - - return wrapper - - -def get_completer(shell): - try: - return _SUPPORTED_COMPLETERS[shell] - except KeyError: - raise NotImplementedError( - "shell (%s) must be in {%s}" % (shell, ",".join(SUPPORTED_SHELLS)) - ) - - -@total_ordering -class Choice(object): - """ - Placeholder to mark a special completion ``. - - >>> ArgumentParser.add_argument(..., choices=[Choice("")]) - """ - - def __init__(self, choice_type: str, required: bool = False) -> None: - """ - See below for parameters. - - choice_type : internal `type` name - required : controls result of comparison to empty strings - """ - self.required = required - self.type = choice_type - - def __repr__(self) -> str: - return self.type + ("" if self.required else "?") - - def __cmp__(self, other: object) -> int: - if self.required: - return 0 if other else -1 - return 0 - - def __eq__(self, other: object) -> bool: - return self.__cmp__(other) == 0 - - def __lt__(self, other: object) -> bool: - return self.__cmp__(other) < 0 - - -class Optional(object): - """Example: `ArgumentParser.add_argument(..., choices=Optional.FILE)`.""" - - FILE = [Choice("file")] - DIR = DIRECTORY = [Choice("directory")] - - -class Required(object): - """Example: `ArgumentParser.add_argument(..., choices=Required.FILE)`.""" - - FILE = [Choice("file", True)] - DIR = DIRECTORY = [Choice("directory", True)] - - -def complete2pattern(opt_complete, shell, choice_type2fn) -> bool: - return ( - opt_complete.get(shell, "") - if isinstance(opt_complete, dict) - else choice_type2fn[opt_complete] - ) - - -def wordify(string: str) -> str: - """Replace non-word chars [-. ] with underscores [_]""" - return re.compile(r"[-.\s:]").sub("_", string) - - -def get_public_subcommands(sub): - """Get all the publicly-visible subcommands for a given subparser.""" - public_parsers = {id(sub.choices[i.dest]) for i in sub._get_subactions()} - return {k for k, v in sub.choices.items() if id(v) in public_parsers} - - -def get_bash_commands(root_parser, root_prefix, choice_functions=None): - """ - Recursive subcommand parser traversal, returning lists of information on - commands (formatted for output to the completions script). - printing bash helper syntax. - - Returns: - subparsers : list of subparsers for each parser - option_strings : list of options strings for each parser - compgens : list of shtab `.complete` functions corresponding to actions - choices : list of choices corresponding to actions - nargs : list of number of args allowed for each action (if not 0 or 1) - """ - choice_type2fn = {k: v["bash"] for k, v in CHOICE_FUNCTIONS.items()} - if choice_functions: - choice_type2fn.update(choice_functions) - - def get_option_strings(parser): - """Flattened list of all `parser`'s option strings.""" - return sum( - ( - opt.option_strings - for opt in parser._get_optional_actions() - if opt.help != SUPPRESS - ), - [], - ) - - def recurse(parser, prefix): - """recurse through subparsers, appending to the return lists""" - subparsers = [] - option_strings = [] - compgens = [] - choices = [] - nargs = [] - - # temp lists for recursion results - sub_subparsers = [] - sub_option_strings = [] - sub_compgens = [] - sub_choices = [] - sub_nargs = [] - - # positional arguments - discovered_subparsers = [] - for i, positional in enumerate(parser._get_positional_actions()): - if positional.help == SUPPRESS: - continue - - if hasattr(positional, "complete"): - # shtab `.complete = ...` functions - compgens.append( - "{}_pos_{}_COMPGEN={}".format( - prefix, - i, - complete2pattern(positional.complete, "bash", choice_type2fn), - ) - ) - - if positional.choices: - # choices (including subparsers & shtab `.complete` functions) - log.debug("choices:{}:{}".format(prefix, sorted(positional.choices))) - - this_positional_choices = [] - for choice in positional.choices: - if isinstance(choice, Choice): - # append special completion type to `compgens` - # NOTE: overrides `.complete` attribute - log.debug( - "Choice.{}:{}:{}".format( - choice.type, prefix, positional.dest - ) - ) - compgens.append( - "{}_pos_{}_COMPGEN={}".format( - prefix, i, choice_type2fn[choice.type] - ) - ) - elif isinstance(positional.choices, dict): - # subparser, so append to list of subparsers & recurse - log.debug("subcommand:%s", choice) - public_cmds = get_public_subcommands(positional) - if choice in public_cmds: - discovered_subparsers.append(str(choice)) - this_positional_choices.append(str(choice)) - ( - new_subparsers, - new_option_strings, - new_compgens, - new_choices, - new_nargs, - ) = recurse( - positional.choices[choice], - prefix + "_" + wordify(choice), - ) - sub_subparsers.extend(new_subparsers) - sub_option_strings.extend(new_option_strings) - sub_compgens.extend(new_compgens) - sub_choices.extend(new_choices) - sub_nargs.extend(new_nargs) - else: - log.debug("skip:subcommand:%s", choice) - else: - # simple choice - this_positional_choices.append(str(choice)) - - if this_positional_choices: - choices.append( - "{}_pos_{}_choices=('{}')".format( - prefix, i, "' '".join(this_positional_choices) - ) - ) - - # skip default `nargs` values - if positional.nargs not in (None, "1", "?"): - nargs.append("{}_pos_{}_nargs={}".format(prefix, i, positional.nargs)) - - if discovered_subparsers: - subparsers.append( - "{}_subparsers=('{}')".format(prefix, "' '".join(discovered_subparsers)) - ) - log.debug("subcommands:{}:{}".format(prefix, discovered_subparsers)) - - # optional arguments - option_strings.append( - "{}_option_strings=('{}')".format( - prefix, "' '".join(get_option_strings(parser)) - ) - ) - for optional in parser._get_optional_actions(): - if optional == SUPPRESS: - continue - - for option_string in optional.option_strings: - if hasattr(optional, "complete"): - # shtab `.complete = ...` functions - compgens.append( - "{}_{}_COMPGEN={}".format( - prefix, - wordify(option_string), - complete2pattern(optional.complete, "bash", choice_type2fn), - ) - ) - - if optional.choices: - # choices (including shtab `.complete` functions) - this_optional_choices = [] - for choice in optional.choices: - # append special completion type to `compgens` - # NOTE: overrides `.complete` attribute - if isinstance(choice, Choice): - log.debug( - "Choice.{}:{}:{}".format( - choice.type, prefix, optional.dest - ) - ) - compgens.append( - "{}_{}_COMPGEN={}".format( - prefix, - wordify(option_string), - choice_type2fn[choice.type], - ) - ) - else: - # simple choice - this_optional_choices.append(str(choice)) - - if this_optional_choices: - choices.append( - "{}_{}_choices=('{}')".format( - prefix, - wordify(option_string), - "' '".join(this_optional_choices), - ) - ) - - # Check for nargs. - if optional.nargs is not None and optional.nargs != 1: - nargs.append( - "{}_{}_nargs={}".format( - prefix, wordify(option_string), optional.nargs - ) - ) - - # append recursion results - subparsers.extend(sub_subparsers) - option_strings.extend(sub_option_strings) - compgens.extend(sub_compgens) - choices.extend(sub_choices) - nargs.extend(sub_nargs) - - return subparsers, option_strings, compgens, choices, nargs - - return recurse(root_parser, root_prefix) - - -@mark_completer("bash") -def complete_bash(parser, root_prefix=None, preamble="", choice_functions=None): - """ - Returns bash syntax autocompletion script. - - See `complete` for arguments. - """ - root_prefix = wordify("_shtab_" + (root_prefix or parser.prog)) - subparsers, option_strings, compgens, choices, nargs = get_bash_commands( - parser, root_prefix, choice_functions=choice_functions - ) - - # References: - # - https://www.gnu.org/software/bash/manual/html_node/ - # Programmable-Completion.html - # - https://opensource.com/article/18/3/creating-bash-completion-script - # - https://stackoverflow.com/questions/12933362 - return Template( - """\ -# AUTOMATCALLY GENERATED by `shtab` - -${subparsers} - -${option_strings} - -${compgens} - -${choices} - -${nargs} - -${preamble} -# $1=COMP_WORDS[1] -_shtab_compgen_files() { - compgen -f -- $1 # files -} - -# $1=COMP_WORDS[1] -_shtab_compgen_dirs() { - compgen -d -- $1 # recurse into subdirs -} - -# $1=COMP_WORDS[1] -_shtab_replace_nonword() { - echo "${1//[^[:word:]]/_}" -} - -# set default values (called for the initial parser & any subparsers) -_set_parser_defaults() { - local subparsers_var="${prefix}_subparsers[@]" - sub_parsers=${!subparsers_var} - - local current_option_strings_var="${prefix}_option_strings[@]" - current_option_strings=${!current_option_strings_var} - - completed_positional_actions=0 - - _set_new_action "pos_${completed_positional_actions}" true -} - -# $1=action identifier -# $2=positional action (bool) -# set all identifiers for an action's parameters -_set_new_action() { - current_action="${prefix}_$(_shtab_replace_nonword $1)" - - local current_action_compgen_var=${current_action}_COMPGEN - current_action_compgen="${!current_action_compgen_var}" - - local current_action_choices_var="${current_action}_choices[@]" - current_action_choices="${!current_action_choices_var}" - - local current_action_nargs_var="${current_action}_nargs" - if [ -n "${!current_action_nargs_var}" ]; then - current_action_nargs="${!current_action_nargs_var}" - else - current_action_nargs=1 - fi - - current_action_args_start_index=$(( $word_index + 1 )) - - current_action_is_positional=$2 -} - -# Notes: -# `COMPREPLY`: what will be rendered after completion is triggered -# `completing_word`: currently typed word to generate completions for -# `${!var}`: evaluates the content of `var` and expand its content as a variable -# hello="world" -# x="hello" -# ${!x} -> ${hello} -> "world" -${root_prefix}() { - local completing_word="${COMP_WORDS[COMP_CWORD]}" - COMPREPLY=() - - prefix=${root_prefix} - word_index=0 - _set_parser_defaults - word_index=1 - - # determine what arguments are appropriate for the current state - # of the arg parser - while [ $word_index -ne $COMP_CWORD ]; do - local this_word="${COMP_WORDS[$word_index]}" - - if [[ -n $sub_parsers && " ${sub_parsers[@]} " =~ " ${this_word} " ]]; then - # valid subcommand: add it to the prefix & reset the current action - prefix="${prefix}_$(_shtab_replace_nonword $this_word)" - _set_parser_defaults - fi - - if [[ " ${current_option_strings[@]} " =~ " ${this_word} " ]]; then - # a new action should be acquired (due to recognised option string or - # no more input expected from current action); - # the next positional action can fill in here - _set_new_action $this_word false - fi - - if [[ "$current_action_nargs" != "*" ]] && \\ - [[ "$current_action_nargs" != "+" ]] && \\ - [[ "$current_action_nargs" != *"..." ]] && \\ - (( $word_index + 1 - $current_action_args_start_index >= \\ - $current_action_nargs )); then - $current_action_is_positional && let "completed_positional_actions += 1" - _set_new_action "pos_${completed_positional_actions}" true - fi - - let "word_index+=1" - done - - # Generate the completions - - if [[ "${completing_word}" == -* ]]; then - # optional argument started: use option strings - COMPREPLY=( $(compgen -W "${current_option_strings[*]}" -- "${completing_word}") ) - else - # use choices & compgen - COMPREPLY=( $(compgen -W "${current_action_choices[*]}" -- "${completing_word}") \\ - $([ -n "${current_action_compgen}" ] \\ - && "${current_action_compgen}" "${completing_word}") ) - fi - - return 0 -} - -complete -o filenames -F ${root_prefix} ${prog}""" - ).safe_substitute( - subparsers="\n".join(subparsers), - option_strings="\n".join(option_strings), - compgens="\n".join(compgens), - choices="\n".join(choices), - nargs="\n".join(nargs), - preamble=( - "\n# Custom Preamble\n" + preamble + "\n# End Custom Preamble\n" - if preamble - else "" - ), - root_prefix=root_prefix, - prog=parser.prog, - ) - - -def escape_zsh(string): - return RE_ZSH_SPECIAL_CHARS.sub(r"\\\1", str(string)) - - -@mark_completer("zsh") -def complete_zsh(parser, root_prefix=None, preamble="", choice_functions=None): - """ - Returns zsh syntax autocompletion script. - - See `complete` for arguments. - """ - prog = parser.prog - root_prefix = wordify("_shtab_" + (root_prefix or prog)) - - choice_type2fn = {k: v["zsh"] for k, v in CHOICE_FUNCTIONS.items()} - if choice_functions: - choice_type2fn.update(choice_functions) - - def format_optional(opt): - return ( - ( - '{nargs}{options}"[{help}]"' - if isinstance(opt, FLAG_OPTION) - else '{nargs}{options}"[{help}]:{dest}:{pattern}"' - ) - .format( - nargs=( - '"(- :)"' - if isinstance(opt, OPTION_END) - else '"*"' - if isinstance(opt, OPTION_MULTI) - else "" - ), - options=( - "{{{}}}".format(",".join(opt.option_strings)) - if len(opt.option_strings) > 1 - else '"{}"'.format("".join(opt.option_strings)) - ), - help=escape_zsh(opt.help or ""), - dest=opt.dest, - pattern=complete2pattern(opt.complete, "zsh", choice_type2fn) - if hasattr(opt, "complete") - else ( - choice_type2fn[opt.choices[0].type] - if isinstance(opt.choices[0], Choice) - else "({})".format(" ".join(map(str, opt.choices))) - ) - if opt.choices - else "", - ) - .replace('""', "") - ) - - def format_positional(opt): - return '"{nargs}:{help}:{pattern}"'.format( - nargs={"+": "(*)", "*": "(*):"}.get(opt.nargs, ""), - help=escape_zsh((opt.help or opt.dest).strip().split("\n")[0]), - pattern=complete2pattern(opt.complete, "zsh", choice_type2fn) - if hasattr(opt, "complete") - else ( - choice_type2fn[opt.choices[0].type] - if isinstance(opt.choices[0], Choice) - else "({})".format(" ".join(map(str, opt.choices))) - ) - if opt.choices - else "", - ) - - # {cmd: {"help": help, "arguments": [arguments]}} - all_commands = { - root_prefix: { - "cmd": prog, - "arguments": [ - format_optional(opt) - for opt in parser._get_optional_actions() - if opt.help != SUPPRESS - ], - "help": (parser.description or "").strip().split("\n")[0], - "commands": [], - "paths": [], - } - } - - def recurse(parser, prefix, paths=None): - paths = paths or [] - subcmds = [] - for sub in parser._get_positional_actions(): - if sub.help == SUPPRESS or not sub.choices: - continue - if not sub.choices or not isinstance(sub.choices, dict): - # positional argument - all_commands[prefix]["arguments"].append(format_positional(sub)) - else: # subparser - log.debug("choices:{}:{}".format(prefix, sorted(sub.choices))) - public_cmds = get_public_subcommands(sub) - for cmd, subparser in sub.choices.items(): - if cmd not in public_cmds: - log.debug("skip:subcommand:%s", cmd) - continue - log.debug("subcommand:%s", cmd) - - # optionals - arguments = [ - format_optional(opt) - for opt in subparser._get_optional_actions() - if opt.help != SUPPRESS - ] - - # positionals - arguments.extend( - format_positional(opt) - for opt in subparser._get_positional_actions() - if not isinstance(opt.choices, dict) - if opt.help != SUPPRESS - ) - - new_pref = prefix + "_" + wordify(cmd) - options = all_commands[new_pref] = { - "cmd": cmd, - "help": (subparser.description or "").strip().split("\n")[0], - "arguments": arguments, - "paths": [*paths, cmd], - } - new_subcmds = recurse(subparser, new_pref, [*paths, cmd]) - options["commands"] = { - all_commands[pref]["cmd"]: all_commands[pref] - for pref in new_subcmds - if pref in all_commands - } - subcmds.extend([*new_subcmds, new_pref]) - log.debug("subcommands:%s:%s", cmd, options) - return subcmds - - recurse(parser, root_prefix) - all_commands[root_prefix]["commands"] = { - options["cmd"]: options - for prefix, options in sorted(all_commands.items()) - if len(options.get("paths", [])) < 2 and prefix != root_prefix - } - subcommands = { - prefix: options - for prefix, options in all_commands.items() - if options.get("commands") - } - subcommands.setdefault(root_prefix, all_commands[root_prefix]) - log.debug("subcommands:%s:%s", root_prefix, sorted(all_commands)) - - def command_case(prefix, options): - name = options["cmd"] - commands = options["commands"] - case_fmt_on_no_sub = ( - """{name}) _arguments -C ${prefix}_{name_wordify}_options ;;""" - ) - case_fmt_on_sub = """{name}) {prefix}_{name_wordify} ;;""" - - cases = [] - for _, options in sorted(commands.items()): - fmt = case_fmt_on_sub if options.get("commands") else case_fmt_on_no_sub - cases.append( - fmt.format( - name=options["cmd"], - name_wordify=wordify(options["cmd"]), - prefix=prefix, - ) - ) - cases = "\n\t".expandtabs(8).join(cases) - - return """\ -{prefix}() {{ - local context state line curcontext="$curcontext" - - _arguments -C ${prefix}_options \\ - ': :{prefix}_commands' \\ - '*::: :->{name}' - - case $state in - {name}) - words=($line[1] "${{words[@]}}") - (( CURRENT += 1 )) - curcontext="${{curcontext%:*:*}}:{prefix}-$line[1]:" - case $line[1] in - {cases} - esac - esac -}} -""".format( - prefix=prefix, name=name, cases=cases - ) - - def command_option(prefix, options): - return """\ -{prefix}_options=( - {arguments} -) -""".format( - prefix=prefix, arguments="\n ".join(options["arguments"]) - ) - - def command_list(prefix, options): - name = " ".join([prog, *options["paths"]]) - commands = "\n ".join( - '"{}:{}"'.format(escape_zsh(cmd), escape_zsh(opt["help"])) - for cmd, opt in sorted(options["commands"].items()) - ) - return """ -{prefix}_commands() {{ - local _commands=( - {commands} - ) - _describe '{name} commands' _commands -}}""".format( - prefix=prefix, name=name, commands=commands - ) - - preamble = ( - """\ -# Custom Preamble -{} - -# End Custom Preamble -""".format( - preamble.rstrip() - ) - if preamble - else "" - ) - # References: - # - https://github.com/zsh-users/zsh-completions - # - http://zsh.sourceforge.net/Doc/Release/Completion-System.html - # - https://mads-hartmann.com/2017/08/06/ - # writing-zsh-completion-scripts.html - # - http://www.linux-mag.com/id/1106/ - return Template( - """\ -#compdef ${prog} - -# AUTOMATCALLY GENERATED by `shtab` - -${command_commands} - -${command_options} - -${command_cases} -${preamble} - -typeset -A opt_args -${root_prefix} "$@\"""" - ).safe_substitute( - prog=prog, - root_prefix=root_prefix, - command_cases="\n".join(starmap(command_case, sorted(subcommands.items()))), - command_commands="\n".join(starmap(command_list, sorted(subcommands.items()))), - command_options="\n".join( - starmap(command_option, sorted(all_commands.items())) - ), - preamble=preamble, - ) - - -@mark_completer("tcsh") -def complete_tcsh(parser, root_prefix=None, preamble="", choice_functions=None): - """ - Return tcsh syntax autocompletion script. - - See `complete` for arguments. - """ - optionals_single = set() - optionals_double = set() - specials = [] - index_choices = defaultdict(dict) - - choice_type2fn = {k: v["tcsh"] for k, v in CHOICE_FUNCTIONS.items()} - if choice_functions: - choice_type2fn.update(choice_functions) - - def get_specials(arg, arg_type, arg_sel): - if arg.choices: - choice_strs = " ".join(map(str, arg.choices)) - yield "'{}/{}/({})/'".format( - arg_type, - arg_sel, - choice_strs, - ) - elif hasattr(arg, "complete"): - complete_fn = complete2pattern(arg.complete, "tcsh", choice_type2fn) - if complete_fn: - yield "'{}/{}/{}/'".format( - arg_type, - arg_sel, - complete_fn, - ) - - def recurse_parser(cparser, positional_idx, requirements=None): - log_prefix = "| " * positional_idx - log.debug("%sParser @ %d", log_prefix, positional_idx) - if requirements: - log.debug("%s- Requires: %s", log_prefix, " ".join(requirements)) - else: - requirements = [] - - for optional in cparser._get_optional_actions(): - log.debug("%s| Optional: %s", log_prefix, optional.dest) - if optional.help != SUPPRESS: - # Mingle all optional arguments for all subparsers - for optional_str in optional.option_strings: - log.debug("%s| | %s", log_prefix, optional_str) - if optional_str.startswith("--"): - optionals_double.add(optional_str[2:]) - elif optional_str.startswith("-"): - optionals_single.add(optional_str[1:]) - specials.extend(get_specials(optional, "n", optional_str)) - - for positional in cparser._get_positional_actions(): - if positional.help != SUPPRESS: - positional_idx += 1 - log.debug( - "%s| Positional #%d: %s", - log_prefix, - positional_idx, - positional.dest, - ) - index_choices[positional_idx][tuple(requirements)] = positional - if not requirements and isinstance(positional.choices, dict): - for subcmd, subparser in positional.choices.items(): - log.debug("%s| | SubParser: %s", log_prefix, subcmd) - recurse_parser( - subparser, positional_idx, requirements + [subcmd] - ) - - recurse_parser(parser, 0) - - for idx, ndict in index_choices.items(): - if len(ndict) == 1: - # Single choice, no requirements - arg = list(ndict.values())[0] - specials.extend(get_specials(arg, "p", str(idx))) - else: - # Multiple requirements - nlist = [] - for nn, arg in ndict.items(): - checks = [ - '[ "$cmd[{}]" == "{}" ]'.format(iidx, n) - for iidx, n in enumerate(nn, start=2) - ] - if arg.choices: - nlist.append( - '( {}echo "{}" || false )'.format( - " && ".join(checks + [""]), # Append the separator - "\\n".join(arg.choices), - ) - ) - - # Ugly hack - specials.append( - "'p@{}@`set cmd=($COMMAND_LINE); {}`@'".format( - str(idx), " || ".join(nlist) - ) - ) - - return Template( - """\ -# AUTOMATICALLY GENERATED by `shtab` - -${preamble} - -complete ${prog} \\ - 'c/--/(${optionals_double_str})/' \\ - 'c/-/(${optionals_single_str} -)/' \\ - ${optionals_special_str} \\ - 'p/*/()/'""" - ).safe_substitute( - preamble=( - "\n# Custom Preamble\n" + preamble + "\n# End Custom Preamble\n" - if preamble - else "" - ), - root_prefix=root_prefix, - prog=parser.prog, - optionals_double_str=" ".join(optionals_double), - optionals_single_str=" ".join(optionals_single), - optionals_special_str=" \\\n ".join(specials), - ) - - -def complete( - parser: ArgumentParser, - shell: str = "bash", - root_prefix: Opt[str] = None, - preamble: Union[str, Dict] = "", - choice_functions: Opt[Any] = None, -) -> str: - """ - parser : argparse.ArgumentParser - shell : str (bash/zsh) - root_prefix : str or `None` - prefix for shell functions to avoid clashes (default: "_{parser.prog}") - preamble : dict or str - mapping shell to text to prepend to generated script - (e.g. `{"bash": "_myprog_custom_function(){ echo hello }"}`) - choice_functions : deprecated - - N.B. `parser.add_argument().complete = ...` can be used to define custom - completions (e.g. filenames). See <../examples/pathcomplete.py>. - """ - if isinstance(preamble, dict): - preamble = preamble.get(shell, "") - completer = get_completer(shell) - return completer( - parser, - root_prefix=root_prefix, - preamble=preamble, - choice_functions=choice_functions, - ) - - -def completion_action(parent=None, preamble=""): - class PrintCompletionAction(Action): - def __call__(self, parser, namespace, values, option_string=None): - print(complete(parent or parser, values, preamble=preamble)) - parser.exit(0) - - return PrintCompletionAction - - -def add_argument_to( - parser, - option_string="--print-completion", - help="print shell completion script", - parent=None, - preamble="", -): - """ - parser : argparse.ArgumentParser - option_string : str or list[str] - iff positional (no `-` prefix) then `parser` is assumed to actually be - a subparser (subcommand mode) - help : str - parent : argparse.ArgumentParser - required in subcommand mode - """ - if isinstance( - option_string, str if sys.version_info[0] > 2 else basestring # NOQA - ): - option_string = [option_string] - kwargs = { - "choices": SUPPORTED_SHELLS, - "default": None, - "help": help, - "action": completion_action(parent, preamble), - } - if option_string[0][0] != "-": # subparser mode - kwargs.update(default=SUPPORTED_SHELLS[0], nargs="?") - assert parent is not None, "subcommand mode: parent required" - parser.add_argument(*option_string, **kwargs) - return parser diff --git a/tyro/_shtab/__main__.py b/tyro/_shtab/__main__.py deleted file mode 100644 index 180d25377..000000000 --- a/tyro/_shtab/__main__.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import absolute_import - -import logging -import sys - -from .main import main - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - sys.exit(main(sys.argv[1:]) or 0) diff --git a/tyro/_shtab/_dist_ver.py b/tyro/_shtab/_dist_ver.py deleted file mode 100644 index 5ad5a81bd..000000000 --- a/tyro/_shtab/_dist_ver.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "1.5.6.dev1+g4c0170c.d20220909" diff --git a/tyro/_shtab/main.py b/tyro/_shtab/main.py deleted file mode 100644 index 75b80926b..000000000 --- a/tyro/_shtab/main.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import absolute_import, print_function - -import argparse -import logging -import os -import sys -from importlib import import_module - -from . import SUPPORTED_SHELLS, __version__, complete - -log = logging.getLogger(__name__) - - -def get_main_parser(): - parser = argparse.ArgumentParser(prog="shtab") - parser.add_argument( - "parser", help="importable parser (or fuction returning parser)" - ) - parser.add_argument( - "--version", action="version", version="%(prog)s " + __version__ - ) - parser.add_argument( - "-s", "--shell", default=SUPPORTED_SHELLS[0], choices=SUPPORTED_SHELLS - ) - parser.add_argument( - "--prefix", help="prepended to generated functions to avoid clashes" - ) - parser.add_argument("--preamble", help="prepended to generated script") - parser.add_argument("--prog", help="custom program name (overrides `parser.prog`)") - parser.add_argument( - "-u", - "--error-unimportable", - default=False, - action="store_true", - help="raise errors if `parser` is not found in $PYTHONPATH", - ) - parser.add_argument( - "--verbose", - dest="loglevel", - action="store_const", - default=logging.INFO, - const=logging.DEBUG, - help="Log debug information", - ) - return parser - - -def main(argv=None): - parser = get_main_parser() - args = parser.parse_args(argv) - logging.basicConfig(level=args.loglevel) - log.debug(args) - - module, other_parser = args.parser.rsplit(".", 1) - if sys.path and sys.path[0]: - # not blank so not searching curdir - sys.path.insert(1, os.curdir) - try: - module = import_module(module) - except ImportError as err: - if args.error_unimportable: - raise - log.debug(str(err)) - return - other_parser = getattr(module, other_parser) - if callable(other_parser): - other_parser = other_parser() - if args.prog: - other_parser.prog = args.prog - print( - complete( - other_parser, - shell=args.shell, - root_prefix=args.prefix or args.parser.split(".", 1)[0], - preamble=args.preamble, - ) - ) diff --git a/tyro/_shtab/py.typed b/tyro/_shtab/py.typed deleted file mode 100644 index 92ab5d63a..000000000 --- a/tyro/_shtab/py.typed +++ /dev/null @@ -1,2 +0,0 @@ -This file exists solely to signal that the `shtab` package carries inline types. -Do not delete it. From 6c3c1fd14e2c6e2601f8aa6851420e16dcc2e563 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 2 Nov 2022 22:53:34 +0800 Subject: [PATCH 206/491] Docs, types, support/tests for attrs and pydantic --- README.md | 69 +++++++-------- docs/source/conf.py | 2 + docs/source/index.md | 23 ++--- docs/source/your_first_cli.md | 8 +- examples/04_additional/08_pydantic.py | 26 ++++++ examples/04_additional/09_attrs.py | 31 +++++++ poetry.lock | 55 +++++++++++- pyproject.toml | 5 +- tests/test_attrs.py | 103 ++++++++++++++++++++++ tests/test_dcargs.py | 3 +- tests/test_pydantic.py | 50 +++++++++++ tests/test_unsupported_but_should_work.py | 75 +--------------- tyro/_cli.py | 15 ++-- tyro/_docstrings.py | 10 ++- tyro/_fields.py | 77 ++++++++++++++-- tyro/_parsers.py | 2 +- tyro/extras/_base_configs.py | 2 +- tyro/extras/_serialization.py | 2 +- 18 files changed, 410 insertions(+), 148 deletions(-) create mode 100644 examples/04_additional/08_pydantic.py create mode 100644 examples/04_additional/09_attrs.py create mode 100644 tests/test_attrs.py create mode 100644 tests/test_pydantic.py diff --git a/README.md b/README.md index f070c1587..6ebbfda15 100644 --- a/README.md +++ b/README.md @@ -43,42 +43,6 @@ configuration objects with type-annotated Python. Our core interface consists of one function, `tyro.cli()`, that generates argument parsers from Python callables and types. -To get started, we recommend visiting the examples in our -[documentation](https://brentyi.github.io/tyro). - -### Why `tyro`? - -1. **Strong typing.** - - Unlike tools dependent on dictionaries, YAML, or dynamic namespaces, - arguments populated by `tyro` benefit from IDE and language server-supported - operations — think tab completion, rename, jump-to-def, docstrings on hover — - as well as static checking tools like `pyright` and `mypy`. - -2. **Minimal overhead.** - - Standard Python type annotations, docstrings, and default values are parsed - to automatically generate command-line interfaces with informative helptext. - - If you're familiar with type annotations and docstrings in Python, you - already know how to use `tyro`! If you're not, learning to use `tyro` reduces - to learning to write modern Python. - - Hate `tyro`? Just remove one line of code, and you're left with beautiful, - type-annotated, and documented vanilla Python that can be used with a range - of other configuration libraries. - -3. **Modularity.** - - `tyro` supports hierarchical configuration structures, which make it easy to - distribute definitions, defaults, and documentation of configurable fields - across modules or source files. - -4. **Tab completion.** - - By extending [shtab](https://github.com/iterative/shtab), `tyro` - automatically generates tab completion scripts for bash, zsh, and tcsh. - ### A minimal example As a replacement for `argparse`: @@ -151,6 +115,39 @@ print(args.a + args.b) For more examples, see our [documentation](https://brentyi.github.io/tyro). +### Why `tyro`? + +1. **Strong typing.** + + Unlike tools dependent on dictionaries, YAML, or dynamic namespaces, + arguments populated by `tyro` benefit from IDE and language server-supported + operations — think tab completion, rename, jump-to-def, docstrings on hover — + as well as static checking tools like `pyright` and `mypy`. + +2. **Minimal overhead.** + + Standard Python type annotations, docstrings, and default values are parsed + to automatically generate command-line interfaces with informative helptext. + + If you're familiar with type annotations and docstrings in Python, you + already know how to use `tyro`! If you're not, learning to use `tyro` reduces + to learning to write modern Python. + + Hate `tyro`? Just remove one line of code, and you're left with beautiful, + type-annotated, and documented vanilla Python that can be used with a range + of other configuration libraries. + +3. **Modularity.** + + `tyro` supports hierarchical configuration structures, which make it easy to + distribute definitions, defaults, and documentation of configurable fields + across modules or source files. + +4. **Tab completion.** + + By extending [shtab](https://github.com/iterative/shtab), `tyro` + automatically generates tab completion scripts for bash, zsh, and tcsh. + ### In the wild `tyro` is still a new library, but being stress tested in several projects! diff --git a/docs/source/conf.py b/docs/source/conf.py index 6ecf03022..5b8366b4b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -384,6 +384,8 @@ todo_include_todos = True # -- Enable Markdown -> RST conversion ---------------------------------------- + + def docstring(app, what, name, obj, options, lines): md = "\n".join(lines) rst = m2r2.convert(md) diff --git a/docs/source/index.md b/docs/source/index.md index 853a17b74..9457b4a62 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -69,17 +69,6 @@ To get started, we recommend browsing the examples to the left. installation your_first_cli -.. toctree:: - :caption: Notes - :hidden: - :maxdepth: 5 - :glob: - - goals_and_alternatives - helptext_generation - tab_completion - building_configuration_systems - .. toctree:: :caption: Basics :hidden: @@ -120,6 +109,18 @@ To get started, we recommend browsing the examples to the left. examples/04_additional/* +.. toctree:: + :caption: Notes + :hidden: + :maxdepth: 5 + :glob: + + goals_and_alternatives + helptext_generation + tab_completion + building_configuration_systems + + .. toctree:: :caption: API Reference :hidden: diff --git a/docs/source/your_first_cli.md b/docs/source/your_first_cli.md index 57bf1185c..dc26f658b 100644 --- a/docs/source/your_first_cli.md +++ b/docs/source/your_first_cli.md @@ -57,7 +57,7 @@ args = tyro.cli(Args) print(args.a + args.b) ``` -And that's it for the core API! By incorporating more advanced type annotations -from the standard library, we can specify a broad range of more advanced -behaviors: variable-length inputs, unions over types, subcommands, and more. Our -examples walk through a selection of these features. +And that's it! By incorporating more advanced type annotations from the standard +library, we can specify a broad range of more advanced behaviors: +variable-length inputs, unions over types, subcommands, and more. Our examples +walk through a selection of these features. diff --git a/examples/04_additional/08_pydantic.py b/examples/04_additional/08_pydantic.py new file mode 100644 index 000000000..5c747f159 --- /dev/null +++ b/examples/04_additional/08_pydantic.py @@ -0,0 +1,26 @@ +"""Pydantic + +In addition to standard dataclasses, `tyro` also supports +[Pydantic](https://github.com/pydantic/pydantic) models. + +Usage: +`python ./08_pydantic.py --help` +`python ./08_pydantic.py --field1 hello` +`python ./08_pydantic.py --field1 hello --field2 5` +""" +from pydantic import BaseModel, Field + +import tyro + + +class Args(BaseModel): + """Description. + This should show up in the helptext!""" + + field1: str + field2: int = Field(3, description="An integer field.") + + +if __name__ == "__main__": + args = tyro.cli(Args) + print(args) diff --git a/examples/04_additional/09_attrs.py b/examples/04_additional/09_attrs.py new file mode 100644 index 000000000..cabd6a996 --- /dev/null +++ b/examples/04_additional/09_attrs.py @@ -0,0 +1,31 @@ +"""Pydantic + +In addition to standard dataclasses, `tyro` also supports +[Pydantic](https://github.com/pydantic/pydantic) models. + +Usage: +`python ./09_attrs.py --help` +`python ./09_attrs.py --field1 hello` +`python ./09_attrs.py --field1 hello --field2 5` +""" + +import attr + +import tyro + + +@attr.s +class Args: + """Description. + This should show up in the helptext!""" + + field1: str = attr.ib() + """A string field.""" + + field2: int = attr.ib(factory=lambda: 5) + """A required integer field.""" + + +if __name__ == "__main__": + args = tyro.cli(Args) + print(args) diff --git a/poetry.lock b/poetry.lock index acba0d084..304b61bdf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -441,6 +441,21 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "pydantic" +version = "1.10.2" +description = "Data validation and settings management using python type hints" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +typing-extensions = ">=4.1.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + [[package]] name = "Pygments" version = "2.13.0" @@ -666,7 +681,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "f273885b6e4daaa758c2bd6e777e974aab9c320d63417dd5476c0e4c46211277" +content-hash = "6795028b77eb155f329599628507fba6ce1bcf27e7917d98171066bc9c85282b" [metadata.files] absl-py = [ @@ -1147,6 +1162,44 @@ py = [ {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] +pydantic = [ + {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, + {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, + {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"}, + {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"}, + {file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"}, + {file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"}, + {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"}, + {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"}, + {file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"}, + {file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"}, + {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"}, + {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"}, + {file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"}, + {file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"}, + {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"}, + {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"}, + {file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"}, + {file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"}, + {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"}, + {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"}, + {file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"}, + {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, + {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, +] Pygments = [ {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, diff --git a/pyproject.toml b/pyproject.toml index de342f5e1..3bd16fe60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ frozendict = "^2.3.4" rich = ">=11.1.0" shtab = "^1.5.6" -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] pytest = "^7.1.2" pytest-cov = "^3.0.0" omegaconf = "^2.2.2" @@ -30,9 +30,8 @@ mypy = "^0.971" pyright = "^1.1.264" coverage = {extras = ["toml"], version = "^6.4.2"} numpy = ">=1.20.0" - -[tool.poetry.group.dev.dependencies] flax = "^0.6.0" +pydantic = "^1.10.2" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tests/test_attrs.py b/tests/test_attrs.py new file mode 100644 index 000000000..85a463358 --- /dev/null +++ b/tests/test_attrs.py @@ -0,0 +1,103 @@ +import contextlib +import io +import pathlib +from typing import cast + +import attr +import pytest +from attrs import define, field + +import tyro + + +def test_attrs_basic(): + @attr.s + class ManyTypesA: + i: int = attr.ib() + s: str = attr.ib() + f: float = attr.ib() + p: pathlib.Path = attr.ib() + + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( + ManyTypesA, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--p", + "~", + ], + ) == ManyTypesA(i=5, s="5", f=5.0, p=pathlib.Path("~")) + + +def test_attrs_defaults(): + @attr.s + class ManyTypesB: + i: int = attr.ib() + s: str = attr.ib() + f: float = attr.ib(default=1.0) + + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( + ManyTypesB, + args=[ + "--i", + "5", + "--s", + "5", + ], + ) == ManyTypesB(i=5, s="5", f=1.0) + + +def test_attrs_helptext(): + @attr.s + class Helptext: + """This docstring should be printed as a description.""" + + x: int = attr.ib() # Documentation 1 + + # Documentation 2 + y: int = attr.ib() + + z: int = attr.ib(default=3) + """Documentation 3""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(Helptext, args=["--help"]) + helptext = f.getvalue() + assert tyro._strings.strip_ansi_sequences(cast(str, Helptext.__doc__)) in helptext + + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + assert "Documentation 3" in helptext + + +def test_attrs_next_gen_and_factory(): + @define + class Helptext: + """This docstring should be printed as a description.""" + + x: int # Documentation 1 + + # Documentation 2 + y: int + + z: int = field(factory=lambda: 3) + """Documentation 3""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(Helptext, args=["--help"]) + helptext = f.getvalue() + assert tyro._strings.strip_ansi_sequences(cast(str, Helptext.__doc__)) in helptext + + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + assert "Documentation 3" in helptext diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 82aece9a2..346b64a4f 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -491,7 +491,8 @@ def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]: def test_fixed_dataclass_type(): - dummy = lambda: 5 # noqa + def dummy(): + return 5 # noqa def main(x: Callable = dummy) -> Callable: return x diff --git a/tests/test_pydantic.py b/tests/test_pydantic.py new file mode 100644 index 000000000..50e1f3491 --- /dev/null +++ b/tests/test_pydantic.py @@ -0,0 +1,50 @@ +import contextlib +import io +import pathlib +from typing import cast + +import pytest +from pydantic import BaseModel, Field + +import tyro + + +def test_pydantic(): + class ManyTypesA(BaseModel): + i: int + s: str = "hello" + f: float = Field(default_factory=lambda: 3.0) + p: pathlib.Path + + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( + ManyTypesA, + args=[ + "--i", + "5", + "--p", + "~", + ], + ) == ManyTypesA(i=5, s="hello", f=3.0, p=pathlib.Path("~")) + + +def test_pydantic_helptext(): + class Helptext(BaseModel): + """This docstring should be printed as a description.""" + + x: int = Field(description="Documentation 1") + + y: int = Field(description="Documentation 2") + + z: int = Field(description="Documentation 3") + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(Helptext, args=["--help"]) + helptext = f.getvalue() + assert tyro._strings.strip_ansi_sequences(cast(str, Helptext.__doc__)) in helptext + + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + assert "Documentation 3" in helptext diff --git a/tests/test_unsupported_but_should_work.py b/tests/test_unsupported_but_should_work.py index 9045e3503..7341d3251 100644 --- a/tests/test_unsupported_but_should_work.py +++ b/tests/test_unsupported_but_should_work.py @@ -3,12 +3,8 @@ Includes things like omegaconf.MISSING, attrs, etc, which mostly work but either likely have corner cases or just seem sketchy. """ -import contextlib -import io -import pathlib -from typing import Tuple, cast +from typing import Tuple -import attr import omegaconf import pytest @@ -60,72 +56,3 @@ def main2( tyro.cli(main2, args="--required-a 3 --optional 4") with pytest.raises(SystemExit): tyro.cli(main2, args="--required-a 3") - - -def test_attrs_basic(): - @attr.s - class ManyTypesA: - i: int = attr.ib() - s: str = attr.ib() - f: float = attr.ib() - p: pathlib.Path = attr.ib() - - # We can directly pass a dataclass to `tyro.cli()`: - assert tyro.cli( - ManyTypesA, - args=[ - "--i", - "5", - "--s", - "5", - "--f", - "5", - "--p", - "~", - ], - ) == ManyTypesA(i=5, s="5", f=5.0, p=pathlib.Path("~")) - - -def test_attrs_defaults(): - @attr.s - class ManyTypesB: - i: int = attr.ib() - s: str = attr.ib() - f: float = attr.ib(default=1.0) - - # We can directly pass a dataclass to `tyro.cli()`: - assert tyro.cli( - ManyTypesB, - args=[ - "--i", - "5", - "--s", - "5", - ], - ) == ManyTypesB(i=5, s="5", f=1.0) - - -def test_attrs_helptext(): - @attr.s - class Helptext: - """This docstring should be printed as a description.""" - - x: int = attr.ib() # Documentation 1 - - # Documentation 2 - y: int = attr.ib() - - z: int = attr.ib(default=3) - """Documentation 3""" - - f = io.StringIO() - with pytest.raises(SystemExit): - with contextlib.redirect_stdout(f): - tyro.cli(Helptext, args=["--help"]) - helptext = f.getvalue() - assert tyro._strings.strip_ansi_sequences(cast(str, Helptext.__doc__)) in helptext - - # Note that required detection seems to be broken here. - assert "Documentation 1" in helptext - assert "Documentation 2" in helptext - assert "Documentation 3" in helptext diff --git a/tyro/_cli.py b/tyro/_cli.py index c4c9a5fca..489250a89 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -47,7 +47,10 @@ def cli( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, + # Note that passing a default makes sense for things like dataclasses, but are not + # supported for general callables. These can, however, be specified in the signature + # of the callable itself. + default: None = None, ) -> OutT: ... @@ -108,14 +111,14 @@ def cli( `argparse.ArgumentParser()`. args: If set, parse arguments from a sequence of strings instead of the commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. - default: An instance of `T` to use for default values; supported for nested - structures like dataclasses and dictionaries, but not if `f` is a function - or standard class. Helpful for merging CLI arguments with values loaded from - elsewhere. (for example, a config object loaded from a yaml file) + default: An instance of `OutT` to use for default values; supported if `f` is a + type like a dataclass or dictionary, but not if `f` is a general callable like + a function or standard class. Helpful for merging CLI arguments with values + loaded from elsewhere. (for example, a config object loaded from a yaml file) Returns: The output of `f(...)` or an instance `f`. If `f` is a class, the two are - typically equivalent. + equivalent. """ return cast( OutT, diff --git a/tyro/_docstrings.py b/tyro/_docstrings.py index 4b50dd26c..fe96d8dda 100644 --- a/tyro/_docstrings.py +++ b/tyro/_docstrings.py @@ -140,9 +140,13 @@ def get_class_tokenization_with_field( try: tokenization = _ClassTokenization.make(search_cls) # type: ignore except OSError as e: - # Dynamic dataclasses will result in an OSError -- this is fine, we just assume - # there's no docstring. - assert "could not find class definition" in e.args[0] + assert ( + # Dynamic dataclasses will result in an OSError -- this is fine, we just assume + # there's no docstring. + "could not find class definition" in e.args[0] + # Pydantic. + or "source code not available" in e.args[0] + ) return None except TypeError as e: # pragma: no cover # Notebooks cause “___ is a built-in class” TypeError. diff --git a/tyro/_fields.py b/tyro/_fields.py index d5acaa50a..b01111df7 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -33,6 +33,16 @@ from . import _docstrings, _instantiators, _resolver, _singleton, _strings from .conf import _markers +# Support attrs and pydantic if they're installed. +try: + import attr +except ImportError: + attr = None # type: ignore +try: + import pydantic +except ImportError: + pydantic = None # type: ignore + @dataclasses.dataclass(frozen=True) class FieldDefinition: @@ -206,14 +216,21 @@ def _try_field_list_from_callable( f_origin = _resolver.unwrap_origin_strip_extras(f) # Try special cases. - if cls is not None and is_typeddict(cls): - return _try_field_list_from_typeddict(cls, default_instance) + if cls is not None: + if is_typeddict(cls): + return _try_field_list_from_typeddict(cls, default_instance) + + if _resolver.is_namedtuple(cls): + return _try_field_list_from_namedtuple(cls, default_instance) + + if _resolver.is_dataclass(cls): + return _try_field_list_from_dataclass(cls, default_instance) - elif cls is not None and _resolver.is_namedtuple(cls): - return _try_field_list_from_namedtuple(cls, default_instance) + if pydantic is not None and issubclass(cls, pydantic.BaseModel): + return _try_field_list_from_pydantic(cls, default_instance) - elif cls is not None and _resolver.is_dataclass(cls): - return _try_field_list_from_dataclass(cls, default_instance) + if attr is not None and attr.has(cls): + return _try_field_list_from_attrs(cls, default_instance) # Standard container types. These are special because they can be nested structures # if they contain other nested types (eg Tuple[Struct, Struct]), or treated as @@ -351,6 +368,54 @@ def _try_field_list_from_dataclass( return field_list +def _try_field_list_from_pydantic( + cls: Type, default_instance: _DefaultInstance +) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: + assert pydantic is not None + + # Handle pydantic models. + field_list = [] + for pd_field in cls.__fields__.values(): + field_list.append( + FieldDefinition.make( + name=pd_field.name, + typ=pd_field.outer_type_, + default=MISSING_NONPROP + if pd_field.required + else pd_field.get_default(), + helptext=pd_field.field_info.description, + ) + ) + return field_list + + +def _try_field_list_from_attrs( + cls: Type, default_instance: _DefaultInstance +) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: + assert attr is not None + + # Handle attr classes. + field_list = [] + for attr_field in attr.fields(cls): + # Default handling. + default = attr_field.default + if default is attr.NOTHING: + default = MISSING_NONPROP + elif isinstance(default, attr.Factory): # type: ignore + default = default.factory() # type: ignore + + assert attr_field.type is not None + field_list.append( + FieldDefinition.make( + name=attr_field.name, + typ=attr_field.type, + default=default, + helptext=_docstrings.get_field_docstring(cls, attr_field.name), + ) + ) + return field_list + + def _field_list_from_tuple( f: Union[Callable, Type], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 62cb0a182..677ff7e2b 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -284,7 +284,7 @@ def from_field( _resolver.apply_type_from_typevar(typ, type_from_typevar) for typ in get_args(typ) ] - options_no_none = [o for o in options if o != type(None)] # noqa + options_no_none = [o for o in options if o is not type(None)] # noqa if not all( [ _fields.is_nested_type(o, _fields.MISSING_NONPROP) diff --git a/tyro/extras/_base_configs.py b/tyro/extras/_base_configs.py index 347d50eb7..1baf6b29a 100644 --- a/tyro/extras/_base_configs.py +++ b/tyro/extras/_base_configs.py @@ -68,7 +68,7 @@ def train( ```python if TYPE_CHECKING: - SelectableConfig = ExperimentConfig + SelectableConfig = Config else: SelectableConfig = subcommand_type_from_defaults(base_mapping) ``` diff --git a/tyro/extras/_serialization.py b/tyro/extras/_serialization.py index f4befc8f5..ea534062d 100644 --- a/tyro/extras/_serialization.py +++ b/tyro/extras/_serialization.py @@ -45,7 +45,7 @@ def handle_type(typ: Type) -> Set[Type]: ) # Handle enums. - elif type(typ) is enum.EnumMeta: + elif isinstance(typ, enum.EnumMeta): return {typ} # Handle Union, Annotated, List, etc. No-op when there are no args. From 9eac8e41f876a28da34c950b60d9209c5cf9b4d1 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 2 Nov 2022 22:57:30 +0800 Subject: [PATCH 207/491] Housekeeping --- .github/workflows/build.yml | 2 +- .github/workflows/coverage.yml | 2 +- .github/workflows/mypy.yml | 2 +- .github/workflows/publish.yml | 2 +- .../examples/04_additional/08_pydantic.rst | 56 +++++++++++++++++ .../examples/04_additional/09_attrs.rst | 60 +++++++++++++++++++ examples/04_additional/09_attrs.py | 2 +- 7 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 docs/source/examples/04_additional/08_pydantic.rst create mode 100644 docs/source/examples/04_additional/09_attrs.rst diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8718ca45e..cfb9ae844 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 8eec8ecfa..d5ad1acb9 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python 3.8 - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: 3.8 - name: Install dependencies diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 507d9046d..e317f125b 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: "Set up Python 3.8" - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: "3.8" - name: Install dependencies diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c3d39b48b..ac50fb86d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: '3.8' - name: Install dependencies diff --git a/docs/source/examples/04_additional/08_pydantic.rst b/docs/source/examples/04_additional/08_pydantic.rst new file mode 100644 index 000000000..085d991b7 --- /dev/null +++ b/docs/source/examples/04_additional/08_pydantic.rst @@ -0,0 +1,56 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Pydantic +========================================== + + +In addition to standard dataclasses, ``tyro`` also supports +`Pydantic `_ models. + + + +.. code-block:: python + :linenos: + + + from pydantic import BaseModel, Field + + import tyro + + + class Args(BaseModel): + """Description. + This should show up in the helptext!""" + + field1: str + field2: int = Field(3, description="An integer field.") + + + if __name__ == "__main__": + args = tyro.cli(Args) + print(args) + +------------ + +.. raw:: html + + python 04_additional/08_pydantic.py --help + +.. program-output:: python ../../examples/04_additional/08_pydantic.py --help + +------------ + +.. raw:: html + + python 04_additional/08_pydantic.py --field1 hello + +.. program-output:: python ../../examples/04_additional/08_pydantic.py --field1 hello + +------------ + +.. raw:: html + + python 04_additional/08_pydantic.py --field1 hello --field2 5 + +.. program-output:: python ../../examples/04_additional/08_pydantic.py --field1 hello --field2 5 diff --git a/docs/source/examples/04_additional/09_attrs.rst b/docs/source/examples/04_additional/09_attrs.rst new file mode 100644 index 000000000..0f4d99408 --- /dev/null +++ b/docs/source/examples/04_additional/09_attrs.rst @@ -0,0 +1,60 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Pydantic +========================================== + + +In addition to standard dataclasses, ``tyro`` also supports +`attrs `_ classes. + + + +.. code-block:: python + :linenos: + + + import attr + + import tyro + + + @attr.s + class Args: + """Description. + This should show up in the helptext!""" + + field1: str = attr.ib() + """A string field.""" + + field2: int = attr.ib(factory=lambda: 5) + """A required integer field.""" + + + if __name__ == "__main__": + args = tyro.cli(Args) + print(args) + +------------ + +.. raw:: html + + python 04_additional/09_attrs.py --help + +.. program-output:: python ../../examples/04_additional/09_attrs.py --help + +------------ + +.. raw:: html + + python 04_additional/09_attrs.py --field1 hello + +.. program-output:: python ../../examples/04_additional/09_attrs.py --field1 hello + +------------ + +.. raw:: html + + python 04_additional/09_attrs.py --field1 hello --field2 5 + +.. program-output:: python ../../examples/04_additional/09_attrs.py --field1 hello --field2 5 diff --git a/examples/04_additional/09_attrs.py b/examples/04_additional/09_attrs.py index cabd6a996..2fa5ee711 100644 --- a/examples/04_additional/09_attrs.py +++ b/examples/04_additional/09_attrs.py @@ -1,7 +1,7 @@ """Pydantic In addition to standard dataclasses, `tyro` also supports -[Pydantic](https://github.com/pydantic/pydantic) models. +[attrs](https://www.attrs.org/) classes. Usage: `python ./09_attrs.py --help` From cd71d94ca561781ece3df2e2fe6ab9b5c0d6ceec Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 2 Nov 2022 22:59:30 +0800 Subject: [PATCH 208/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3bd16fe60..9f2480c1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.27" +version = "0.3.28" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] From c76ef324521d7f857421c4e9913530300fbca724 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 2 Nov 2022 23:02:45 +0800 Subject: [PATCH 209/491] Fix branch names in icons --- README.md | 4 ++-- docs/source/index.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6ebbfda15..15c890112 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,10 @@

build - mypy + mypy lint - codecov + codecov codecov diff --git a/docs/source/index.md b/docs/source/index.md index 9457b4a62..1c5b8e18f 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -134,13 +134,13 @@ To get started, we recommend browsing the examples to the left. .. |build| image:: https://github.com/brentyi/tyro/workflows/build/badge.svg :alt: Build status icon :target: https://github.com/brentyi/tyro -.. |mypy| image:: https://github.com/brentyi/tyro/workflows/mypy/badge.svg?branch=master +.. |mypy| image:: https://github.com/brentyi/tyro/workflows/mypy/badge.svg?branch=main :alt: Mypy status icon :target: https://github.com/brentyi/tyro .. |lint| image:: https://github.com/brentyi/tyro/workflows/lint/badge.svg :alt: Lint status icon :target: https://github.com/brentyi/tyro -.. |coverage| image:: https://codecov.io/gh/brentyi/tyro/branch/master/graph/badge.svg +.. |coverage| image:: https://codecov.io/gh/brentyi/tyro/branch/main/graph/badge.svg :alt: Test coverage status icon :target: https://codecov.io/gh/brentyi/tyro .. |downloads| image:: https://pepy.tech/badge/tyro From d19c9e701815194b5599ce5c7db016663ec3eda3 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 2 Nov 2022 23:12:04 +0800 Subject: [PATCH 210/491] Update codecov action (running into codecov/codecov-action#844) --- .github/workflows/coverage.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index d5ad1acb9..0c071c395 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -23,10 +23,11 @@ jobs: run: | poetry run pytest --cov=tyro --cov-report=xml - name: Upload to Codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 with: - token: ${{ secrets.CODECOV_TOKEN }} - file: ./coverage.xml + files: ./coverage.xml flags: unittests name: codecov-umbrella fail_ci_if_error: true + verbose: true + From c8c6adf08f918edd6a94d541466a5d0c1572a2c2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 2 Nov 2022 23:19:33 +0800 Subject: [PATCH 211/491] Fix docs typo --- docs/source/examples/04_additional/07_flax.rst | 2 +- docs/source/examples/04_additional/08_pydantic.rst | 2 +- docs/source/examples/04_additional/09_attrs.rst | 2 +- examples/04_additional/07_flax.py | 2 +- examples/04_additional/08_pydantic.py | 2 +- examples/04_additional/09_attrs.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/examples/04_additional/07_flax.rst b/docs/source/examples/04_additional/07_flax.rst index 40b3209df..d9116016b 100644 --- a/docs/source/examples/04_additional/07_flax.rst +++ b/docs/source/examples/04_additional/07_flax.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -Directly Instantiating Flax Modules +JAX/Flax Integration ========================================== diff --git a/docs/source/examples/04_additional/08_pydantic.rst b/docs/source/examples/04_additional/08_pydantic.rst index 085d991b7..3c74d8e7b 100644 --- a/docs/source/examples/04_additional/08_pydantic.rst +++ b/docs/source/examples/04_additional/08_pydantic.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -Pydantic +Pydantic Integration ========================================== diff --git a/docs/source/examples/04_additional/09_attrs.rst b/docs/source/examples/04_additional/09_attrs.rst index 0f4d99408..ab4b2de9f 100644 --- a/docs/source/examples/04_additional/09_attrs.rst +++ b/docs/source/examples/04_additional/09_attrs.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -Pydantic +Attrs Integration ========================================== diff --git a/examples/04_additional/07_flax.py b/examples/04_additional/07_flax.py index 323a1a355..6707964ee 100644 --- a/examples/04_additional/07_flax.py +++ b/examples/04_additional/07_flax.py @@ -1,4 +1,4 @@ -"""Directly Instantiating Flax Modules +"""JAX/Flax Integration If you use [flax.linen](https://github.com/google/flax), modules can be instantiated directly from `tyro.cli`. diff --git a/examples/04_additional/08_pydantic.py b/examples/04_additional/08_pydantic.py index 5c747f159..5af836d90 100644 --- a/examples/04_additional/08_pydantic.py +++ b/examples/04_additional/08_pydantic.py @@ -1,4 +1,4 @@ -"""Pydantic +"""Pydantic Integration In addition to standard dataclasses, `tyro` also supports [Pydantic](https://github.com/pydantic/pydantic) models. diff --git a/examples/04_additional/09_attrs.py b/examples/04_additional/09_attrs.py index 2fa5ee711..0adca8a48 100644 --- a/examples/04_additional/09_attrs.py +++ b/examples/04_additional/09_attrs.py @@ -1,4 +1,4 @@ -"""Pydantic +"""Attrs Integration In addition to standard dataclasses, `tyro` also supports [attrs](https://www.attrs.org/) classes. From a46a4a334bdd48fca8a789101489dd29738e06e0 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 2 Nov 2022 23:31:16 +0800 Subject: [PATCH 212/491] Upgrade coverage --- poetry.lock | 104 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/poetry.lock b/poetry.lock index 304b61bdf..44c922f87 100644 --- a/poetry.lock +++ b/poetry.lock @@ -73,7 +73,7 @@ test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] [[package]] name = "coverage" -version = "6.4.4" +version = "6.5.0" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -681,7 +681,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "6795028b77eb155f329599628507fba6ce1bcf27e7917d98171066bc9c85282b" +content-hash = "49867c0eda45ffab0a881ae1181315655a200d447d3ca4f85bc25713acb4a699" [metadata.files] absl-py = [ @@ -712,56 +712,56 @@ commonmark = [ {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, ] coverage = [ - {file = "coverage-6.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e7b4da9bafad21ea45a714d3ea6f3e1679099e420c8741c74905b92ee9bfa7cc"}, - {file = "coverage-6.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fde17bc42e0716c94bf19d92e4c9f5a00c5feb401f5bc01101fdf2a8b7cacf60"}, - {file = "coverage-6.4.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdbb0d89923c80dbd435b9cf8bba0ff55585a3cdb28cbec65f376c041472c60d"}, - {file = "coverage-6.4.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:67f9346aeebea54e845d29b487eb38ec95f2ecf3558a3cffb26ee3f0dcc3e760"}, - {file = "coverage-6.4.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42c499c14efd858b98c4e03595bf914089b98400d30789511577aa44607a1b74"}, - {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c35cca192ba700979d20ac43024a82b9b32a60da2f983bec6c0f5b84aead635c"}, - {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9cc4f107009bca5a81caef2fca843dbec4215c05e917a59dec0c8db5cff1d2aa"}, - {file = "coverage-6.4.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5f444627b3664b80d078c05fe6a850dd711beeb90d26731f11d492dcbadb6973"}, - {file = "coverage-6.4.4-cp310-cp310-win32.whl", hash = "sha256:66e6df3ac4659a435677d8cd40e8eb1ac7219345d27c41145991ee9bf4b806a0"}, - {file = "coverage-6.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:35ef1f8d8a7a275aa7410d2f2c60fa6443f4a64fae9be671ec0696a68525b875"}, - {file = "coverage-6.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c1328d0c2f194ffda30a45f11058c02410e679456276bfa0bbe0b0ee87225fac"}, - {file = "coverage-6.4.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61b993f3998ee384935ee423c3d40894e93277f12482f6e777642a0141f55782"}, - {file = "coverage-6.4.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d5dd4b8e9cd0deb60e6fcc7b0647cbc1da6c33b9e786f9c79721fd303994832f"}, - {file = "coverage-6.4.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7026f5afe0d1a933685d8f2169d7c2d2e624f6255fb584ca99ccca8c0e966fd7"}, - {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9c7b9b498eb0c0d48b4c2abc0e10c2d78912203f972e0e63e3c9dc21f15abdaa"}, - {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ee2b2fb6eb4ace35805f434e0f6409444e1466a47f620d1d5763a22600f0f892"}, - {file = "coverage-6.4.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ab066f5ab67059d1f1000b5e1aa8bbd75b6ed1fc0014559aea41a9eb66fc2ce0"}, - {file = "coverage-6.4.4-cp311-cp311-win32.whl", hash = "sha256:9d6e1f3185cbfd3d91ac77ea065d85d5215d3dfa45b191d14ddfcd952fa53796"}, - {file = "coverage-6.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:e3d3c4cc38b2882f9a15bafd30aec079582b819bec1b8afdbde8f7797008108a"}, - {file = "coverage-6.4.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a095aa0a996ea08b10580908e88fbaf81ecf798e923bbe64fb98d1807db3d68a"}, - {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef6f44409ab02e202b31a05dd6666797f9de2aa2b4b3534e9d450e42dea5e817"}, - {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b7101938584d67e6f45f0015b60e24a95bf8dea19836b1709a80342e01b472f"}, - {file = "coverage-6.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14a32ec68d721c3d714d9b105c7acf8e0f8a4f4734c811eda75ff3718570b5e3"}, - {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6a864733b22d3081749450466ac80698fe39c91cb6849b2ef8752fd7482011f3"}, - {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:08002f9251f51afdcc5e3adf5d5d66bb490ae893d9e21359b085f0e03390a820"}, - {file = "coverage-6.4.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a3b2752de32c455f2521a51bd3ffb53c5b3ae92736afde67ce83477f5c1dd928"}, - {file = "coverage-6.4.4-cp37-cp37m-win32.whl", hash = "sha256:f855b39e4f75abd0dfbcf74a82e84ae3fc260d523fcb3532786bcbbcb158322c"}, - {file = "coverage-6.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ee6ae6bbcac0786807295e9687169fba80cb0617852b2fa118a99667e8e6815d"}, - {file = "coverage-6.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:564cd0f5b5470094df06fab676c6d77547abfdcb09b6c29c8a97c41ad03b103c"}, - {file = "coverage-6.4.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cbbb0e4cd8ddcd5ef47641cfac97d8473ab6b132dd9a46bacb18872828031685"}, - {file = "coverage-6.4.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6113e4df2fa73b80f77663445be6d567913fb3b82a86ceb64e44ae0e4b695de1"}, - {file = "coverage-6.4.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d032bfc562a52318ae05047a6eb801ff31ccee172dc0d2504614e911d8fa83e"}, - {file = "coverage-6.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e431e305a1f3126477abe9a184624a85308da8edf8486a863601d58419d26ffa"}, - {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cf2afe83a53f77aec067033199797832617890e15bed42f4a1a93ea24794ae3e"}, - {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:783bc7c4ee524039ca13b6d9b4186a67f8e63d91342c713e88c1865a38d0892a"}, - {file = "coverage-6.4.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ff934ced84054b9018665ca3967fc48e1ac99e811f6cc99ea65978e1d384454b"}, - {file = "coverage-6.4.4-cp38-cp38-win32.whl", hash = "sha256:e1fabd473566fce2cf18ea41171d92814e4ef1495e04471786cbc943b89a3781"}, - {file = "coverage-6.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:4179502f210ebed3ccfe2f78bf8e2d59e50b297b598b100d6c6e3341053066a2"}, - {file = "coverage-6.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:98c0b9e9b572893cdb0a00e66cf961a238f8d870d4e1dc8e679eb8bdc2eb1b86"}, - {file = "coverage-6.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fc600f6ec19b273da1d85817eda339fb46ce9eef3e89f220055d8696e0a06908"}, - {file = "coverage-6.4.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a98d6bf6d4ca5c07a600c7b4e0c5350cd483c85c736c522b786be90ea5bac4f"}, - {file = "coverage-6.4.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01778769097dbd705a24e221f42be885c544bb91251747a8a3efdec6eb4788f2"}, - {file = "coverage-6.4.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfa0b97eb904255e2ab24166071b27408f1f69c8fbda58e9c0972804851e0558"}, - {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:fcbe3d9a53e013f8ab88734d7e517eb2cd06b7e689bedf22c0eb68db5e4a0a19"}, - {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:15e38d853ee224e92ccc9a851457fb1e1f12d7a5df5ae44544ce7863691c7a0d"}, - {file = "coverage-6.4.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6913dddee2deff8ab2512639c5168c3e80b3ebb0f818fed22048ee46f735351a"}, - {file = "coverage-6.4.4-cp39-cp39-win32.whl", hash = "sha256:354df19fefd03b9a13132fa6643527ef7905712109d9c1c1903f2133d3a4e145"}, - {file = "coverage-6.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:1238b08f3576201ebf41f7c20bf59baa0d05da941b123c6656e42cdb668e9827"}, - {file = "coverage-6.4.4-pp36.pp37.pp38-none-any.whl", hash = "sha256:f67cf9f406cf0d2f08a3515ce2db5b82625a7257f88aad87904674def6ddaec1"}, - {file = "coverage-6.4.4.tar.gz", hash = "sha256:e16c45b726acb780e1e6f88b286d3c10b3914ab03438f32117c4aa52d7f30d58"}, + {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, + {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83516205e254a0cb77d2d7bb3632ee019d93d9f4005de31dca0a8c3667d5bc04"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:97117225cdd992a9c2a5515db1f66b59db634f59d0679ca1fa3fe8da32749cae"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1170fa54185845505fbfa672f1c1ab175446c887cce8212c44149581cf2d466"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:11b990d520ea75e7ee8dcab5bc908072aaada194a794db9f6d7d5cfd19661e5a"}, + {file = "coverage-6.5.0-cp310-cp310-win32.whl", hash = "sha256:5dbec3b9095749390c09ab7c89d314727f18800060d8d24e87f01fb9cfb40b32"}, + {file = "coverage-6.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:59f53f1dc5b656cafb1badd0feb428c1e7bc19b867479ff72f7a9dd9b479f10e"}, + {file = "coverage-6.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33a7da4376d5977fbf0a8ed91c4dffaaa8dbf0ddbf4c8eea500a2486d8bc4d7b"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a6b7d95969b8845250586f269e81e5dfdd8ff828ddeb8567a4a2eaa7313460c4"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1ef221513e6f68b69ee9e159506d583d31aa3567e0ae84eaad9d6ec1107dddaa"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cca4435eebea7962a52bdb216dec27215d0df64cf27fc1dd538415f5d2b9da6b"}, + {file = "coverage-6.5.0-cp311-cp311-win32.whl", hash = "sha256:98e8a10b7a314f454d9eff4216a9a94d143a7ee65018dd12442e898ee2310578"}, + {file = "coverage-6.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:bc8ef5e043a2af066fa8cbfc6e708d58017024dc4345a1f9757b329a249f041b"}, + {file = "coverage-6.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4433b90fae13f86fafff0b326453dd42fc9a639a0d9e4eec4d366436d1a41b6d"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4f05d88d9a80ad3cac6244d36dd89a3c00abc16371769f1340101d3cb899fc3"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:94e2565443291bd778421856bc975d351738963071e9b8839ca1fc08b42d4bef"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:027018943386e7b942fa832372ebc120155fd970837489896099f5cfa2890f79"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:255758a1e3b61db372ec2736c8e2a1fdfaf563977eedbdf131de003ca5779b7d"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:851cf4ff24062c6aec510a454b2584f6e998cada52d4cb58c5e233d07172e50c"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:12adf310e4aafddc58afdb04d686795f33f4d7a6fa67a7a9d4ce7d6ae24d949f"}, + {file = "coverage-6.5.0-cp37-cp37m-win32.whl", hash = "sha256:b5604380f3415ba69de87a289a2b56687faa4fe04dbee0754bfcae433489316b"}, + {file = "coverage-6.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4a8dbc1f0fbb2ae3de73eb0bdbb914180c7abfbf258e90b311dcd4f585d44bd2"}, + {file = "coverage-6.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c"}, + {file = "coverage-6.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:20c8ac5386253717e5ccc827caad43ed66fea0efe255727b1053a8154d952398"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dbdb91cd8c048c2b09eb17713b0c12a54fbd587d79adcebad543bc0cd9a3410b"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:de3001a203182842a4630e7b8d1a2c7c07ec1b45d3084a83d5d227a3806f530f"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e07f4a4a9b41583d6eabec04f8b68076ab3cd44c20bd29332c6572dda36f372e"}, + {file = "coverage-6.5.0-cp38-cp38-win32.whl", hash = "sha256:6d4817234349a80dbf03640cec6109cd90cba068330703fa65ddf56b60223a6d"}, + {file = "coverage-6.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:7ccf362abd726b0410bf8911c31fbf97f09f8f1061f8c1cf03dfc4b6372848f6"}, + {file = "coverage-6.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745"}, + {file = "coverage-6.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:265de0fa6778d07de30bcf4d9dc471c3dc4314a23a3c6603d356a3c9abc2dfcf"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7b6be138d61e458e18d8e6ddcddd36dd96215edfe5f1168de0b1b32635839b62"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:42eafe6778551cf006a7c43153af1211c3aaab658d4d66fa5fcc021613d02518"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:723e8130d4ecc8f56e9a611e73b31219595baa3bb252d539206f7bbbab6ffc1f"}, + {file = "coverage-6.5.0-cp39-cp39-win32.whl", hash = "sha256:d9ecf0829c6a62b9b573c7bb6d4dcd6ba8b6f80be9ba4fc7ed50bf4ac9aecd72"}, + {file = "coverage-6.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc2af30ed0d5ae0b1abdb4ebdce598eafd5b35397d4d75deb341a614d333d987"}, + {file = "coverage-6.5.0-pp36.pp37.pp38-none-any.whl", hash = "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a"}, + {file = "coverage-6.5.0.tar.gz", hash = "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84"}, ] cycler = [ {file = "cycler-0.11.0-py3-none-any.whl", hash = "sha256:3a27e95f763a428a739d2add979fa7494c912a32c17c4c38c4d5f082cad165a3"}, diff --git a/pyproject.toml b/pyproject.toml index 9f2480c1d..c5f54c4b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,10 +28,10 @@ attrs = "^21.4.0" torch = "^1.10.0" mypy = "^0.971" pyright = "^1.1.264" -coverage = {extras = ["toml"], version = "^6.4.2"} numpy = ">=1.20.0" flax = "^0.6.0" pydantic = "^1.10.2" +coverage = {extras = ["toml"], version = "^6.5.0"} [build-system] requires = ["poetry-core>=1.0.0"] From 4d8fb1eb4407df18fbd92a06d38b60a93c8fa12d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 2 Nov 2022 23:55:26 +0800 Subject: [PATCH 213/491] Fix coverage exclude lines --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c5f54c4b4..ffa709bca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,14 +52,14 @@ exclude_lines = [ # Don't compute coverage for abstract methods, properties "@abstract", - "@abc.abstract", + "@abc\\.abstract", # or warnings "warnings", # or empty function bodies "pass", - "...", + "\\.\\.\\.", # or typing imports "TYPE_CHECKING", From 409f5a9a1b9906f2ef72cc2a7e924ee17a2d7cac Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 3 Nov 2022 00:06:22 +0800 Subject: [PATCH 214/491] Re-add codecov token --- .github/workflows/coverage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 0c071c395..c4549834a 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -25,6 +25,7 @@ jobs: - name: Upload to Codecov uses: codecov/codecov-action@v3 with: + token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml flags: unittests name: codecov-umbrella From 99f73e14b3e54188b1b0a58d72ed365ac4e3de80 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 3 Nov 2022 00:42:18 +0800 Subject: [PATCH 215/491] Test coverage improvements --- .github/workflows/coverage.yml | 4 ++-- pyproject.toml | 3 +++ tests/test_errors.py | 20 +++++++++++++++++++- tyro/_cli.py | 1 - tyro/_instantiators.py | 7 ------- tyro/_parsers.py | 15 +-------------- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index c4549834a..be421f157 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.10 uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: "3.10" - name: Install dependencies run: | curl -sSL https://install.python-poetry.org | python3 - diff --git a/pyproject.toml b/pyproject.toml index ffa709bca..eaef34763 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,4 +72,7 @@ exclude_lines = [ # or fallback imports "except ImportError:", + + # or anything that's deprecated + "deprecated", ] diff --git a/tests/test_errors.py b/tests/test_errors.py index 814b89630..0732d6da1 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -1,5 +1,5 @@ import dataclasses -from typing import List, Tuple, Union +from typing import List, Tuple, TypeVar, Union import pytest @@ -97,3 +97,21 @@ def main(*, a) -> None: with pytest.raises(tyro.UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) + + +def test_tuple_needs_default(): + def main(arg: tuple) -> None: # type: ignore + pass + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) + + +def test_unbound_typevar(): + T = TypeVar("T") + + def main(arg: T) -> None: # type: ignore + pass + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) diff --git a/tyro/_cli.py b/tyro/_cli.py index 489250a89..88d2220d4 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -276,7 +276,6 @@ def fix_arg(arg: str) -> str: f, description=description, parent_classes=set(), # Used for recursive calls. - parent_type_from_typevar=None, # Used for recursive calls. default_instance=default_instance_internal, # Overrides for default values. prefix="", # Used for recursive calls. ) diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 62c3e4b1d..4104bbaaa 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -104,13 +104,6 @@ def instantiator_from_type( - A metadata structure, which specifies parameters for argparse. """ - # Resolve typevars. - if typ in type_from_typevar: - return instantiator_from_type( - type_from_typevar[typ], # type: ignore - type_from_typevar, - ) - # Handle Any. if typ is Any: raise UnsupportedTypeAnnotationError("`Any` is not a parsable type.") diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 677ff7e2b..dff720e31 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -40,7 +40,6 @@ def from_callable_or_type( f: Callable[..., T], description: Optional[str], parent_classes: Set[Type], - parent_type_from_typevar: Optional[Dict[TypeVar, Type]], default_instance: Union[ T, _fields.PropagatingMissingType, _fields.NonpropagatingMissingType ], @@ -52,10 +51,6 @@ def from_callable_or_type( # Resolve generic types. f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) - if parent_type_from_typevar is not None: - for typevar, typ in type_from_typevar.items(): - if typ in parent_type_from_typevar: - type_from_typevar[typevar] = parent_type_from_typevar[typ] # type: ignore # Cycle detection. # @@ -135,7 +130,6 @@ def from_callable_or_type( field.typ, description=None, parent_classes=parent_classes, - parent_type_from_typevar=type_from_typevar, default_instance=field.default, prefix=_strings.make_field_name([prefix, field.name]), subcommand_prefix=subcommand_prefix, @@ -238,13 +232,7 @@ def format_group_name(nested_field_name: str) -> str: arg.add_argument(positional_group) continue - if arg.prefix in group_from_prefix: - arg.add_argument(group_from_prefix[arg.prefix]) - else: - # Suppressed argument: still need to add them, but they won't show up in - # the helptext so it doesn't matter which group. - assert arg.lowered.help is argparse.SUPPRESS - arg.add_argument(group_from_prefix[""]) + arg.add_argument(group_from_prefix[arg.prefix]) # Create subparser tree. if len(self.subparsers_from_name) > 0: @@ -393,7 +381,6 @@ def from_field( else option, description=subcommand_config.description, parent_classes=parent_classes, - parent_type_from_typevar=type_from_typevar, default_instance=subcommand_config.default, prefix=prefix, subcommand_prefix=prefix, From d1c8989903958ed1651923c27f9ce4040de9a1c1 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 3 Nov 2022 01:28:57 +0800 Subject: [PATCH 216/491] dataclasses.InitVar[] support, test --- tests/test_dcargs.py | 24 ++++++++++++++++++++++++ tyro/_resolver.py | 15 +++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 346b64a4f..032be0a1c 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -547,3 +547,27 @@ def main() -> argparse.ArgumentParser: return parser assert isinstance(tyro.cli(main, args=[]), argparse.ArgumentParser) + + +def test_dataclass_init_var(): + @dataclasses.dataclass + class DataclassWithInitVar: + i: dataclasses.InitVar[int] + x: str + + def __post_init__(self, i: int) -> None: + self.x += str(i) + + # We can directly pass a dataclass to `tyro.cli()`: + assert ( + tyro.cli( + DataclassWithInitVar, + args=[ + "--i", + "5", + "--x", + "5", + ], + ).x + == "55" + ) diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 4c3c2ea13..1622dce15 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -6,6 +6,7 @@ from typing import ( Any, Callable, + ClassVar, Dict, FrozenSet, List, @@ -18,6 +19,7 @@ cast, ) +from attr import dataclass from typing_extensions import Annotated, get_args, get_origin, get_type_hints TypeOrCallable = TypeVar("TypeOrCallable", Type, Callable) @@ -69,18 +71,27 @@ def resolve_generic_types( def resolved_fields(cls: Type) -> List[dataclasses.Field]: - """Similar to dataclasses.fields, but resolves forward references.""" + """Similar to dataclasses.fields(), but includes dataclasses.InitVar types and + resolves forward references.""" assert dataclasses.is_dataclass(cls) fields = [] annotations = get_type_hints(cls, include_extras=True) - for field in dataclasses.fields(cls): + for field in getattr(cls, "__dataclass_fields__").values(): # Avoid mutating original field. field = copy.copy(field) # Resolve forward references. field.type = annotations[field.name] + # Skip ClassVars. + if get_origin(field.type) is ClassVar: + continue + + # Unwrap InitVar types. + if isinstance(field.type, dataclasses.InitVar): + field.type = field.type.type + fields.append(field) return fields From a352ae99c30f30e616a2e51fdc7e97e5825d2a41 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 3 Nov 2022 01:34:32 +0800 Subject: [PATCH 217/491] Skip InitVar test for Python 3.7 --- tests/test_dcargs.py | 24 ------------------------ tests/test_initvar_ignore_py37.py | 27 +++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 24 deletions(-) create mode 100644 tests/test_initvar_ignore_py37.py diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 032be0a1c..346b64a4f 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -547,27 +547,3 @@ def main() -> argparse.ArgumentParser: return parser assert isinstance(tyro.cli(main, args=[]), argparse.ArgumentParser) - - -def test_dataclass_init_var(): - @dataclasses.dataclass - class DataclassWithInitVar: - i: dataclasses.InitVar[int] - x: str - - def __post_init__(self, i: int) -> None: - self.x += str(i) - - # We can directly pass a dataclass to `tyro.cli()`: - assert ( - tyro.cli( - DataclassWithInitVar, - args=[ - "--i", - "5", - "--x", - "5", - ], - ).x - == "55" - ) diff --git a/tests/test_initvar_ignore_py37.py b/tests/test_initvar_ignore_py37.py new file mode 100644 index 000000000..6082ec1da --- /dev/null +++ b/tests/test_initvar_ignore_py37.py @@ -0,0 +1,27 @@ +import dataclasses + +import tyro + + +def test_dataclass_init_var(): + @dataclasses.dataclass + class DataclassWithInitVar: + i: dataclasses.InitVar[int] + x: str + + def __post_init__(self, i: int) -> None: + self.x += str(i) + + # We can directly pass a dataclass to `tyro.cli()`: + assert ( + tyro.cli( + DataclassWithInitVar, + args=[ + "--i", + "5", + "--x", + "5", + ], + ).x + == "55" + ) From 245b47bdb4af78ef71f58bee50ebc4bfb026f25d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 3 Nov 2022 07:06:53 +0800 Subject: [PATCH 218/491] Fix error when all arguments in a group are suppressed - Regression in https://github.com/brentyi/tyro/commit/99f73e14b3e54188b1b0a58d72ed365ac4e3de80 - Reported in https://github.com/nerfstudio-project/nerfstudio/issues/882 - Closes #18 --- tests/{test_metadata.py => test_conf.py} | 51 ++++++++++++++++++++++++ tests/test_errors.py | 16 ++++++++ tyro/_fields.py | 8 ++++ tyro/_parsers.py | 8 +++- tyro/_resolver.py | 1 - tyro/conf/_markers.py | 3 ++ 6 files changed, 85 insertions(+), 2 deletions(-) rename tests/{test_metadata.py => test_conf.py} (90%) diff --git a/tests/test_metadata.py b/tests/test_conf.py similarity index 90% rename from tests/test_metadata.py rename to tests/test_conf.py index 4573c8ae7..be91679d5 100644 --- a/tests/test_metadata.py +++ b/tests/test_conf.py @@ -401,3 +401,54 @@ class A: args=["--x", "True"], default=A(False), ) == A(True) + + +def test_suppressed_group(): + """Reproduction of https://github.com/nerfstudio-project/nerfstudio/issues/882.""" + + @dataclasses.dataclass + class Inner: + a: int + b: int + + def main( + value: int, + inner: tyro.conf.Suppress[Inner] = Inner(1, 2), + ) -> int: + return value + inner.a + inner.b + + assert tyro.cli(main, args=["--value", "5"]) == 8 + + +def test_fixed_group(): + """Reproduction of https://github.com/nerfstudio-project/nerfstudio/issues/882.""" + + @dataclasses.dataclass + class Inner: + a: int + b: int + + def main( + value: int, + inner: tyro.conf.Fixed[Inner] = Inner(1, 2), + ) -> int: + return value + inner.a + inner.b + + assert tyro.cli(main, args=["--value", "5"]) == 8 + + +def test_fixed_suppressed_group(): + """Reproduction of https://github.com/nerfstudio-project/nerfstudio/issues/882.""" + + @dataclasses.dataclass + class Inner: + a: int + b: int + + def main( + value: int, + inner: tyro.conf.Fixed[Inner] = Inner(1, 2), + ) -> int: + return value + inner.a + inner.b + + assert tyro.cli(main, args=["--value", "5"]) == 8 diff --git a/tests/test_errors.py b/tests/test_errors.py index 0732d6da1..405d25275 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -115,3 +115,19 @@ def main(arg: T) -> None: # type: ignore with pytest.raises(tyro.UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) + + +def test_missing_default_fixed(): + def main(value: tyro.conf.SuppressFixed[tyro.conf.Fixed[int]]) -> int: + return value + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) + + +def test_missing_default_suppressed(): + def main(value: tyro.conf.Suppress[int]) -> int: + return value + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) diff --git a/tyro/_fields.py b/tyro/_fields.py index b01111df7..689debf88 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -57,6 +57,14 @@ class FieldDefinition: # user-facing argument name doesn't match the keyword expected by our callable. name_override: Optional[Any] + def __post_init__(self): + if ( + _markers.Fixed in self.markers or _markers.Suppress in self.markers + ) and self.default in MISSING_SINGLETONS: + raise _instantiators.UnsupportedTypeAnnotationError( + f"Field {self.name} is missing a default value!" + ) + @staticmethod def make( name: str, diff --git a/tyro/_parsers.py b/tyro/_parsers.py index dff720e31..551ae0f4f 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -232,7 +232,13 @@ def format_group_name(nested_field_name: str) -> str: arg.add_argument(positional_group) continue - arg.add_argument(group_from_prefix[arg.prefix]) + if arg.prefix in group_from_prefix: + arg.add_argument(group_from_prefix[arg.prefix]) + else: + # Suppressed argument: still need to add them, but they won't show up in + # the helptext so it doesn't matter which group. + assert arg.lowered.help is argparse.SUPPRESS + arg.add_argument(group_from_prefix[""]) # Create subparser tree. if len(self.subparsers_from_name) > 0: diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 1622dce15..6a521ef28 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -19,7 +19,6 @@ cast, ) -from attr import dataclass from typing_extensions import Annotated, get_args, get_origin, get_type_hints TypeOrCallable = TypeVar("TypeOrCallable", Type, Callable) diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index ffa4f9265..1201ec859 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -18,6 +18,9 @@ """A type `T` can be annotated as `Positional[T]` if we want to parse it as a positional argument.""" +# TODO: the verb tenses here are inconsistent, naming could be revisited. +# Perhaps Suppress should be Suppressed? But SuppressedFixed would be weird. + Fixed = Annotated[T, None] """A type `T` can be annotated as `Fixed[T]` to prevent `tyro.cli` from parsing it; a default value should be set instead. Note that fields with defaults that can't be parsed From 120afd7629f72663a876f136340ac04f51b8ebef Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 3 Nov 2022 07:10:25 +0800 Subject: [PATCH 219/491] Bump version --- pyproject.toml | 2 +- tests/test_conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eaef34763..9fbf15510 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.28" +version = "0.3.29" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] diff --git a/tests/test_conf.py b/tests/test_conf.py index be91679d5..5ba185ea5 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -421,7 +421,7 @@ def main( def test_fixed_group(): - """Reproduction of https://github.com/nerfstudio-project/nerfstudio/issues/882.""" + """Inspired by https://github.com/nerfstudio-project/nerfstudio/issues/882.""" @dataclasses.dataclass class Inner: From 588a0c483c4b9e6ddc38e72f6c00aa2dee0e8cfd Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 7 Nov 2022 05:43:42 -0800 Subject: [PATCH 220/491] PEP 585 support in Python 3.9 (addresses #19) --- tests/conftest.py | 3 ++ .../test_new_style_annotations_above_py39.py | 52 +++++++++++++++++++ tyro/_resolver.py | 20 +++---- 3 files changed, 66 insertions(+), 9 deletions(-) create mode 100644 tests/test_new_style_annotations_above_py39.py diff --git a/tests/conftest.py b/tests/conftest.py index 58c8975e4..2da5d1997 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,5 +7,8 @@ if sys.version_info.major == 3 and sys.version_info.minor == 10: collect_ignore_glob.append("*_ignore_py310.py") +if not (sys.version_info.major == 3 and sys.version_info.minor >= 9): + collect_ignore_glob.append("*_above_py39.py") + if not (sys.version_info.major == 3 and sys.version_info.minor == 10): collect_ignore_glob.append("*_only_py310.py") diff --git a/tests/test_new_style_annotations_above_py39.py b/tests/test_new_style_annotations_above_py39.py new file mode 100644 index 000000000..5ef86858c --- /dev/null +++ b/tests/test_new_style_annotations_above_py39.py @@ -0,0 +1,52 @@ +from typing import Any, Literal, Optional, Union + +import pytest + +import tyro + + +def test_list(): + def main(x: list[bool]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] + + +def test_tuple(): + def main(x: tuple[bool, str]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "True", "False"]) == (True, "False") + + +def test_tuple_variable(): + def main(x: tuple[Union[bool, str], ...]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "True", "Wrong"]) == (True, "Wrong") + + +def test_super_nested(): + def main( + x: Optional[ + list[ + tuple[ + Optional[int], + Literal[3, 4], + Union[tuple[int, int], tuple[str, str]], + ] + ] + ] = None + ) -> Any: + return x + + assert tyro.cli(main, args=[]) is None + assert tyro.cli(main, args="--x None".split(" ")) is None + assert tyro.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] + assert tyro.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] + assert tyro.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ + (2, 3, ("x", "2")), + (2, 3, (1, 2)), + ] + with pytest.raises(SystemExit): + tyro.cli(main, args=["--help"]) diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 6a521ef28..62856e979 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -3,6 +3,7 @@ import copy import dataclasses import sys +import types from typing import ( Any, Callable, @@ -210,21 +211,22 @@ def apply_type_from_typevar( assert isinstance(args[0], list) args = tuple(args[0]) + args[1:] - # Convert Python 3.10-style types to their typing library equivalents, which + # Convert Python 3.9 and 3.10 types to their typing library equivalents, which # support `.copy_with()`. - if sys.version_info[:2] >= (3, 10): - import types - - for new, old in { - # PEP 585 + if sys.version_info[:2] >= (3, 9): + shim_table = { + # PEP 585. Requires Python 3.9. tuple: Tuple, list: List, dict: Dict, set: Set, frozenset: FrozenSet, - # PEP 604 - types.UnionType: Union, - }.items(): + } + if hasattr(types, "UnionType"): # type: ignore + # PEP 604. Requires Python 3.10. + shim_table[types.UnionType] = Union # type: ignore + + for new, old in shim_table.items(): if isinstance(typ, new) or get_origin(typ) is new: # type: ignore typ = old.__getitem__(args) # type: ignore From 79c9dd3d72cad12d7b9f477fa6c9220ec3b8dec3 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 7 Nov 2022 05:57:24 -0800 Subject: [PATCH 221/491] Improve error message for subcommands with bad defaults --- tests/test_nested.py | 22 ++++++++++++++++++++++ tyro/_parsers.py | 8 +++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/test_nested.py b/tests/test_nested.py index 8df60ed6f..358bcb8c5 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -319,6 +319,28 @@ class DefaultInstanceSubparser: tyro.cli(DefaultInstanceSubparser, args=["--x", "1", "c", "--bc.y", "3"]) +def test_subparser_with_default_bad(): + @dataclasses.dataclass + class DefaultHTTPServer: + y: int + + @dataclasses.dataclass + class DefaultSMTPServer: + z: int + + @dataclasses.dataclass + class DefaultSubparser: + x: int + bc: Union[DefaultHTTPServer, DefaultSMTPServer] = dataclasses.field( + default_factory=lambda: 5 # type: ignore + ) + + with pytest.raises(AssertionError): + tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "5"] + ) + + def test_optional_subparser(): @dataclasses.dataclass class OptionalHTTPServer: diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 551ae0f4f..fddbbf699 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -354,7 +354,13 @@ def from_field( if default_name is None: default_name = subcommand_name_from_type.get(type(field.default), None) - assert default_name is not None + if default_name is None: + raise AssertionError( + f"`{prefix}` was provided a default value of type" + f" {type(field.default)} but no matching subcommand was found. A" + " type may be missing in the Union type declaration for" + f" `{prefix}`, which is currently set to {field.typ}." + ) # Add subcommands for each option. parser_from_name: Dict[str, ParserSpecification] = {} From 64fda6fa7732ed2ab79016a86a797bfcf98a0b45 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 7 Nov 2022 05:58:25 -0800 Subject: [PATCH 222/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9fbf15510..f6a30583c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.29" +version = "0.3.30" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] From 2634912000b41f449c0502ebd1502d8616eda1f7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 9 Nov 2022 19:34:59 -0800 Subject: [PATCH 223/491] Fix marker edge cases, tests --- README.md | 9 ++++++--- docs/source/index.md | 9 ++++++--- tests/test_conf.py | 16 +++++++++++++--- tests/test_positional_ignore_py37.py | 6 +++--- tyro/_arguments.py | 1 + tyro/_fields.py | 7 ++----- tyro/_parsers.py | 7 ++++++- 7 files changed, 37 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 15c890112..b6eec5f7e 100644 --- a/README.md +++ b/README.md @@ -129,9 +129,12 @@ For more examples, see our [documentation](https://brentyi.github.io/tyro). Standard Python type annotations, docstrings, and default values are parsed to automatically generate command-line interfaces with informative helptext. - If you're familiar with type annotations and docstrings in Python, you - already know how to use `tyro`! If you're not, learning to use `tyro` reduces - to learning to write modern Python. + `tyro` works seamlessly with tools you already use: examples are included for + [`dataclasses`](https://docs.python.org/3/library/dataclasses.html), + [`attrs`](https://www.attrs.org/), + [`pydantic`](https://pydantic-docs.helpmanual.io/), + [`flax.linen`](https://flax.readthedocs.io/en/latest/api_reference/flax.linen.html), + and more. Hate `tyro`? Just remove one line of code, and you're left with beautiful, type-annotated, and documented vanilla Python that can be used with a range diff --git a/docs/source/index.md b/docs/source/index.md index 1c5b8e18f..e3261aee0 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -24,9 +24,12 @@ To get started, we recommend browsing the examples to the left. Standard Python type annotations, docstrings, and default values are parsed to automatically generate command-line interfaces with informative helptext. - If you're familiar with type annotations and docstrings in Python, you - already know how to use `tyro`! If you're not, learning to use `tyro` reduces - to learning to write modern Python. + `tyro` works seamlessly with tools you already use: examples are included for + [dataclasses](https://docs.python.org/3/library/dataclasses.html), + [attrs](https://www.attrs.org/), + [pydantic](https://pydantic-docs.helpmanual.io/), + [flax.linen](https://flax.readthedocs.io/en/latest/api_reference/flax.linen.html), + and more. Hate `tyro`? Just remove one line of code, and you're left with beautiful, type-annotated, and documented vanilla Python that can be used with a range diff --git a/tests/test_conf.py b/tests/test_conf.py index 5ba185ea5..c49e4a6e9 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -12,6 +12,7 @@ def test_omit_subcommand_prefix(): @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 + flag: bool = True @dataclasses.dataclass class DefaultInstanceSMTPServer: @@ -28,14 +29,23 @@ class DefaultInstanceSubparser: assert ( tyro.cli( DefaultInstanceSubparser, - args=["--x", "1", "bc:default-instance-http-server", "--y", "5"], + args=[ + "--x", + "1", + "bc:default-instance-http-server", + "--y", + "5", + "--no-flag", + ], ) == tyro.cli( DefaultInstanceSubparser, args=["--x", "1", "bc:default-instance-http-server", "--y", "5"], - default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), + default=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=3, flag=False) + ), ) - == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5, flag=False)) ) assert ( tyro.cli( diff --git a/tests/test_positional_ignore_py37.py b/tests/test_positional_ignore_py37.py index b19c6b3ea..91c87731c 100644 --- a/tests/test_positional_ignore_py37.py +++ b/tests/test_positional_ignore_py37.py @@ -39,10 +39,10 @@ def __init__(self, a: int, hello_world: int, /, c: int): def nest1(a: int, b: int, thing: A, /, c: int) -> A: return thing - assert isinstance(tyro.cli(nest1, args="0 1 2 3 --thing.c 4 --c 4".split(" ")), A) - assert tyro.cli(nest1, args="0 1 2 3 --thing.c 4 --c 4".split(" ")).hello_world == 3 + assert isinstance(tyro.cli(nest1, args="0 1 2 3 4 --c 4".split(" ")), A) + assert tyro.cli(nest1, args="0 1 2 3 4 --c 4".split(" ")).hello_world == 3 with pytest.raises(SystemExit): - tyro.cli(nest1, args="0 1 2 3 4 --thing.c 4 --c 4".split(" ")) + tyro.cli(nest1, args="0 1 2 3 4 4 --c 4".split(" ")) def test_nested_positional_alt(): diff --git a/tyro/_arguments.py b/tyro/_arguments.py index cc48d42d9..ff8e1314f 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -170,6 +170,7 @@ def _rule_handle_boolean_flags( arg.field.default in _fields.MISSING_SINGLETONS or arg.field.is_positional() or _markers.FlagConversionOff in arg.field.markers + or _markers.Fixed in arg.field.markers ): # Treat bools as a normal parameter. return lowered diff --git a/tyro/_fields.py b/tyro/_fields.py index 689debf88..5bbc1169b 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -27,7 +27,7 @@ import docstring_parser import typing_extensions -from typing_extensions import Annotated, get_args, get_type_hints, is_typeddict +from typing_extensions import get_args, get_type_hints, is_typeddict from . import conf # Avoid circular import. from . import _docstrings, _instantiators, _resolver, _singleton, _strings @@ -75,7 +75,7 @@ def make( markers: Tuple[_markers.Marker, ...] = (), name_override: Optional[Any] = None, ): - _, inferred_markers = _resolver.unwrap_annotated(typ, _markers.Marker) + typ, inferred_markers = _resolver.unwrap_annotated(typ, _markers.Marker) return FieldDefinition( name, typ, @@ -86,11 +86,8 @@ def make( ) def add_markers(self, markers: Tuple[_markers.Marker, ...]) -> FieldDefinition: - if len(markers) == 0: - return self return dataclasses.replace( self, - typ=Annotated.__class_getitem__((self.typ,) + markers), # type: ignore markers=self.markers.union(markers), ) diff --git a/tyro/_parsers.py b/tyro/_parsers.py index fddbbf699..ff66002ce 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -127,7 +127,12 @@ def from_callable_or_type( ), ) nested_parser = ParserSpecification.from_callable_or_type( - field.typ, + # Recursively apply marker types. + field.typ + if len(field.markers) == 0 + else Annotated.__class_getitem__( # type: ignore + (field.typ,) + tuple(field.markers) + ), description=None, parent_classes=parent_classes, default_instance=field.default, From f024f92f616de72f873e41e1f699a5ea6ef05f15 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 9 Nov 2022 19:37:35 -0800 Subject: [PATCH 224/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f6a30583c..7e06d8b63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.30" +version = "0.3.31" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] From cd1589a549dce69e5e10f4db6d484f5114212066 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 11 Nov 2022 10:16:28 -0800 Subject: [PATCH 225/491] Add `tyro.conf.arg()`, support "help" metadata --- pyproject.toml | 2 +- tests/test_conf.py | 67 ++++++++++++++++++- tests/test_helptext.py | 44 ------------ tyro/_arguments.py | 10 +++ tyro/_calling.py | 4 +- tyro/_fields.py | 38 ++++++++--- tyro/_parsers.py | 8 +-- tyro/_strings.py | 4 +- tyro/conf/__init__.py | 5 +- tyro/conf/{_subcommands.py => _confstruct.py} | 26 ++++++- tyro/conf/_markers.py | 6 +- 11 files changed, 145 insertions(+), 69 deletions(-) rename tyro/conf/{_subcommands.py => _confstruct.py} (70%) diff --git a/pyproject.toml b/pyproject.toml index 7e06d8b63..abe431b05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.31" +version = "0.3.32" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] diff --git a/tests/test_conf.py b/tests/test_conf.py index c49e4a6e9..1a8b7d551 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Generic, TypeVar, Union +from typing import Any, Callable, Generic, TypeVar, Union import pytest from helptext_utils import get_helptext @@ -462,3 +462,68 @@ def main( return value + inner.a + inner.b assert tyro.cli(main, args=["--value", "5"]) == 8 + + +def test_suppressed(): + @dataclasses.dataclass + class Struct: + a: int = 5 + b: tyro.conf.Suppress[str] = "7" + + def main(x: Any = Struct()): + pass + + helptext = get_helptext(main) + assert "--x.a" in helptext + assert "--x.b" not in helptext + + +def test_suppress_manual_fixed(): + @dataclasses.dataclass + class Struct: + a: int = 5 + b: tyro.conf.SuppressFixed[tyro.conf.Fixed[str]] = "7" + + def main(x: Any = Struct()): + pass + + helptext = get_helptext(main) + assert "--x.a" in helptext + assert "--x.b" not in helptext + + +def test_suppress_auto_fixed(): + @dataclasses.dataclass + class Struct: + a: int = 5 + b: Callable = lambda x: 5 + + def main(x: tyro.conf.SuppressFixed[Any] = Struct()): + pass + + helptext = get_helptext(main) + assert "--x.a" in helptext + assert "--x.b" not in helptext + + +def test_argconf_help(): + @dataclasses.dataclass + class Struct: + a: Annotated[ + int, tyro.conf.arg(name="nice", help="Hello world", metavar="NUMBER") + ] = 5 + b: tyro.conf.Suppress[str] = "7" + + def main(x: Any = Struct()) -> int: + return x.a + + helptext = get_helptext(main) + assert "Hello world" in helptext + assert "INT" not in helptext + assert "NUMBER" in helptext + assert "--x.a" not in helptext + assert "--x.nice" in helptext + assert "--x.b" not in helptext + + assert tyro.cli(main, args=[]) == 5 + assert tyro.cli(main, args=["--x.nice", "3"]) == 3 diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 20ef6daf1..c572746f4 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -8,8 +8,6 @@ from helptext_utils import get_helptext from typing_extensions import Annotated, Literal -import tyro - def test_helptext(): @dataclasses.dataclass @@ -566,45 +564,3 @@ def main2(x: Callable = nn.ReLU): assert "--x {fixed}" in helptext assert "(fixed to:" in helptext assert "torch" in helptext - - -def test_suppressed(): - @dataclasses.dataclass - class Struct: - a: int = 5 - b: tyro.conf.Suppress[str] = "7" - - def main(x: Any = Struct()): - pass - - helptext = get_helptext(main) - assert "--x.a" in helptext - assert "--x.b" not in helptext - - -def test_suppress_manual_fixed(): - @dataclasses.dataclass - class Struct: - a: int = 5 - b: tyro.conf.SuppressFixed[tyro.conf.Fixed[str]] = "7" - - def main(x: Any = Struct()): - pass - - helptext = get_helptext(main) - assert "--x.a" in helptext - assert "--x.b" not in helptext - - -def test_suppress_auto_fixed(): - @dataclasses.dataclass - class Struct: - a: int = 5 - b: Callable = lambda x: 5 - - def main(x: tyro.conf.SuppressFixed[Any] = Struct()): - pass - - helptext = get_helptext(main) - assert "--x.a" in helptext - assert "--x.b" not in helptext diff --git a/tyro/_arguments.py b/tyro/_arguments.py index ff8e1314f..a8284894a 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -63,6 +63,16 @@ def add_argument( # the field default to a string format, then back to the desired type. kwargs["default"] = _fields.MISSING_NONPROP + # Apply overrides in our arg configuration object. + # Note that the `name` field is applied when the field object is instantiated! + kwargs.update( + { + k: v + for k, v in vars(self.field.argconf).items() + if v is not None and k != "name" + } + ) + # Add argument! Note that the name must be passed in as a position argument. arg = parser.add_argument(name_or_flag, **kwargs) diff --git a/tyro/_calling.py b/tyro/_calling.py index 8bfba395f..a71fff45f 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -166,9 +166,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: if field.is_positional(): args.append(value) else: - kwargs[ - field.name if field.name_override is None else field.name_override - ] = value + kwargs[field.call_argname] = value # Note: we unwrap types both before and after narrowing. This is because narrowing # sometimes produces types like `Tuple[T1, T2, ...]`, where we actually want just diff --git a/tyro/_fields.py b/tyro/_fields.py index 5bbc1169b..a0fc68cd9 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -31,7 +31,7 @@ from . import conf # Avoid circular import. from . import _docstrings, _instantiators, _resolver, _singleton, _strings -from .conf import _markers +from .conf import _confstruct, _markers # Support attrs and pydantic if they're installed. try: @@ -52,10 +52,12 @@ class FieldDefinition: helptext: Optional[str] markers: FrozenSet[_markers.Marker] + argconf: _confstruct._ArgConfiguration + # Override the name in our kwargs. Currently only used for dictionary types when # the key values aren't strings, but in the future could be used whenever the # user-facing argument name doesn't match the keyword expected by our callable. - name_override: Optional[Any] + call_argname: Any def __post_init__(self): if ( @@ -71,18 +73,27 @@ def make( typ: Type, default: Any, helptext: Optional[str], + call_argname_override: Optional[Any] = None, *, markers: Tuple[_markers.Marker, ...] = (), - name_override: Optional[Any] = None, ): + # Try to extract argconf overrides from type. + _, argconfs = _resolver.unwrap_annotated(typ, _confstruct._ArgConfiguration) + if len(argconfs) == 0: + argconf = _confstruct._ArgConfiguration(None, None, None) + else: + assert len(argconfs) == 1 + (argconf,) = argconfs + typ, inferred_markers = _resolver.unwrap_annotated(typ, _markers.Marker) return FieldDefinition( - name, + name if argconf.name is None else argconf.name, typ, default, helptext, frozenset(inferred_markers).union(markers), - name_override, + argconf, + call_argname_override if call_argname_override is not None else name, ) def add_markers(self, markers: Tuple[_markers.Marker, ...]) -> FieldDefinition: @@ -201,7 +212,7 @@ def _try_field_list_from_callable( default_instance: _DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: f, found_subcommand_configs = _resolver.unwrap_annotated( - f, conf._subcommands._SubcommandConfiguration + f, conf._confstruct._SubcommandConfiguration ) if len(found_subcommand_configs) > 0: default_instance = found_subcommand_configs[0].default @@ -362,12 +373,23 @@ def _try_field_list_from_dataclass( field_list = [] for dc_field in filter(lambda field: field.init, _resolver.resolved_fields(cls)): default = _get_dataclass_field_default(dc_field, default_instance) + + # Try to get helptext from field metadata. This is also intended to be + # compatible with HuggingFace-style config objects. + helptext = dc_field.metadata.get("help", None) + assert isinstance(helptext, (str, type(None))) + + # Try to get helptext from docstrings. Note that this can't be generated + # dynamically. + if helptext is None: + helptext = _docstrings.get_field_docstring(cls, dc_field.name) + field_list.append( FieldDefinition.make( name=dc_field.name, typ=dc_field.type, default=default, - helptext=_docstrings.get_field_docstring(cls, dc_field.name), + helptext=helptext, ) ) return field_list @@ -542,7 +564,7 @@ def _try_field_list_from_dict( default=v, helptext=None, # Dictionary specific key: - name_override=k, + call_argname_override=k, ) ) return field_list diff --git a/tyro/_parsers.py b/tyro/_parsers.py index ff66002ce..8feaab232 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -18,7 +18,7 @@ _resolver, _strings, ) -from .conf import _markers, _subcommands +from .conf import _confstruct, _markers T = TypeVar("T") @@ -294,14 +294,14 @@ def from_field( # Get subcommand configurations from `tyro.conf.subcommand()`. subcommand_config_from_name: Dict[ - str, _subcommands._SubcommandConfiguration + str, _confstruct._SubcommandConfiguration ] = {} subcommand_name_from_default_hash: Dict[int, str] = {} subcommand_name_from_type: Dict[Type, str] = {} # Used for default matching. for option in options_no_none: subcommand_name = _strings.subparser_name_from_type(prefix, option) option, found_subcommand_configs = _resolver.unwrap_annotated( - option, _subcommands._SubcommandConfiguration + option, _confstruct._SubcommandConfiguration ) default_hash = None if len(found_subcommand_configs) != 0: @@ -378,7 +378,7 @@ def from_field( if subcommand_name in subcommand_config_from_name: subcommand_config = subcommand_config_from_name[subcommand_name] else: - subcommand_config = _subcommands._SubcommandConfiguration( + subcommand_config = _confstruct._SubcommandConfiguration( "unused", description=None, default=_fields.MISSING_NONPROP, diff --git a/tyro/_strings.py b/tyro/_strings.py index 7d94eb1f1..8b31fc503 100644 --- a/tyro/_strings.py +++ b/tyro/_strings.py @@ -60,11 +60,11 @@ def hyphen_separated_from_camel_case(name: str) -> str: def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: - from .conf import _subcommands # Prevent circular imports + from .conf import _confstruct # Prevent circular imports cls, type_from_typevar = _resolver.resolve_generic_types(cls) cls, found_subcommand_configs = _resolver.unwrap_annotated( - cls, _subcommands._SubcommandConfiguration + cls, _confstruct._SubcommandConfiguration ) # Subparser name from `tyro.metadata.subcommand()`. diff --git a/tyro/conf/__init__.py b/tyro/conf/__init__.py index 1833f784d..9a84b819b 100644 --- a/tyro/conf/__init__.py +++ b/tyro/conf/__init__.py @@ -8,6 +8,7 @@ Features here are supported, but generally unnecessary and should be used sparingly. """ +from ._confstruct import arg, subcommand from ._markers import ( AvoidSubcommands, Fixed, @@ -17,9 +18,10 @@ Suppress, SuppressFixed, ) -from ._subcommands import subcommand __all__ = [ + "arg", + "subcommand", "AvoidSubcommands", "Fixed", "FlagConversionOff", @@ -27,5 +29,4 @@ "Positional", "Suppress", "SuppressFixed", - "subcommand", ] diff --git a/tyro/conf/_subcommands.py b/tyro/conf/_confstruct.py similarity index 70% rename from tyro/conf/_subcommands.py rename to tyro/conf/_confstruct.py index 63b3d5d7e..0bfeaa05a 100644 --- a/tyro/conf/_subcommands.py +++ b/tyro/conf/_confstruct.py @@ -23,7 +23,7 @@ def subcommand( prefix_name: bool = True, ) -> Any: """Returns a metadata object for configuring subcommands with `typing.Annotated`. - This is useful but can make code harder to read, so usage is discouraged. + Useful for aesthetics. Consider the standard approach for creating subcommands: @@ -53,3 +53,27 @@ def subcommand( ``` """ return _SubcommandConfiguration(name, default, description, prefix_name) + + +@dataclasses.dataclass(frozen=True) +class _ArgConfiguration: + name: Optional[str] + metavar: Optional[str] + help: Optional[str] + + +def arg( + *, + name: Optional[str] = None, + metavar: Optional[str] = None, + help: Optional[str] = None, +) -> Any: + """Returns a metadata object for configuring arguments with `typing.Annotated`. + Useful for aesthetics. + + Usage: + ```python + x: Annotated[int, tyro.conf.arg(...)] + ``` + """ + return _ArgConfiguration(name=name, metavar=metavar, help=help) diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index 1201ec859..9bd6c6498 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -52,10 +52,10 @@ If we have a structure with the field: - cmd: Union[Commit, Checkout] + cmd: Union[NestedTypeA, NestedTypeB] -By default, --cmd.branch may be generated as a flag for each dataclass in the union. -If subcommand prefixes are omitted, we would instead simply have --branch. +By default, `--cmd.arg` may be generated as a flag for each dataclass in the union. +If subcommand prefixes are omitted, we would instead simply have `--arg`. """ From 7b3a70aceedd514cfe6531103cc4ccc82acd2e83 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 11 Nov 2022 10:24:48 -0800 Subject: [PATCH 226/491] Comment nit --- tyro/_fields.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tyro/_fields.py b/tyro/_fields.py index a0fc68cd9..3af08078f 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -54,9 +54,8 @@ class FieldDefinition: argconf: _confstruct._ArgConfiguration - # Override the name in our kwargs. Currently only used for dictionary types when - # the key values aren't strings, but in the future could be used whenever the - # user-facing argument name doesn't match the keyword expected by our callable. + # Override the name in our kwargs. Useful whenever the user-facing argument name + # doesn't match the keyword expected by our callable. call_argname: Any def __post_init__(self): From 77770cc63ba46c1005eb377efb6e3b8ae727fa24 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 11 Nov 2022 16:43:24 -0800 Subject: [PATCH 227/491] Fix positional arg corner case, help formatting for `tyro.conf.arg()` --- examples/04_additional/06_conf.py | 17 ++++++++++++++--- pyproject.toml | 2 +- tests/test_conf.py | 16 ++++++++++++++++ tyro/_arguments.py | 17 ++++++----------- tyro/_calling.py | 2 +- tyro/_fields.py | 13 ++++++++++++- tyro/conf/_markers.py | 4 ++++ 7 files changed, 54 insertions(+), 17 deletions(-) diff --git a/examples/04_additional/06_conf.py b/examples/04_additional/06_conf.py index 7d7ce518c..05d6ca279 100644 --- a/examples/04_additional/06_conf.py +++ b/examples/04_additional/06_conf.py @@ -7,6 +7,7 @@ Usage: `python ./06_conf.py --help` +`python ./06_conf.py 5 --boolean True` """ import dataclasses @@ -34,15 +35,25 @@ class CommitArgs: @dataclasses.dataclass class Args: - # A boolean field with flag conversion turned off. - boolean: tyro.conf.FlagConversionOff[bool] = False - # A numeric field parsed as a positional argument. positional: tyro.conf.Positional[int] = 3 + # A boolean field with flag conversion turned off. + boolean: tyro.conf.FlagConversionOff[bool] = False + # A numeric field that can't be changed via the CLI. fixed: tyro.conf.Fixed[int] = 5 + # A field with manually overridden properties. + manual: Annotated[ + str, + tyro.conf.arg( + name="renamed", + metavar="STRING", + help="A field with manually overridden properties!", + ), + ] = "Hello" + # A union over nested structures, but without subcommand generation. When a default # is provided, the type is simply fixed to that default. union_without_subcommand: tyro.conf.AvoidSubcommands[ diff --git a/pyproject.toml b/pyproject.toml index abe431b05..7343a9877 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.32" +version = "0.3.33" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] diff --git a/tests/test_conf.py b/tests/test_conf.py index 1a8b7d551..fd0eb0cdd 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -527,3 +527,19 @@ def main(x: Any = Struct()) -> int: assert tyro.cli(main, args=[]) == 5 assert tyro.cli(main, args=["--x.nice", "3"]) == 3 + + +def test_positional(): + def main(x: tyro.conf.Positional[int], y: int) -> int: + return x + y + + assert tyro.cli(main, args="5 --y 3".split(" ")) == 8 + assert tyro.cli(main, args="--y 3 5".split(" ")) == 8 + + +def test_positional_order_swap(): + def main(x: int, y: tyro.conf.Positional[int]) -> int: + return x + y + + assert tyro.cli(main, args="5 --x 3".split(" ")) == 8 + assert tyro.cli(main, args="--x 3 5".split(" ")) == 8 diff --git a/tyro/_arguments.py b/tyro/_arguments.py index a8284894a..f381f3020 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -65,13 +65,8 @@ def add_argument( # Apply overrides in our arg configuration object. # Note that the `name` field is applied when the field object is instantiated! - kwargs.update( - { - k: v - for k, v in vars(self.field.argconf).items() - if v is not None and k != "name" - } - ) + if self.field.argconf.metavar is not None: + kwargs["metavar"] = self.field.argconf.metavar # Add argument! Note that the name must be passed in as a position argument. arg = parser.add_argument(name_or_flag, **kwargs) @@ -314,13 +309,13 @@ def _rule_generate_helptext( help_parts = [] - docstring_help = arg.field.helptext + primary_help = arg.field.helptext - if docstring_help is not None and docstring_help != "": + if primary_help is not None and primary_help != "": # Note that the percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string - docstring_help = docstring_help.replace("%", "%%") - help_parts.append(_rich_tag_if_enabled(docstring_help, "helptext")) + primary_help = primary_help.replace("%", "%%") + help_parts.append(_rich_tag_if_enabled(primary_help, "helptext")) default = lowered.default if lowered.is_fixed(): diff --git a/tyro/_calling.py b/tyro/_calling.py index a71fff45f..36b660246 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -163,7 +163,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: consumed_keywords |= consumed_keywords_child if value is not _fields.EXCLUDE_FROM_CALL: - if field.is_positional(): + if field.is_positional_call(): args.append(value) else: kwargs[field.call_argname] = value diff --git a/tyro/_fields.py b/tyro/_fields.py index 3af08078f..f96d7367c 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -83,6 +83,7 @@ def make( else: assert len(argconfs) == 1 (argconf,) = argconfs + helptext = argconf.help typ, inferred_markers = _resolver.unwrap_annotated(typ, _markers.Marker) return FieldDefinition( @@ -102,6 +103,7 @@ def add_markers(self, markers: Tuple[_markers.Marker, ...]) -> FieldDefinition: ) def is_positional(self) -> bool: + """Returns True if the argument should be positional in the commandline.""" return ( # Explicit positionals. _markers.Positional in self.markers @@ -109,6 +111,15 @@ def is_positional(self) -> bool: or self.name == _strings.dummy_field_name ) + def is_positional_call(self) -> bool: + """Returns True if the argument should be positional in underlying Python call.""" + return ( + # Explicit positionals. + _markers._PositionalCall in self.markers + # Dummy dataclasses should have a single positional field. + or self.name == _strings.dummy_field_name + ) + class PropagatingMissingType(_singleton.Singleton): pass @@ -646,7 +657,7 @@ def _field_list_from_params( typ=hints[param.name], default=default, helptext=helptext, - markers=(_markers.Positional,) + markers=(_markers.Positional, _markers._PositionalCall) if param.kind is inspect.Parameter.POSITIONAL_ONLY else (), ) diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index 9bd6c6498..d267ee432 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -18,6 +18,10 @@ """A type `T` can be annotated as `Positional[T]` if we want to parse it as a positional argument.""" +# Private marker. For when an argument is not only positional in the CLI, but also in +# the callable. +_PositionalCall = Annotated[T, None] + # TODO: the verb tenses here are inconsistent, naming could be revisited. # Perhaps Suppress should be Suppressed? But SuppressedFixed would be weird. From 84ce6609d4429a0b241275d7fec94f9d17e5666f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 12 Nov 2022 04:31:18 -0800 Subject: [PATCH 228/491] Docs sync --- .../source/examples/04_additional/06_conf.rst | 24 ++++++++++++++++--- examples/04_additional/06_conf.py | 2 +- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/source/examples/04_additional/06_conf.rst b/docs/source/examples/04_additional/06_conf.rst index 1e99a8a11..83c36839c 100644 --- a/docs/source/examples/04_additional/06_conf.rst +++ b/docs/source/examples/04_additional/06_conf.rst @@ -41,15 +41,25 @@ Features here are supported, but generally unnecessary and should be used sparin @dataclasses.dataclass class Args: + # A numeric field parsed as a positional argument. + positional: tyro.conf.Positional[int] + # A boolean field with flag conversion turned off. boolean: tyro.conf.FlagConversionOff[bool] = False - # A numeric field parsed as a positional argument. - positional: tyro.conf.Positional[int] = 3 - # A numeric field that can't be changed via the CLI. fixed: tyro.conf.Fixed[int] = 5 + # A field with manually overridden properties. + manual: Annotated[ + str, + tyro.conf.arg( + name="renamed", + metavar="STRING", + help="A field with manually overridden properties!", + ), + ] = "Hello" + # A union over nested structures, but without subcommand generation. When a default # is provided, the type is simply fixed to that default. union_without_subcommand: tyro.conf.AvoidSubcommands[ @@ -76,3 +86,11 @@ Features here are supported, but generally unnecessary and should be used sparin python 04_additional/06_conf.py --help .. program-output:: python ../../examples/04_additional/06_conf.py --help + +------------ + +.. raw:: html + + python 04_additional/06_conf.py 5 --boolean True + +.. program-output:: python ../../examples/04_additional/06_conf.py 5 --boolean True diff --git a/examples/04_additional/06_conf.py b/examples/04_additional/06_conf.py index 05d6ca279..5d00945e6 100644 --- a/examples/04_additional/06_conf.py +++ b/examples/04_additional/06_conf.py @@ -36,7 +36,7 @@ class CommitArgs: @dataclasses.dataclass class Args: # A numeric field parsed as a positional argument. - positional: tyro.conf.Positional[int] = 3 + positional: tyro.conf.Positional[int] # A boolean field with flag conversion turned off. boolean: tyro.conf.FlagConversionOff[bool] = False From 94b490126c324cf245f702c4ef4f257ac5a83dbd Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 19 Nov 2022 03:02:00 -0800 Subject: [PATCH 229/491] Rewrite subcommand internals --- tyro/_calling.py | 9 ++- tyro/_cli.py | 16 ++++ tyro/_parsers.py | 161 ++++++++++++++++++--------------------- tyro/conf/_confstruct.py | 1 + 4 files changed, 96 insertions(+), 91 deletions(-) diff --git a/tyro/_calling.py b/tyro/_calling.py index 36b660246..a390d1e45 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -24,6 +24,7 @@ def call_from_args( default_instance: Union[T, _fields.NonpropagatingMissingType], value_from_prefixed_field_name: Dict[str, Any], field_name_prefix: str, + subparser_def_from_prefixed_field_name: Dict[str, _parsers.SubparsersSpecification], ) -> Tuple[T, Set[str]]: """Call `f` with arguments specified by a dictionary of values from argparse. @@ -103,14 +104,13 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: field.default, value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, + subparser_def_from_prefixed_field_name=subparser_def_from_prefixed_field_name, ) consumed_keywords |= consumed_keywords_child else: # Unions over dataclasses (subparsers). This is the only other option. - assert len(parser_definition.subparsers_from_name) > 0 - assert prefixed_field_name in parser_definition.subparsers_from_name - - subparser_def = parser_definition.subparsers_from_name[prefixed_field_name] + assert parser_definition.subparsers is not None + subparser_def = subparser_def_from_prefixed_field_name[prefixed_field_name] subparser_dest = _strings.make_subparser_dest(name=prefixed_field_name) consumed_keywords.add(subparser_dest) @@ -159,6 +159,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: field.default if type(field.default) is chosen_f else None, value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, + subparser_def_from_prefixed_field_name=subparser_def_from_prefixed_field_name, ) consumed_keywords |= consumed_keywords_child diff --git a/tyro/_cli.py b/tyro/_cli.py index 88d2220d4..2398fc15e 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -320,6 +320,21 @@ def fix_arg(arg: str) -> str: for k, v in value_from_prefixed_field_name.items() } + # Build a map from subparser names to definitions. + subparser_def_from_prefixed_field_name = {} + + def _cache_subparsers(parser_definition: _parsers.ParserSpecification) -> None: + subparsers = parser_definition.subparsers + if subparsers is None: + return + subparser_def_from_prefixed_field_name[ + subparsers.prefix if subparsers.prefix != _strings.dummy_field_name else "" + ] = subparsers + for p in subparsers.parser_from_name.values(): + _cache_subparsers(p) + + _cache_subparsers(parser_definition) + try: # Attempt to call `f` using whatever was passed in. out, consumed_keywords = _calling.call_from_args( @@ -328,6 +343,7 @@ def fix_arg(arg: str) -> str: default_instance_internal, value_from_prefixed_field_name, field_name_prefix="", + subparser_def_from_prefixed_field_name=subparser_def_from_prefixed_field_name, ) except _calling.InstantiationError as e: # Emulate argparse's error behavior when invalid arguments are passed in. diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 8feaab232..e88db8865 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -4,7 +4,6 @@ import argparse import dataclasses -import itertools from typing import Any, Callable, Dict, List, Optional, Set, Type, TypeVar, Union, cast from typing_extensions import Annotated, get_args, get_origin @@ -31,7 +30,7 @@ class ParserSpecification: description: str args: List[_arguments.ArgumentDefinition] helptext_from_nested_class_field_name: Dict[str, Optional[str]] - subparsers_from_name: Dict[str, SubparsersSpecification] + subparsers: Optional[SubparsersSpecification] prefix: str has_required_args: bool @@ -69,7 +68,7 @@ def from_callable_or_type( has_required_args = False args = [] helptext_from_nested_class_field_name = {} - subparsers_from_name = {} + subparsers = None field_list = _fields.field_list_from_callable( f=f, default_instance=default_instance @@ -111,10 +110,13 @@ def from_callable_or_type( ): # Don't make a subparser. field = dataclasses.replace(field, typ=type(field.default)) + elif subparsers is None: + subparsers = subparsers_attempt + continue else: - subparsers_from_name[ - _strings.make_field_name([prefix, subparsers_attempt.name]) - ] = subparsers_attempt + subparsers = subparsers.add_subparsers_to_leaves( + subparsers_attempt + ) continue # (2) Handle nested callables. @@ -144,7 +146,14 @@ def from_callable_or_type( args.extend(nested_parser.args) # Include nested subparsers. - subparsers_from_name.update(nested_parser.subparsers_from_name) + if nested_parser.subparsers is not None: + subparsers = ( + nested_parser.subparsers + if subparsers is None + else subparsers.add_subparsers_to_leaves( + nested_parser.subparsers + ) + ) # Include nested strings. for ( @@ -176,15 +185,6 @@ def from_callable_or_type( if arg.lowered.required: has_required_args = True - # If a later subparser is required, all previous ones should be as well. - subparsers_required = False - for name, subparsers in list(subparsers_from_name.items())[::-1]: - if subparsers.required: - subparsers_required = True - subparsers_from_name[name] = dataclasses.replace( - subparsers, required=subparsers_required - ) - return ParserSpecification( f=f, description=_strings.remove_single_line_breaks( @@ -194,7 +194,7 @@ def from_callable_or_type( ), args=args, helptext_from_nested_class_field_name=helptext_from_nested_class_field_name, - subparsers_from_name=subparsers_from_name, + subparsers=subparsers, prefix=prefix, has_required_args=has_required_args, ) @@ -202,6 +202,17 @@ def from_callable_or_type( def apply(self, parser: argparse.ArgumentParser) -> None: """Create defined arguments and subparsers.""" + # Generate helptext. + parser.description = self.description + self.apply_args(parser) + + # Create subparser tree. + if self.subparsers is not None: + self.subparsers.apply(parser) + + def apply_args(self, parser: argparse.ArgumentParser) -> None: + """Create defined arguments and subparsers.""" + # Generate helptext. parser.description = self.description @@ -245,14 +256,6 @@ def format_group_name(nested_field_name: str) -> str: assert arg.lowered.help is argparse.SUPPRESS arg.add_argument(group_from_prefix[""]) - # Create subparser tree. - if len(self.subparsers_from_name) > 0: - prev_subparser_tree_nodes = [parser] # Root node. - for subparsers in self.subparsers_from_name.values(): - prev_subparser_tree_nodes = subparsers.apply( - self, prev_subparser_tree_nodes - ) - @dataclasses.dataclass(frozen=True) class SubparsersSpecification: @@ -422,11 +425,9 @@ def from_field( default_parser = parser_from_name[default_name] if any(map(lambda arg: arg.lowered.required, default_parser.args)): required = True - if any( - map( - lambda subparsers: subparsers.required, - default_parser.subparsers_from_name.values(), - ) + if ( + default_parser.subparsers is not None + and default_parser.subparsers.required ): required = True @@ -459,11 +460,7 @@ def from_field( can_be_none=options != options_no_none, ) - def apply( - self, - parent_parser: ParserSpecification, - prev_subparser_tree_nodes: List[argparse.ArgumentParser], - ) -> List[argparse.ArgumentParser]: + def apply(self, parent_parser: argparse.ArgumentParser) -> None: title = "subcommands" metavar = ( "{" @@ -481,58 +478,48 @@ def apply( title = "optional " + title metavar = f"[{metavar}]" - subparser_tree_nodes: List[argparse.ArgumentParser] = [] - for p in prev_subparser_tree_nodes: - # Add subparsers to every node in previous level of the tree. - argparse_subparsers = p.add_subparsers( - dest=_strings.make_subparser_dest(self.prefix), - description=self.description, - required=self.required, - title=title, - metavar=metavar, - ) - - if self.can_be_none: - subparser = argparse_subparsers.add_parser( - name=_strings.subparser_name_from_type(self.prefix, None), - formatter_class=_argparse_formatter.DcargsArgparseHelpFormatter, - help="", - ) - subparser_tree_nodes.append(subparser) - - for name, subparser_def in self.parser_from_name.items(): - helptext = subparser_def.description.replace("%", "%%") - if len(helptext) > 0: - # TODO: calling a private function here. - helptext = _arguments._rich_tag_if_enabled( - helptext.strip(), "helptext" - ) - - subparser = argparse_subparsers.add_parser( - name, - formatter_class=_argparse_formatter.DcargsArgparseHelpFormatter, - help=helptext, - ) - subparser_def.apply(subparser) + # Add subparsers to every node in previous level of the tree. + argparse_subparsers = parent_parser.add_subparsers( + dest=_strings.make_subparser_dest(self.prefix), + description=self.description, + required=self.required, + title=title, + metavar=metavar, + ) - def _get_leaf_subparsers( - node: argparse.ArgumentParser, - ) -> List[argparse.ArgumentParser]: - if node._subparsers is None: - return [node] - else: - # Magic! - return list( - itertools.chain( - *map( - _get_leaf_subparsers, - node._subparsers._actions[ - -1 - ]._name_parser_map.values(), # type: ignore - ) - ) - ) + if self.can_be_none: + subparser = argparse_subparsers.add_parser( + name=_strings.subparser_name_from_type(self.prefix, None), + formatter_class=_argparse_formatter.DcargsArgparseHelpFormatter, + help="", + ) - subparser_tree_nodes.extend(_get_leaf_subparsers(subparser)) + for name, subparser_def in self.parser_from_name.items(): + helptext = subparser_def.description.replace("%", "%%") + if len(helptext) > 0: + # TODO: calling a private function here. + helptext = _arguments._rich_tag_if_enabled(helptext.strip(), "helptext") - return subparser_tree_nodes + subparser = argparse_subparsers.add_parser( + name, + formatter_class=_argparse_formatter.DcargsArgparseHelpFormatter, + help=helptext, + ) + subparser_def.apply(subparser) + + def add_subparsers_to_leaves( + self, subparsers: SubparsersSpecification + ) -> SubparsersSpecification: + new_parsers_from_name = {} + for name, parser in self.parser_from_name.items(): + new_parsers_from_name[name] = dataclasses.replace( + parser, + subparsers=subparsers + if parser.subparsers is None + else parser.subparsers.add_subparsers_to_leaves(subparsers), + ) + return dataclasses.replace( + self, + parser_from_name=new_parsers_from_name, + required=self.required or subparsers.required, + ) diff --git a/tyro/conf/_confstruct.py b/tyro/conf/_confstruct.py index 0bfeaa05a..887587dec 100644 --- a/tyro/conf/_confstruct.py +++ b/tyro/conf/_confstruct.py @@ -60,6 +60,7 @@ class _ArgConfiguration: name: Optional[str] metavar: Optional[str] help: Optional[str] + # TODO - add prefix_name: bool def arg( From 62f49a1283f3f50c06efb95da84ebfb1f7170b20 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 19 Nov 2022 17:54:08 -0800 Subject: [PATCH 230/491] Cleanup --- tyro/_argparse_formatter.py | 8 +++---- tyro/_cli.py | 6 ++--- tyro/_parsers.py | 44 +++++++++++++++++++------------------ 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tyro/_argparse_formatter.py b/tyro/_argparse_formatter.py index a29a38554..d5ed0ea3e 100644 --- a/tyro/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -28,7 +28,7 @@ @dataclasses.dataclass -class DcargsTheme: +class TyroTheme: border: Style = Style() description: Style = Style() invocation: Style = Style() @@ -65,7 +65,7 @@ def set_accent_color(accent_color: Optional[str]) -> None: # TODO: this is a prototype; for a v1.0.0 release we should revisit whether the global # state here is acceptable or not. -THEME = DcargsTheme() +THEME = TyroTheme() set_accent_color(None) @@ -128,7 +128,7 @@ def str_from_rich( return out.get().rstrip("\n") -class DcargsArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): +class TyroArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): def __init__(self, prog: str): indent_increment = 4 width = shutil.get_terminal_size().columns - 2 @@ -383,7 +383,7 @@ def _tyro_format_nonroot(self): item_content = func(*args) if ( getattr(func, "__func__", None) - is DcargsArgparseHelpFormatter._format_action + is TyroArgparseHelpFormatter._format_action ): (action,) = args assert isinstance(action, argparse.Action) diff --git a/tyro/_cli.py b/tyro/_cli.py index 2398fc15e..11187f5b6 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -284,7 +284,7 @@ def fix_arg(arg: str) -> str: with _argparse_formatter.ansi_context(): parser = argparse.ArgumentParser( prog=prog, - formatter_class=_argparse_formatter.DcargsArgparseHelpFormatter, + formatter_class=_argparse_formatter.TyroArgparseHelpFormatter, ) parser_definition.apply(parser) @@ -327,9 +327,7 @@ def _cache_subparsers(parser_definition: _parsers.ParserSpecification) -> None: subparsers = parser_definition.subparsers if subparsers is None: return - subparser_def_from_prefixed_field_name[ - subparsers.prefix if subparsers.prefix != _strings.dummy_field_name else "" - ] = subparsers + subparser_def_from_prefixed_field_name[subparsers.prefix] = subparsers for p in subparsers.parser_from_name.values(): _cache_subparsers(p) diff --git a/tyro/_parsers.py b/tyro/_parsers.py index e88db8865..258f02c16 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -114,8 +114,8 @@ def from_callable_or_type( subparsers = subparsers_attempt continue else: - subparsers = subparsers.add_subparsers_to_leaves( - subparsers_attempt + subparsers = add_subparsers_to_leaves( + subparsers, subparsers_attempt ) continue @@ -150,8 +150,8 @@ def from_callable_or_type( subparsers = ( nested_parser.subparsers if subparsers is None - else subparsers.add_subparsers_to_leaves( - nested_parser.subparsers + else add_subparsers_to_leaves( + subparsers, nested_parser.subparsers ) ) @@ -490,7 +490,7 @@ def apply(self, parent_parser: argparse.ArgumentParser) -> None: if self.can_be_none: subparser = argparse_subparsers.add_parser( name=_strings.subparser_name_from_type(self.prefix, None), - formatter_class=_argparse_formatter.DcargsArgparseHelpFormatter, + formatter_class=_argparse_formatter.TyroArgparseHelpFormatter, help="", ) @@ -502,24 +502,26 @@ def apply(self, parent_parser: argparse.ArgumentParser) -> None: subparser = argparse_subparsers.add_parser( name, - formatter_class=_argparse_formatter.DcargsArgparseHelpFormatter, + formatter_class=_argparse_formatter.TyroArgparseHelpFormatter, help=helptext, ) subparser_def.apply(subparser) - def add_subparsers_to_leaves( - self, subparsers: SubparsersSpecification - ) -> SubparsersSpecification: - new_parsers_from_name = {} - for name, parser in self.parser_from_name.items(): - new_parsers_from_name[name] = dataclasses.replace( - parser, - subparsers=subparsers - if parser.subparsers is None - else parser.subparsers.add_subparsers_to_leaves(subparsers), - ) - return dataclasses.replace( - self, - parser_from_name=new_parsers_from_name, - required=self.required or subparsers.required, + +def add_subparsers_to_leaves( + root: Optional[SubparsersSpecification], leaf: SubparsersSpecification +) -> SubparsersSpecification: + if root is None: + return leaf + + new_parsers_from_name = {} + for name, parser in root.parser_from_name.items(): + new_parsers_from_name[name] = dataclasses.replace( + parser, + subparsers=add_subparsers_to_leaves(parser.subparsers, leaf), ) + return dataclasses.replace( + root, + parser_from_name=new_parsers_from_name, + required=root.required or leaf.required, + ) From fd13815ed7d952b713edf253f0c9091d17dd1238 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 19 Nov 2022 18:59:07 -0800 Subject: [PATCH 231/491] Add `tyro.conf.ConsolidateSubcommandArgs`, `tyro.conf.configure()` --- tests/test_conf.py | 119 ++++++++++++++++++++++++++++++++++++++++++ tyro/_cli.py | 1 + tyro/_parsers.py | 50 +++++++++++++++--- tyro/conf/__init__.py | 4 ++ tyro/conf/_markers.py | 53 ++++++++++++++++++- 5 files changed, 220 insertions(+), 7 deletions(-) diff --git a/tests/test_conf.py b/tests/test_conf.py index fd0eb0cdd..c0ff1fa4d 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -543,3 +543,122 @@ def main(x: int, y: tyro.conf.Positional[int]) -> int: assert tyro.cli(main, args="5 --x 3".split(" ")) == 8 assert tyro.cli(main, args="--x 3 5".split(" ")) == 8 + + +def test_omit_subcommand_prefix_and_consolidate_subcommand_args(): + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + flag: bool = True + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + bc: tyro.conf.OmitSubcommandPrefixes[ + Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + ] + + assert ( + tyro.cli( + tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], + args=[ + "bc:default-instance-http-server", + "--x", + "1", + "--y", + "5", + "--no-flag", + ], + ) + == tyro.cli( + tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], + args=[ + "bc:default-instance-http-server", + "--x", + "1", + "--y", + "5", + ], + default=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=3, flag=False) + ), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5, flag=False)) + ) + assert ( + tyro.cli( + tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], + args=[ + "bc:default-instance-http-server", + "--x", + "1", + "--y", + "8", + ], + ) + == tyro.cli( + tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], + args=[ + "bc:default-instance-http-server", + "--x", + "1", + "--y", + "8", + ], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + ) + + +def test_omit_subcommand_prefix_and_consolidate_subcommand_args_in_function(): + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + flag: bool = True + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + + @tyro.conf.configure( + tyro.conf.OmitSubcommandPrefixes, + tyro.conf.ConsolidateSubcommandArgs, + ) + def func(parent: DefaultInstanceSubparser) -> DefaultInstanceSubparser: + return parent + + assert tyro.cli( + func, + args=[ + "parent.bc:default-instance-http-server", + "--parent.x", + "1", + # --y and --no-flag are in a subcommand with prefix omission. + "--y", + "5", + "--no-flag", + ], + ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5, flag=False)) + assert tyro.cli( + func, + args=[ + "parent.bc:default-instance-http-server", + "--parent.x", + "1", + # --y is in a subcommand with prefix omission. + "--y", + "8", + ], + ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) diff --git a/tyro/_cli.py b/tyro/_cli.py index 11187f5b6..3f9fcfdad 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -278,6 +278,7 @@ def fix_arg(arg: str) -> str: parent_classes=set(), # Used for recursive calls. default_instance=default_instance_internal, # Overrides for default values. prefix="", # Used for recursive calls. + subcommand_prefix="", # Used for recursive calls. ) # Generate parser! diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 258f02c16..99aa0083a 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -4,7 +4,19 @@ import argparse import dataclasses -from typing import Any, Callable, Dict, List, Optional, Set, Type, TypeVar, Union, cast +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, + cast, +) from typing_extensions import Annotated, get_args, get_origin @@ -33,6 +45,7 @@ class ParserSpecification: subparsers: Optional[SubparsersSpecification] prefix: str has_required_args: bool + consolidate_subcommand_args: bool @staticmethod def from_callable_or_type( @@ -47,6 +60,12 @@ def from_callable_or_type( ) -> ParserSpecification: """Create a parser definition from a callable or type.""" + # Consolidate subcommand types. + consolidate_subcommand_args = ( + _markers.ConsolidateSubcommandArgs + in _resolver.unwrap_annotated(f, _markers.Marker)[1] + ) + # Resolve generic types. f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) @@ -197,18 +216,32 @@ def from_callable_or_type( subparsers=subparsers, prefix=prefix, has_required_args=has_required_args, + consolidate_subcommand_args=consolidate_subcommand_args, ) - def apply(self, parser: argparse.ArgumentParser) -> None: + def apply( + self, parser: argparse.ArgumentParser + ) -> Tuple[argparse.ArgumentParser, ...]: """Create defined arguments and subparsers.""" # Generate helptext. parser.description = self.description - self.apply_args(parser) # Create subparser tree. if self.subparsers is not None: - self.subparsers.apply(parser) + leaves = self.subparsers.apply(parser) + else: + leaves = (parser,) + + # Depending on whether we want to consolidate subcommand args, we can either + # apply arguments to the intermediate parser or only on the leaves. + if self.consolidate_subcommand_args: + for leaf in leaves: + self.apply_args(leaf) + else: + self.apply_args(parser) + + return leaves def apply_args(self, parser: argparse.ArgumentParser) -> None: """Create defined arguments and subparsers.""" @@ -460,7 +493,9 @@ def from_field( can_be_none=options != options_no_none, ) - def apply(self, parent_parser: argparse.ArgumentParser) -> None: + def apply( + self, parent_parser: argparse.ArgumentParser + ) -> Tuple[argparse.ArgumentParser, ...]: title = "subcommands" metavar = ( "{" @@ -494,6 +529,7 @@ def apply(self, parent_parser: argparse.ArgumentParser) -> None: help="", ) + subparser_tree_leaves: List[argparse.ArgumentParser] = [] for name, subparser_def in self.parser_from_name.items(): helptext = subparser_def.description.replace("%", "%%") if len(helptext) > 0: @@ -505,7 +541,9 @@ def apply(self, parent_parser: argparse.ArgumentParser) -> None: formatter_class=_argparse_formatter.TyroArgparseHelpFormatter, help=helptext, ) - subparser_def.apply(subparser) + subparser_tree_leaves.extend(subparser_def.apply(subparser)) + + return tuple(subparser_tree_leaves) def add_subparsers_to_leaves( diff --git a/tyro/conf/__init__.py b/tyro/conf/__init__.py index 9a84b819b..3fa3556cf 100644 --- a/tyro/conf/__init__.py +++ b/tyro/conf/__init__.py @@ -11,22 +11,26 @@ from ._confstruct import arg, subcommand from ._markers import ( AvoidSubcommands, + ConsolidateSubcommandArgs, Fixed, FlagConversionOff, OmitSubcommandPrefixes, Positional, Suppress, SuppressFixed, + configure, ) __all__ = [ "arg", "subcommand", "AvoidSubcommands", + "ConsolidateSubcommandArgs", "Fixed", "FlagConversionOff", "OmitSubcommandPrefixes", "Positional", "Suppress", "SuppressFixed", + "configure", ] diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index d267ee432..ec71a8ba8 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Type, TypeVar +from typing import TYPE_CHECKING, Callable, Type, TypeVar from typing_extensions import Annotated @@ -51,6 +51,28 @@ Can be used directly on union types, `AvoidSubcommands[Union[...]]`, or recursively applied to nested types.""" +ConsolidateSubcommandArgs = Annotated[T, None] +"""Consolidate arguments applied to subcommands. Makes CLI less sensitive to argument +ordering, at the cost of support for optional subcommands. + +By default, `tyro` will generate a traditional CLI interface where args are applied to +the directly preceding subcommand. When we have two subcommands `s1` and `s2`: +``` +python x.py {--root options} s1 {--s1 options} s2 {--s2 options} +``` + +This can be frustrating because the resulting CLI is sensitive the exact positioning and +ordering of options. + +To consolidate subcommands, we push arguments to the end, after all subcommands: +``` +python x.py s1 s2 {--root, s1, and s2 options} +``` + +This is more robust to reordering of options, ensuring that any new options can simply +be placed at the end of the command> +""" + OmitSubcommandPrefixes = Annotated[T, None] """Make flags used for keyword arguments in subcommands shorter by omitting prefixes. @@ -62,16 +84,45 @@ If subcommand prefixes are omitted, we would instead simply have `--arg`. """ +CallableType = TypeVar("CallableType", bound=Callable) # Dynamically generate marker singletons. # These can be used one of two ways: # - Marker[T] # - Annotated[T, Marker] + + class Marker(_singleton.Singleton): def __getitem__(self, key): return Annotated.__class_getitem__((key, self)) # type: ignore +def configure(*markers: Marker) -> Callable[[CallableType], CallableType]: + """Decorator for configuring functions. + + Configuration markers are implemented via `typing.Annotated` and straightforward to + apply to types, for example: + + ```python + field: tyro.conf.FlagConversionOff[bool] + ``` + + This decorator makes markers applicable to general functions as well: + + ```python + # Recursively apply FlagConversionOff to all field in `main()`. + @tyro.conf.configure_function(tyro.conf.FlagConversionOff) + def main(field: bool) -> None: + ... + ``` + """ + + def _inner(callable: CallableType) -> CallableType: + return Annotated.__class_getitem__((callable,) + tuple(markers)) # type: ignore + + return _inner + + if not TYPE_CHECKING: def _make_marker(description: str) -> Marker: From 31f8b275035c6d5ed881e352bb03755fb059dd20 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 19 Nov 2022 19:24:43 -0800 Subject: [PATCH 232/491] Helptext cleanup, version bump --- .../02_nesting/03_multiple_subcommands.rst | 13 +++++++++++-- examples/02_nesting/03_multiple_subcommands.py | 4 +++- pyproject.toml | 2 +- tyro/_parsers.py | 18 +++++++++++------- 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/docs/source/examples/02_nesting/03_multiple_subcommands.rst b/docs/source/examples/02_nesting/03_multiple_subcommands.rst index 85590a40f..78b3d9ea2 100644 --- a/docs/source/examples/02_nesting/03_multiple_subcommands.rst +++ b/docs/source/examples/02_nesting/03_multiple_subcommands.rst @@ -52,6 +52,7 @@ Multiple unions over nested types are populated using a series of subcommands. # Train script. + @tyro.conf.configure(tyro.conf.ConsolidateSubcommandArgs) def train( dataset: Union[Mnist, ImageNet] = Mnist(), optimizer: Union[Adam, Sgd] = Adam(), @@ -100,6 +101,14 @@ Multiple unions over nested types are populated using a series of subcommands. .. raw:: html - python 02_nesting/03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 + python 02_nesting/03_multiple_subcommands.py dataset:mnist optimizer:adam --help -.. program-output:: python ../../examples/02_nesting/03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 +.. program-output:: python ../../examples/02_nesting/03_multiple_subcommands.py dataset:mnist optimizer:adam --help + +------------ + +.. raw:: html + + python 02_nesting/03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 --dataset.binary + +.. program-output:: python ../../examples/02_nesting/03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 --dataset.binary diff --git a/examples/02_nesting/03_multiple_subcommands.py b/examples/02_nesting/03_multiple_subcommands.py index 7c31066a1..99d4bf79a 100644 --- a/examples/02_nesting/03_multiple_subcommands.py +++ b/examples/02_nesting/03_multiple_subcommands.py @@ -6,7 +6,8 @@ `python ./03_multiple_subcommands.py` `python ./03_multiple_subcommands.py --help` `python ./03_multiple_subcommands.py dataset:mnist --help` -`python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4` +`python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --help` +`python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 --dataset.binary` """ from __future__ import annotations @@ -47,6 +48,7 @@ class Sgd: # Train script. +@tyro.conf.configure(tyro.conf.ConsolidateSubcommandArgs) def train( dataset: Union[Mnist, ImageNet] = Mnist(), optimizer: Union[Adam, Sgd] = Adam(), diff --git a/pyproject.toml b/pyproject.toml index 7343a9877..1d869fc0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.33" +version = "0.3.34" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 99aa0083a..3b7e7ff55 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -241,6 +241,9 @@ def apply( else: self.apply_args(parser) + # Break some API boundaries to rename the "optional arguments" => "arguments". + parser._action_groups[1].title = "arguments" + return leaves def apply_args(self, parser: argparse.ArgumentParser) -> None: @@ -250,17 +253,18 @@ def apply_args(self, parser: argparse.ArgumentParser) -> None: parser.description = self.description # Make argument groups. - def format_group_name(nested_field_name: str) -> str: - return (nested_field_name + " arguments").strip() + def format_group_name(prefix: str) -> str: + return (prefix + " arguments").strip() group_from_prefix: Dict[str, argparse._ArgumentGroup] = { "": parser._action_groups[1], + **{ + cast(str, group.title).partition(" ")[0]: group + for group in parser._action_groups[2:] + }, } - - # Break some API boundaries to rename the optional group. - parser._action_groups[1].title = format_group_name("") - positional_group = parser.add_argument_group("positional arguments") - parser._action_groups = parser._action_groups[::-1] + positional_group = parser._action_groups[0] + assert positional_group.title == "positional arguments" # Add each argument group. Note that groups with only suppressed arguments won't # be added. From 0bf605f684f07f8ecc9c7be682b6d0bfaae0bee7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 19 Nov 2022 19:38:31 -0800 Subject: [PATCH 233/491] Add group title assert --- tyro/_parsers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 3b7e7ff55..7ca39daaf 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -242,6 +242,7 @@ def apply( self.apply_args(parser) # Break some API boundaries to rename the "optional arguments" => "arguments". + assert parser._action_groups[1].title == "optional arguments" parser._action_groups[1].title = "arguments" return leaves From ed092a5f1e10edaf126523ca505ca537627b31ad Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 20 Nov 2022 16:44:17 -0800 Subject: [PATCH 234/491] Fix errors reported in nerfstudio https://github.com/nerfstudio-project/nerfstudio/issues/988 https://github.com/nerfstudio-project/nerfstudio/issues/987 --- poetry.lock | 495 +++++++++++++++++++++++++------------------ tests/test_nested.py | 57 ++++- tyro/_calling.py | 9 +- tyro/_cli.py | 14 -- tyro/_parsers.py | 42 ++-- 5 files changed, 371 insertions(+), 246 deletions(-) diff --git a/poetry.lock b/poetry.lock index 44c922f87..a32170521 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,6 +1,6 @@ [[package]] name = "absl-py" -version = "1.2.0" +version = "1.3.0" description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." category = "dev" optional = false @@ -54,11 +54,11 @@ toolz = ">=0.9.0" [[package]] name = "colorama" -version = "0.4.5" +version = "0.4.6" description = "Cross-platform colored terminal text." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" [[package]] name = "commonmark" @@ -110,37 +110,19 @@ optional = false python-versions = ">=3.6,<4.0" [[package]] -name = "etils" -version = "0.8.0" -description = "Collection of common python utils" +name = "exceptiongroup" +version = "1.0.4" +description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false python-versions = ">=3.7" -[package.dependencies] -importlib_resources = {version = "*", optional = true, markers = "extra == \"epath\""} -typing_extensions = {version = "*", optional = true, markers = "extra == \"epath\""} -zipp = {version = "*", optional = true, markers = "extra == \"epath\""} - [package.extras] -all = ["etils[array-types]", "etils[ecolab]", "etils[edc]", "etils[enp]", "etils[epath]", "etils[epy]", "etils[etqdm]", "etils[etree-dm]", "etils[etree-jax]", "etils[etree-tf]", "etils[etree]"] -array-types = ["etils[enp]"] -dev = ["chex", "pylint (>=2.6.0)", "pytest", "pytest-subtests", "pytest-xdist", "yapf"] -ecolab = ["etils[enp]", "etils[epy]", "jupyter", "mediapy", "numpy"] -edc = ["etils[epy]"] -enp = ["etils[epy]", "numpy"] -epath = ["etils[epy]", "importlib_resources", "typing_extensions", "zipp"] -epy = ["typing_extensions"] -etqdm = ["absl-py", "etils[epy]", "tqdm"] -etree = ["etils[array_types]", "etils[enp]", "etils[epy]", "etils[etqdm]"] -etree-dm = ["dm-tree", "etils[etree]"] -etree-jax = ["etils[etree]", "jax[cpu]"] -etree-tf = ["etils[etree]", "tf-nightly"] -lazy-imports = ["etils[ecolab]"] +test = ["pytest (>=6)"] [[package]] name = "flax" -version = "0.6.0" +version = "0.6.2" description = "Flax: A neural network library for JAX designed for flexibility" category = "dev" optional = false @@ -153,15 +135,16 @@ msgpack = "*" numpy = ">=1.12" optax = "*" PyYAML = ">=5.4.1" -rich = ">=11.1,<12.0" +rich = ">=11.1" +tensorstore = "*" typing-extensions = ">=4.1.1" [package.extras] -testing = ["atari-py (==0.2.5)", "clu", "gym (==0.18.3)", "jaxlib", "jraph (>=0.0.6dev0)", "ml-collections", "opencv-python", "pytest", "pytest-cov", "pytest-custom-exit-code", "pytest-xdist (==1.34.0)", "pytype", "sentencepiece", "svn", "tensorflow", "tensorflow-datasets", "tensorflow-text (>=2.4.0)", "torch"] +testing = ["atari-py (==0.2.5)", "clu", "gym (==0.18.3)", "jaxlib", "jraph (>=0.0.6dev0)", "ml-collections", "opencv-python", "pytest", "pytest-cov", "pytest-custom-exit-code", "pytest-xdist (==1.34.0)", "pytype", "sentencepiece", "tensorflow", "tensorflow-datasets", "tensorflow-text (>=2.4.0)", "torch"] [[package]] name = "fonttools" -version = "4.37.3" +version = "4.38.0" description = "Tools to manipulate font files" category = "dev" optional = false @@ -191,7 +174,7 @@ python-versions = ">=3.6" [[package]] name = "importlib-metadata" -version = "4.12.0" +version = "5.0.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -202,24 +185,9 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] - -[[package]] -name = "importlib-resources" -version = "5.9.0" -description = "Read resources from Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} - -[package.extras] -docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] -testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] [[package]] name = "iniconfig" @@ -231,15 +199,13 @@ python-versions = "*" [[package]] name = "jax" -version = "0.3.17" +version = "0.3.25" description = "Differentiate, compile, and transform Numpy code." category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -absl-py = "*" -etils = {version = "*", extras = ["epath"]} numpy = ">=1.20" opt_einsum = "*" scipy = ">=1.5" @@ -247,25 +213,24 @@ typing_extensions = "*" [package.extras] australis = ["protobuf (>=3.13,<4)"] -ci = ["jaxlib (==0.3.15)"] -cpu = ["jaxlib (==0.3.15)"] -cuda = ["jaxlib (==0.3.15+cuda11.cudnn82)"] -cuda11_cudnn805 = ["jaxlib (==0.3.15+cuda11.cudnn805)"] -cuda11_cudnn82 = ["jaxlib (==0.3.15+cuda11.cudnn82)"] -minimum-jaxlib = ["jaxlib (==0.3.14)"] -tpu = ["jaxlib (==0.3.15)", "libtpu-nightly (==0.1.dev20220723)", "requests"] +ci = ["jaxlib (==0.3.24)"] +cpu = ["jaxlib (==0.3.25)"] +cuda = ["jaxlib (==0.3.25+cuda11.cudnn82)"] +cuda11_cudnn805 = ["jaxlib (==0.3.25+cuda11.cudnn805)"] +cuda11_cudnn82 = ["jaxlib (==0.3.25+cuda11.cudnn82)"] +minimum-jaxlib = ["jaxlib (==0.3.22)"] +tpu = ["jaxlib (==0.3.25)", "libtpu-nightly (==0.1.dev20221109)", "requests"] [[package]] name = "jaxlib" -version = "0.3.15" +version = "0.3.25" description = "XLA library for JAX" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -absl-py = "*" -numpy = ">=1.19" +numpy = ">=1.20" scipy = ">=1.5" [[package]] @@ -352,6 +317,54 @@ category = "dev" optional = false python-versions = ">=3.7" +[[package]] +name = "nvidia-cublas-cu11" +version = "11.10.3.66" +description = "CUBLAS native runtime libraries" +category = "dev" +optional = false +python-versions = ">=3" + +[package.dependencies] +setuptools = "*" +wheel = "*" + +[[package]] +name = "nvidia-cuda-nvrtc-cu11" +version = "11.7.99" +description = "NVRTC native runtime libraries" +category = "dev" +optional = false +python-versions = ">=3" + +[package.dependencies] +setuptools = "*" +wheel = "*" + +[[package]] +name = "nvidia-cuda-runtime-cu11" +version = "11.7.99" +description = "CUDA Runtime native Libraries" +category = "dev" +optional = false +python-versions = ">=3" + +[package.dependencies] +setuptools = "*" +wheel = "*" + +[[package]] +name = "nvidia-cudnn-cu11" +version = "8.5.0.96" +description = "cuDNN runtime libraries" +category = "dev" +optional = false +python-versions = ">=3" + +[package.dependencies] +setuptools = "*" +wheel = "*" + [[package]] name = "omegaconf" version = "2.2.3" @@ -408,7 +421,7 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" [[package]] name = "Pillow" -version = "9.2.0" +version = "9.3.0" description = "Python Imaging Library (Fork)" category = "dev" optional = false @@ -433,14 +446,6 @@ importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] -[[package]] -name = "py" -version = "1.11.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - [[package]] name = "pydantic" version = "1.10.2" @@ -480,7 +485,7 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pyright" -version = "1.1.271" +version = "1.1.280" description = "Command line wrapper for pyright" category = "dev" optional = false @@ -496,7 +501,7 @@ dev = ["twine (>=3.4.1)"] [[package]] name = "pytest" -version = "7.1.3" +version = "7.2.0" description = "pytest: simple powerful testing with Python" category = "dev" optional = false @@ -505,12 +510,12 @@ python-versions = ">=3.7" [package.dependencies] attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" -py = ">=1.8.2" -tomli = ">=1.0.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] @@ -551,17 +556,16 @@ python-versions = ">=3.6" [[package]] name = "rich" -version = "11.2.0" +version = "12.6.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false -python-versions = ">=3.6.2,<4.0.0" +python-versions = ">=3.6.3,<4.0.0" [package.dependencies] -colorama = ">=0.4.0,<0.5.0" commonmark = ">=0.9.0,<0.10.0" pygments = ">=2.6.0,<3.0.0" -typing-extensions = {version = ">=3.7.4,<5.0", markers = "python_version < \"3.8\""} +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} [package.extras] jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] @@ -579,15 +583,15 @@ numpy = ">=1.16.5" [[package]] name = "setuptools" -version = "65.3.0" +version = "65.6.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -609,11 +613,11 @@ toml = ["setuptools (>=42)"] [[package]] name = "shtab" -version = "1.5.6" +version = "1.5.8" description = "Automagic shell tab completion for Python CLI applications" category = "main" optional = false -python-versions = ">=3.2" +python-versions = ">=3.7" [[package]] name = "six" @@ -623,6 +627,17 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "tensorstore" +version = "0.1.28" +description = "Read and write large, multi-dimensional arrays" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +numpy = ">=1.16.0" + [[package]] name = "tomli" version = "2.0.1" @@ -641,15 +656,22 @@ python-versions = ">=3.5" [[package]] name = "torch" -version = "1.12.1" +version = "1.13.0" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" category = "dev" optional = false python-versions = ">=3.7.0" [package.dependencies] +nvidia-cublas-cu11 = "11.10.3.66" +nvidia-cuda-nvrtc-cu11 = "11.7.99" +nvidia-cuda-runtime-cu11 = "11.7.99" +nvidia-cudnn-cu11 = "8.5.0.96" typing-extensions = "*" +[package.extras] +opt-einsum = ["opt-einsum (>=3.3)"] + [[package]] name = "typed-ast" version = "1.5.4" @@ -660,23 +682,34 @@ python-versions = ">=3.6" [[package]] name = "typing-extensions" -version = "4.3.0" +version = "4.4.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "wheel" +version = "0.38.4" +description = "A built-package format for Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +test = ["pytest (>=3.0.0)"] + [[package]] name = "zipp" -version = "3.8.1" +version = "3.10.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false python-versions = ">=3.7" [package.extras] -docs = ["jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] -testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [metadata] lock-version = "1.1" @@ -685,8 +718,8 @@ content-hash = "49867c0eda45ffab0a881ae1181315655a200d447d3ca4f85bc25713acb4a699 [metadata.files] absl-py = [ - {file = "absl-py-1.2.0.tar.gz", hash = "sha256:f568809938c49abbda89826223c992b630afd23c638160ad7840cfe347710d97"}, - {file = "absl_py-1.2.0-py3-none-any.whl", hash = "sha256:5d15f85b8cc859c6245bc9886ba664460ed96a6fee895416caa37d669ee74a9a"}, + {file = "absl-py-1.3.0.tar.gz", hash = "sha256:463c38a08d2e4cef6c498b76ba5bd4858e4c6ef51da1a5a1f27139a022e20248"}, + {file = "absl_py-1.3.0-py3-none-any.whl", hash = "sha256:34995df9bd7a09b3b8749e230408f5a2a2dd7a68a0d33c12a3d0cb15a041a507"}, ] antlr4-python3-runtime = [ {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, @@ -704,8 +737,8 @@ chex = [ {file = "chex-0.1.5.tar.gz", hash = "sha256:686858320f8f220c82a6c7eeb54dcdcaa4f3d7f66690dacd13a24baa1ee8299e"}, ] colorama = [ - {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, - {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] commonmark = [ {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, @@ -795,17 +828,17 @@ docstring-parser = [ {file = "docstring_parser-0.14.1-py3-none-any.whl", hash = "sha256:14ac6ec1f1ba6905c4d8cb90fd0bc55394f5678183752c90e44812bf28d7a515"}, {file = "docstring_parser-0.14.1.tar.gz", hash = "sha256:2c77522e31b7c88b1ab457a1f3c9ae38947ad719732260ba77ee8a3deb58622a"}, ] -etils = [ - {file = "etils-0.8.0-py3-none-any.whl", hash = "sha256:b6299d2a835206b063498e443bd8036edb974058899169677dd6ddb92430bb84"}, - {file = "etils-0.8.0.tar.gz", hash = "sha256:d1d5af7bd9c784a273c4e1eccfaa8feaca5e0481a08717b5313fa231da22a903"}, +exceptiongroup = [ + {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, + {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, ] flax = [ - {file = "flax-0.6.0-py3-none-any.whl", hash = "sha256:98197d17a82bb239c13d9317508cc4cee8daf8728e8db8fcfbe80277a7dfd9e6"}, - {file = "flax-0.6.0.tar.gz", hash = "sha256:6d9bc9d5e86291610e646c945a314912a8769508aa95ed751790f983f6e29c3d"}, + {file = "flax-0.6.2-py3-none-any.whl", hash = "sha256:9f933c87fb5762fbbf0920531e770bc385f24ef6eeb2f473641591fdbde9de89"}, + {file = "flax-0.6.2.tar.gz", hash = "sha256:a6247b412f14466fefcc70d043bd0facf72552d322acdda8a8700285308d390f"}, ] fonttools = [ - {file = "fonttools-4.37.3-py3-none-any.whl", hash = "sha256:a5bc5f5d48faa4085310b8ebd4c5d33bf27c6636c5f10a7de792510af2745a81"}, - {file = "fonttools-4.37.3.zip", hash = "sha256:f32ef6ec966cf0e7d2aa88601fed2e3a8f2851c26b5db2c80ccc8f82bee4eedc"}, + {file = "fonttools-4.38.0-py3-none-any.whl", hash = "sha256:820466f43c8be8c3009aef8b87e785014133508f0de64ec469e4efb643ae54fb"}, + {file = "fonttools-4.38.0.zip", hash = "sha256:2bb244009f9bf3fa100fc3ead6aeb99febe5985fa20afbfbaa2f8946c2fbdaf1"}, ] frozendict = [ {file = "frozendict-2.3.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4a3b32d47282ae0098b9239a6d53ec539da720258bd762d62191b46f2f87c5fc"}, @@ -827,32 +860,31 @@ frozendict = [ {file = "frozendict-2.3.4.tar.gz", hash = "sha256:15b4b18346259392b0d27598f240e9390fafbff882137a9c48a1e0104fb17f78"}, ] importlib-metadata = [ - {file = "importlib_metadata-4.12.0-py3-none-any.whl", hash = "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23"}, - {file = "importlib_metadata-4.12.0.tar.gz", hash = "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670"}, -] -importlib-resources = [ - {file = "importlib_resources-5.9.0-py3-none-any.whl", hash = "sha256:f78a8df21a79bcc30cfd400bdc38f314333de7c0fb619763f6b9dabab8268bb7"}, - {file = "importlib_resources-5.9.0.tar.gz", hash = "sha256:5481e97fb45af8dcf2f798952625591c58fe599d0735d86b10f54de086a61681"}, + {file = "importlib_metadata-5.0.0-py3-none-any.whl", hash = "sha256:ddb0e35065e8938f867ed4928d0ae5bf2a53b7773871bfe6bcc7e4fcdc7dea43"}, + {file = "importlib_metadata-5.0.0.tar.gz", hash = "sha256:da31db32b304314d044d3c12c79bd59e307889b287ad12ff387b3500835fc2ab"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] jax = [ - {file = "jax-0.3.17.tar.gz", hash = "sha256:2a2794e4e0c93595a1b1d625026580c0686be93bd60d4f6906b090446692cadc"}, + {file = "jax-0.3.25.tar.gz", hash = "sha256:18bea69321cb95ea5ea913adfe5e2c1d453cade9d4cfd0dc814ecba9fc0cb6e3"}, ] jaxlib = [ - {file = "jaxlib-0.3.15-cp310-none-macosx_10_14_x86_64.whl", hash = "sha256:7555be95d567572579a2010ed03506e31ff87d55c21c1bc80eddcccd87b3f723"}, - {file = "jaxlib-0.3.15-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:89f5e27c59f8f5c2e5f3ad377ace2bc03e68728ff27aaf9e82e59d2a1a4c3c6d"}, - {file = "jaxlib-0.3.15-cp310-none-manylinux2014_x86_64.whl", hash = "sha256:3af8343024ec44488bb542e62c7a61488df558e6da4bf072ebbdac319362a03f"}, - {file = "jaxlib-0.3.15-cp37-none-macosx_10_14_x86_64.whl", hash = "sha256:8f999a69fda69976bdcfa45abcc99614160035ee878a544e713affd45cd5b4d1"}, - {file = "jaxlib-0.3.15-cp37-none-manylinux2014_x86_64.whl", hash = "sha256:36570c1699b5bb4038f1f708d976afefff9fb745cc131c0c8c9340a072b38244"}, - {file = "jaxlib-0.3.15-cp38-none-macosx_10_14_x86_64.whl", hash = "sha256:19038f91f49f44df1a9d2c00e11f7f61e922e4651c10cd928ac3dcb745234dbf"}, - {file = "jaxlib-0.3.15-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:30385dafe8299c60da1e1f6f3e0d095cfd2aeefc398c67501a772c8238fc57cd"}, - {file = "jaxlib-0.3.15-cp38-none-manylinux2014_x86_64.whl", hash = "sha256:a97e80d533d1c66fe0c963e6a25cf85e00732710b6e17af568b1f4f026b52919"}, - {file = "jaxlib-0.3.15-cp39-none-macosx_10_14_x86_64.whl", hash = "sha256:e8d63b156cbac46a15c2024f63761b0bc1f73df94d6a97fb641326c74aeab6ef"}, - {file = "jaxlib-0.3.15-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:c290ca60b1c08d0ac991be922146775a80943e7a970cde3fc42d6667ca0c115d"}, - {file = "jaxlib-0.3.15-cp39-none-manylinux2014_x86_64.whl", hash = "sha256:ef58c67341c87789417891f7bd832fc84e23044ef6b085fb9c8306c60ec43bda"}, + {file = "jaxlib-0.3.25-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:09508f7000c0fa958fba29267338e8de75b31d7ea29bd79719a568c38f0f8d31"}, + {file = "jaxlib-0.3.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c75c8efd3702687968820446e3fb9ff997f8a2a07ab92e33b80e2f12eab3d9a"}, + {file = "jaxlib-0.3.25-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:6e2f4e51041b8371aa3976b5a3a9cdcdccb1bd7b040c9b1345cbf24bd28a8d19"}, + {file = "jaxlib-0.3.25-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:f2d517635fd77e2729c0ab7863be0d290927d01b2abb2f5dc955c821f8b0d53e"}, + {file = "jaxlib-0.3.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:13446a8382aa9ed944c16af636ca111d0afbbead91eed5cc2dc71195045e71b3"}, + {file = "jaxlib-0.3.25-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:71866aeaafbc9a12b59dcbe443353772ef235b40c53f8bd7403d39311822c276"}, + {file = "jaxlib-0.3.25-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:1e59ba58c9e93c1e1cef243f2609ec0b0c0a81160c20b9555aecdea32ccd6a78"}, + {file = "jaxlib-0.3.25-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:5295354ed5db111e6f3e88cdfa4010d11c33dd926ac61735b9096b4e0746aa7b"}, + {file = "jaxlib-0.3.25-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:9f3116389ee834b3cdeb30001b085f4b55d7741366034f041c1d377154aa5afa"}, + {file = "jaxlib-0.3.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:78b29c72d0680829db9377ed9be326875849258a60b8173b4a388b34ad18bc78"}, + {file = "jaxlib-0.3.25-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:2e008e0a6c10aa7e949555e98dc0471e0d550d5d7c109771e38a971b49480538"}, + {file = "jaxlib-0.3.25-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:fec043cdd55f3257d02e9d8880b33860eacadcae1bd5e26f43fdd08ada23614d"}, + {file = "jaxlib-0.3.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50193ba0cbf879021c9d73d7bcfa7eafb9138895d057b774c301aac3701f9a5"}, + {file = "jaxlib-0.3.25-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:1f1448f102a9d05186f579b6931fa0c607783ecc915fdfaa482c19538affa180"}, ] kiwisolver = [ {file = "kiwisolver-1.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2f5e60fabb7343a836360c4f0919b8cd0d6dbf08ad2ca6b9cf90bf0c76a3c4f6"}, @@ -1078,6 +1110,23 @@ numpy = [ {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"}, {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, ] +nvidia-cublas-cu11 = [ + {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl", hash = "sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded"}, + {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-win_amd64.whl", hash = "sha256:8ac17ba6ade3ed56ab898a036f9ae0756f1e81052a317bf98f8c6d18dc3ae49e"}, +] +nvidia-cuda-nvrtc-cu11 = [ + {file = "nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:9f1562822ea264b7e34ed5930567e89242d266448e936b85bc97a3370feabb03"}, + {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:f7d9610d9b7c331fa0da2d1b2858a4a8315e6d49765091d28711c8946e7425e7"}, + {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:f2effeb1309bdd1b3854fc9b17eaf997808f8b25968ce0c7070945c4265d64a3"}, +] +nvidia-cuda-runtime-cu11 = [ + {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:cc768314ae58d2641f07eac350f40f99dcb35719c4faff4bc458a7cd2b119e31"}, + {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:bc77fa59a7679310df9d5c70ab13c4e34c64ae2124dd1efd7e5474b71be125c7"}, +] +nvidia-cudnn-cu11 = [ + {file = "nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:402f40adfc6f418f9dae9ab402e773cfed9beae52333f6d86ae3107a1b9527e7"}, + {file = "nvidia_cudnn_cu11-8.5.0.96-py3-none-manylinux1_x86_64.whl", hash = "sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108"}, +] omegaconf = [ {file = "omegaconf-2.2.3-py3-none-any.whl", hash = "sha256:d6f2cbf79a992899eb76c6cb1aedfcf0fe7456a8654382edd5ee0c1b199c0657"}, {file = "omegaconf-2.2.3.tar.gz", hash = "sha256:59ff9fba864ffbb5fb710b64e8a9ba37c68fa339a2e2bb4f1b648d6901552523"}, @@ -1095,73 +1144,72 @@ packaging = [ {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] Pillow = [ - {file = "Pillow-9.2.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:a9c9bc489f8ab30906d7a85afac4b4944a572a7432e00698a7239f44a44e6efb"}, - {file = "Pillow-9.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:510cef4a3f401c246cfd8227b300828715dd055463cdca6176c2e4036df8bd4f"}, - {file = "Pillow-9.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7888310f6214f19ab2b6df90f3f06afa3df7ef7355fc025e78a3044737fab1f5"}, - {file = "Pillow-9.2.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:831e648102c82f152e14c1a0938689dbb22480c548c8d4b8b248b3e50967b88c"}, - {file = "Pillow-9.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1cc1d2451e8a3b4bfdb9caf745b58e6c7a77d2e469159b0d527a4554d73694d1"}, - {file = "Pillow-9.2.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:136659638f61a251e8ed3b331fc6ccd124590eeff539de57c5f80ef3a9594e58"}, - {file = "Pillow-9.2.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:6e8c66f70fb539301e064f6478d7453e820d8a2c631da948a23384865cd95544"}, - {file = "Pillow-9.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:37ff6b522a26d0538b753f0b4e8e164fdada12db6c6f00f62145d732d8a3152e"}, - {file = "Pillow-9.2.0-cp310-cp310-win32.whl", hash = "sha256:c79698d4cd9318d9481d89a77e2d3fcaeff5486be641e60a4b49f3d2ecca4e28"}, - {file = "Pillow-9.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:254164c57bab4b459f14c64e93df11eff5ded575192c294a0c49270f22c5d93d"}, - {file = "Pillow-9.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:adabc0bce035467fb537ef3e5e74f2847c8af217ee0be0455d4fec8adc0462fc"}, - {file = "Pillow-9.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:336b9036127eab855beec9662ac3ea13a4544a523ae273cbf108b228ecac8437"}, - {file = "Pillow-9.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50dff9cc21826d2977ef2d2a205504034e3a4563ca6f5db739b0d1026658e004"}, - {file = "Pillow-9.2.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb6259196a589123d755380b65127ddc60f4c64b21fc3bb46ce3a6ea663659b0"}, - {file = "Pillow-9.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b0554af24df2bf96618dac71ddada02420f946be943b181108cac55a7a2dcd4"}, - {file = "Pillow-9.2.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:15928f824870535c85dbf949c09d6ae7d3d6ac2d6efec80f3227f73eefba741c"}, - {file = "Pillow-9.2.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:bdd0de2d64688ecae88dd8935012c4a72681e5df632af903a1dca8c5e7aa871a"}, - {file = "Pillow-9.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5b87da55a08acb586bad5c3aa3b86505f559b84f39035b233d5bf844b0834b1"}, - {file = "Pillow-9.2.0-cp311-cp311-win32.whl", hash = "sha256:b6d5e92df2b77665e07ddb2e4dbd6d644b78e4c0d2e9272a852627cdba0d75cf"}, - {file = "Pillow-9.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6bf088c1ce160f50ea40764f825ec9b72ed9da25346216b91361eef8ad1b8f8c"}, - {file = "Pillow-9.2.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:2c58b24e3a63efd22554c676d81b0e57f80e0a7d3a5874a7e14ce90ec40d3069"}, - {file = "Pillow-9.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eef7592281f7c174d3d6cbfbb7ee5984a671fcd77e3fc78e973d492e9bf0eb3f"}, - {file = "Pillow-9.2.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dcd7b9c7139dc8258d164b55696ecd16c04607f1cc33ba7af86613881ffe4ac8"}, - {file = "Pillow-9.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a138441e95562b3c078746a22f8fca8ff1c22c014f856278bdbdd89ca36cff1b"}, - {file = "Pillow-9.2.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:93689632949aff41199090eff5474f3990b6823404e45d66a5d44304e9cdc467"}, - {file = "Pillow-9.2.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:f3fac744f9b540148fa7715a435d2283b71f68bfb6d4aae24482a890aed18b59"}, - {file = "Pillow-9.2.0-cp37-cp37m-win32.whl", hash = "sha256:fa768eff5f9f958270b081bb33581b4b569faabf8774726b283edb06617101dc"}, - {file = "Pillow-9.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:69bd1a15d7ba3694631e00df8de65a8cb031911ca11f44929c97fe05eb9b6c1d"}, - {file = "Pillow-9.2.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:030e3460861488e249731c3e7ab59b07c7853838ff3b8e16aac9561bb345da14"}, - {file = "Pillow-9.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:74a04183e6e64930b667d321524e3c5361094bb4af9083db5c301db64cd341f3"}, - {file = "Pillow-9.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d33a11f601213dcd5718109c09a52c2a1c893e7461f0be2d6febc2879ec2402"}, - {file = "Pillow-9.2.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fd6f5e3c0e4697fa7eb45b6e93996299f3feee73a3175fa451f49a74d092b9f"}, - {file = "Pillow-9.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a647c0d4478b995c5e54615a2e5360ccedd2f85e70ab57fbe817ca613d5e63b8"}, - {file = "Pillow-9.2.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:4134d3f1ba5f15027ff5c04296f13328fecd46921424084516bdb1b2548e66ff"}, - {file = "Pillow-9.2.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:bc431b065722a5ad1dfb4df354fb9333b7a582a5ee39a90e6ffff688d72f27a1"}, - {file = "Pillow-9.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1536ad017a9f789430fb6b8be8bf99d2f214c76502becc196c6f2d9a75b01b76"}, - {file = "Pillow-9.2.0-cp38-cp38-win32.whl", hash = "sha256:2ad0d4df0f5ef2247e27fc790d5c9b5a0af8ade9ba340db4a73bb1a4a3e5fb4f"}, - {file = "Pillow-9.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:ec52c351b35ca269cb1f8069d610fc45c5bd38c3e91f9ab4cbbf0aebc136d9c8"}, - {file = "Pillow-9.2.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:0ed2c4ef2451de908c90436d6e8092e13a43992f1860275b4d8082667fbb2ffc"}, - {file = "Pillow-9.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ad2f835e0ad81d1689f1b7e3fbac7b01bb8777d5a985c8962bedee0cc6d43da"}, - {file = "Pillow-9.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea98f633d45f7e815db648fd7ff0f19e328302ac36427343e4432c84432e7ff4"}, - {file = "Pillow-9.2.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7761afe0126d046974a01e030ae7529ed0ca6a196de3ec6937c11df0df1bc91c"}, - {file = "Pillow-9.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a54614049a18a2d6fe156e68e188da02a046a4a93cf24f373bffd977e943421"}, - {file = "Pillow-9.2.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:5aed7dde98403cd91d86a1115c78d8145c83078e864c1de1064f52e6feb61b20"}, - {file = "Pillow-9.2.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:13b725463f32df1bfeacbf3dd197fb358ae8ebcd8c5548faa75126ea425ccb60"}, - {file = "Pillow-9.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:808add66ea764ed97d44dda1ac4f2cfec4c1867d9efb16a33d158be79f32b8a4"}, - {file = "Pillow-9.2.0-cp39-cp39-win32.whl", hash = "sha256:337a74fd2f291c607d220c793a8135273c4c2ab001b03e601c36766005f36885"}, - {file = "Pillow-9.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:fac2d65901fb0fdf20363fbd345c01958a742f2dc62a8dd4495af66e3ff502a4"}, - {file = "Pillow-9.2.0-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:ad2277b185ebce47a63f4dc6302e30f05762b688f8dc3de55dbae4651872cdf3"}, - {file = "Pillow-9.2.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c7b502bc34f6e32ba022b4a209638f9e097d7a9098104ae420eb8186217ebbb"}, - {file = "Pillow-9.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d1f14f5f691f55e1b47f824ca4fdcb4b19b4323fe43cc7bb105988cad7496be"}, - {file = "Pillow-9.2.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:dfe4c1fedfde4e2fbc009d5ad420647f7730d719786388b7de0999bf32c0d9fd"}, - {file = "Pillow-9.2.0-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:f07f1f00e22b231dd3d9b9208692042e29792d6bd4f6639415d2f23158a80013"}, - {file = "Pillow-9.2.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1802f34298f5ba11d55e5bb09c31997dc0c6aed919658dfdf0198a2fe75d5490"}, - {file = "Pillow-9.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17d4cafe22f050b46d983b71c707162d63d796a1235cdf8b9d7a112e97b15bac"}, - {file = "Pillow-9.2.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:96b5e6874431df16aee0c1ba237574cb6dff1dcb173798faa6a9d8b399a05d0e"}, - {file = "Pillow-9.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:0030fdbd926fb85844b8b92e2f9449ba89607231d3dd597a21ae72dc7fe26927"}, - {file = "Pillow-9.2.0.tar.gz", hash = "sha256:75e636fd3e0fb872693f23ccb8a5ff2cd578801251f3a4f6854c6a5d437d3c04"}, + {file = "Pillow-9.3.0-1-cp37-cp37m-win32.whl", hash = "sha256:e6ea6b856a74d560d9326c0f5895ef8050126acfdc7ca08ad703eb0081e82b74"}, + {file = "Pillow-9.3.0-1-cp37-cp37m-win_amd64.whl", hash = "sha256:32a44128c4bdca7f31de5be641187367fe2a450ad83b833ef78910397db491aa"}, + {file = "Pillow-9.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:0b7257127d646ff8676ec8a15520013a698d1fdc48bc2a79ba4e53df792526f2"}, + {file = "Pillow-9.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b90f7616ea170e92820775ed47e136208e04c967271c9ef615b6fbd08d9af0e3"}, + {file = "Pillow-9.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68943d632f1f9e3dce98908e873b3a090f6cba1cbb1b892a9e8d97c938871fbe"}, + {file = "Pillow-9.3.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be55f8457cd1eac957af0c3f5ece7bc3f033f89b114ef30f710882717670b2a8"}, + {file = "Pillow-9.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d77adcd56a42d00cc1be30843d3426aa4e660cab4a61021dc84467123f7a00c"}, + {file = "Pillow-9.3.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:829f97c8e258593b9daa80638aee3789b7df9da5cf1336035016d76f03b8860c"}, + {file = "Pillow-9.3.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:801ec82e4188e935c7f5e22e006d01611d6b41661bba9fe45b60e7ac1a8f84de"}, + {file = "Pillow-9.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:871b72c3643e516db4ecf20efe735deb27fe30ca17800e661d769faab45a18d7"}, + {file = "Pillow-9.3.0-cp310-cp310-win32.whl", hash = "sha256:655a83b0058ba47c7c52e4e2df5ecf484c1b0b0349805896dd350cbc416bdd91"}, + {file = "Pillow-9.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:9f47eabcd2ded7698106b05c2c338672d16a6f2a485e74481f524e2a23c2794b"}, + {file = "Pillow-9.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:57751894f6618fd4308ed8e0c36c333e2f5469744c34729a27532b3db106ee20"}, + {file = "Pillow-9.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7db8b751ad307d7cf238f02101e8e36a128a6cb199326e867d1398067381bff4"}, + {file = "Pillow-9.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3033fbe1feb1b59394615a1cafaee85e49d01b51d54de0cbf6aa8e64182518a1"}, + {file = "Pillow-9.3.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22b012ea2d065fd163ca096f4e37e47cd8b59cf4b0fd47bfca6abb93df70b34c"}, + {file = "Pillow-9.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9a65733d103311331875c1dca05cb4606997fd33d6acfed695b1232ba1df193"}, + {file = "Pillow-9.3.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:502526a2cbfa431d9fc2a079bdd9061a2397b842bb6bc4239bb176da00993812"}, + {file = "Pillow-9.3.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:90fb88843d3902fe7c9586d439d1e8c05258f41da473952aa8b328d8b907498c"}, + {file = "Pillow-9.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:89dca0ce00a2b49024df6325925555d406b14aa3efc2f752dbb5940c52c56b11"}, + {file = "Pillow-9.3.0-cp311-cp311-win32.whl", hash = "sha256:3168434d303babf495d4ba58fc22d6604f6e2afb97adc6a423e917dab828939c"}, + {file = "Pillow-9.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:18498994b29e1cf86d505edcb7edbe814d133d2232d256db8c7a8ceb34d18cef"}, + {file = "Pillow-9.3.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:772a91fc0e03eaf922c63badeca75e91baa80fe2f5f87bdaed4280662aad25c9"}, + {file = "Pillow-9.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa4107d1b306cdf8953edde0534562607fe8811b6c4d9a486298ad31de733b2"}, + {file = "Pillow-9.3.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b4012d06c846dc2b80651b120e2cdd787b013deb39c09f407727ba90015c684f"}, + {file = "Pillow-9.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77ec3e7be99629898c9a6d24a09de089fa5356ee408cdffffe62d67bb75fdd72"}, + {file = "Pillow-9.3.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:6c738585d7a9961d8c2821a1eb3dcb978d14e238be3d70f0a706f7fa9316946b"}, + {file = "Pillow-9.3.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:828989c45c245518065a110434246c44a56a8b2b2f6347d1409c787e6e4651ee"}, + {file = "Pillow-9.3.0-cp37-cp37m-win32.whl", hash = "sha256:82409ffe29d70fd733ff3c1025a602abb3e67405d41b9403b00b01debc4c9a29"}, + {file = "Pillow-9.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:41e0051336807468be450d52b8edd12ac60bebaa97fe10c8b660f116e50b30e4"}, + {file = "Pillow-9.3.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:b03ae6f1a1878233ac620c98f3459f79fd77c7e3c2b20d460284e1fb370557d4"}, + {file = "Pillow-9.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4390e9ce199fc1951fcfa65795f239a8a4944117b5935a9317fb320e7767b40f"}, + {file = "Pillow-9.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40e1ce476a7804b0fb74bcfa80b0a2206ea6a882938eaba917f7a0f004b42502"}, + {file = "Pillow-9.3.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0a06a052c5f37b4ed81c613a455a81f9a3a69429b4fd7bb913c3fa98abefc20"}, + {file = "Pillow-9.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03150abd92771742d4a8cd6f2fa6246d847dcd2e332a18d0c15cc75bf6703040"}, + {file = "Pillow-9.3.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:15c42fb9dea42465dfd902fb0ecf584b8848ceb28b41ee2b58f866411be33f07"}, + {file = "Pillow-9.3.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:51e0e543a33ed92db9f5ef69a0356e0b1a7a6b6a71b80df99f1d181ae5875636"}, + {file = "Pillow-9.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3dd6caf940756101205dffc5367babf288a30043d35f80936f9bfb37f8355b32"}, + {file = "Pillow-9.3.0-cp38-cp38-win32.whl", hash = "sha256:f1ff2ee69f10f13a9596480335f406dd1f70c3650349e2be67ca3139280cade0"}, + {file = "Pillow-9.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:276a5ca930c913f714e372b2591a22c4bd3b81a418c0f6635ba832daec1cbcfc"}, + {file = "Pillow-9.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:73bd195e43f3fadecfc50c682f5055ec32ee2c933243cafbfdec69ab1aa87cad"}, + {file = "Pillow-9.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1c7c8ae3864846fc95f4611c78129301e203aaa2af813b703c55d10cc1628535"}, + {file = "Pillow-9.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e0918e03aa0c72ea56edbb00d4d664294815aa11291a11504a377ea018330d3"}, + {file = "Pillow-9.3.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0915e734b33a474d76c28e07292f196cdf2a590a0d25bcc06e64e545f2d146c"}, + {file = "Pillow-9.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af0372acb5d3598f36ec0914deed2a63f6bcdb7b606da04dc19a88d31bf0c05b"}, + {file = "Pillow-9.3.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:ad58d27a5b0262c0c19b47d54c5802db9b34d38bbf886665b626aff83c74bacd"}, + {file = "Pillow-9.3.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:97aabc5c50312afa5e0a2b07c17d4ac5e865b250986f8afe2b02d772567a380c"}, + {file = "Pillow-9.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9aaa107275d8527e9d6e7670b64aabaaa36e5b6bd71a1015ddd21da0d4e06448"}, + {file = "Pillow-9.3.0-cp39-cp39-win32.whl", hash = "sha256:bac18ab8d2d1e6b4ce25e3424f709aceef668347db8637c2296bcf41acb7cf48"}, + {file = "Pillow-9.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:b472b5ea442148d1c3e2209f20f1e0bb0eb556538690fa70b5e1f79fa0ba8dc2"}, + {file = "Pillow-9.3.0-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:ab388aaa3f6ce52ac1cb8e122c4bd46657c15905904b3120a6248b5b8b0bc228"}, + {file = "Pillow-9.3.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb8e7f2abee51cef77673be97760abff1674ed32847ce04b4af90f610144c7b"}, + {file = "Pillow-9.3.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca31dd6014cb8b0b2db1e46081b0ca7d936f856da3b39744aef499db5d84d02"}, + {file = "Pillow-9.3.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c7025dce65566eb6e89f56c9509d4f628fddcedb131d9465cacd3d8bac337e7e"}, + {file = "Pillow-9.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ebf2029c1f464c59b8bdbe5143c79fa2045a581ac53679733d3a91d400ff9efb"}, + {file = "Pillow-9.3.0-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:b59430236b8e58840a0dfb4099a0e8717ffb779c952426a69ae435ca1f57210c"}, + {file = "Pillow-9.3.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12ce4932caf2ddf3e41d17fc9c02d67126935a44b86df6a206cf0d7161548627"}, + {file = "Pillow-9.3.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae5331c23ce118c53b172fa64a4c037eb83c9165aba3a7ba9ddd3ec9fa64a699"}, + {file = "Pillow-9.3.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:0b07fffc13f474264c336298d1b4ce01d9c5a011415b79d4ee5527bb69ae6f65"}, + {file = "Pillow-9.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:073adb2ae23431d3b9bcbcff3fe698b62ed47211d0716b067385538a1b0f28b8"}, + {file = "Pillow-9.3.0.tar.gz", hash = "sha256:c935a22a557a560108d780f9a0fc426dd7459940dc54faa49d83249c8d3e760f"}, ] pluggy = [ {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -py = [ - {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, - {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, -] pydantic = [ {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, @@ -1209,12 +1257,12 @@ pyparsing = [ {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] pyright = [ - {file = "pyright-1.1.271-py3-none-any.whl", hash = "sha256:1d8da0eca3eade0500cf19da90aaf029263b6e339c6210c158c863758fe21de1"}, - {file = "pyright-1.1.271.tar.gz", hash = "sha256:67b18ae5e6a5e291120bda19880d0e78c9c05e3cb10bd4439921bacc0bfd780f"}, + {file = "pyright-1.1.280-py3-none-any.whl", hash = "sha256:25917a14d873252c5c2e6fdbec322888c0480f6db95068ff6459befa9af3c92a"}, + {file = "pyright-1.1.280.tar.gz", hash = "sha256:4bcb167251419b3b736137b0535cb6bbfb5bef16eb74057eaf3ccaccb01d74c1"}, ] pytest = [ - {file = "pytest-7.1.3-py3-none-any.whl", hash = "sha256:1377bda3466d70b55e3f5cecfa55bb7cfcf219c7964629b967c37cf0bda818b7"}, - {file = "pytest-7.1.3.tar.gz", hash = "sha256:4f365fec2dff9c1162f834d9f18af1ba13062db0c708bf7b946f8a5c76180c39"}, + {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, + {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, ] pytest-cov = [ {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, @@ -1267,8 +1315,8 @@ PyYAML = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] rich = [ - {file = "rich-11.2.0-py3-none-any.whl", hash = "sha256:d5f49ad91fb343efcae45a2b2df04a9755e863e50413623ab8c9e74f05aee52b"}, - {file = "rich-11.2.0.tar.gz", hash = "sha256:1a6266a5738115017bb64a66c59c717e7aa047b3ae49a011ede4abdeffc6536e"}, + {file = "rich-12.6.0-py3-none-any.whl", hash = "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e"}, + {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, ] scipy = [ {file = "scipy-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a15a1f3fc0abff33e792d6049161b7795909b40b97c6cc2934ed54384017ab76"}, @@ -1292,21 +1340,39 @@ scipy = [ {file = "scipy-1.6.1.tar.gz", hash = "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11"}, ] setuptools = [ - {file = "setuptools-65.3.0-py3-none-any.whl", hash = "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82"}, - {file = "setuptools-65.3.0.tar.gz", hash = "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57"}, + {file = "setuptools-65.6.0-py3-none-any.whl", hash = "sha256:6211d2f5eddad8757bd0484923ca7c0a6302ebc4ab32ea5e94357176e0ca0840"}, + {file = "setuptools-65.6.0.tar.gz", hash = "sha256:d1eebf881c6114e51df1664bc2c9133d022f78d12d5f4f665b9191f084e2862d"}, ] setuptools-scm = [ {file = "setuptools_scm-6.4.2-py3-none-any.whl", hash = "sha256:acea13255093849de7ccb11af9e1fb8bde7067783450cee9ef7a93139bddf6d4"}, {file = "setuptools_scm-6.4.2.tar.gz", hash = "sha256:6833ac65c6ed9711a4d5d2266f8024cfa07c533a0e55f4c12f6eff280a5a9e30"}, ] shtab = [ - {file = "shtab-1.5.6-py2.py3-none-any.whl", hash = "sha256:59cc8b291ce43cea24f9c66a633d054967893bd6a55dc8bfcc65849073ee01d0"}, - {file = "shtab-1.5.6.tar.gz", hash = "sha256:0b394e4625cb0632ad3b26a51c16ee35628b707b137108502a71473abb2be4c3"}, + {file = "shtab-1.5.8-py2.py3-none-any.whl", hash = "sha256:1d326ea131edbba96e0489470a2c969da1976a586a2d9376bb4923a6fe8eaae0"}, + {file = "shtab-1.5.8.tar.gz", hash = "sha256:1f944e2e33c1554be69e6b26ef638ba3b516ac9449fdd2a40d197f9061c8bed8"}, ] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +tensorstore = [ + {file = "tensorstore-0.1.28-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:deb32f1e74ab7b836ecec02a759a558fc57522a8dea0db4f19a01a78e93abab5"}, + {file = "tensorstore-0.1.28-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bd8615308ad3e29111238dfdd3aca1f3fb79b55c3dac1c04c96c8a319985addf"}, + {file = "tensorstore-0.1.28-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee16297a9493ed0d6d1000f216e79d29a0b80e7e97646e8520afd44138165839"}, + {file = "tensorstore-0.1.28-cp310-cp310-win_amd64.whl", hash = "sha256:b6f7e0c7455d9e164e6143714519e7c96cc4166b61ed7d268a8ab84a77e0d593"}, + {file = "tensorstore-0.1.28-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:1c53e1e0e7996b8ba41854135d748c2bc8802b94a79122b56448aeb1f77efd9f"}, + {file = "tensorstore-0.1.28-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bd122a44db81369808c84b7739ad2d8453f89c548c5d61d62f2b1f3331dbe9f"}, + {file = "tensorstore-0.1.28-cp37-cp37m-win_amd64.whl", hash = "sha256:197159da3ee97c08d8769fa151ff167328108bf07a13886f688b21f64abfb7d6"}, + {file = "tensorstore-0.1.28-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:e480da4f95f7c95302c4b3e2a7fc32b8cfe1aa51742d72f84a335e49462a2465"}, + {file = "tensorstore-0.1.28-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7948ead0d5a55303ffa97a35659aa56a20c60bd89d9487643f51ac8d2e19b52c"}, + {file = "tensorstore-0.1.28-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:818aeb015b249069a686fb20f0c0b71663410ac6411cee88df18b8e05b567cbc"}, + {file = "tensorstore-0.1.28-cp38-cp38-win_amd64.whl", hash = "sha256:6719eaba4ec2692d890c3c67c389fcae073d40d9e8c27e2dabd27fc9fd745e5c"}, + {file = "tensorstore-0.1.28-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:d7c86f63a6d4d7a84e3201ba9746ca51015612d375d77b9302eb93e9b88bc7aa"}, + {file = "tensorstore-0.1.28-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dc82cbb77255fd2f7aa191f1e030f13276bf7272ea1027fcc6900537b30bfbcb"}, + {file = "tensorstore-0.1.28-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4de09e3fae30f8a9ab7648600c78e109ddd1d14af4280e055439fdd52a6dca9"}, + {file = "tensorstore-0.1.28-cp39-cp39-win_amd64.whl", hash = "sha256:f9270586401ee60ff79a4cd54c62ab5b06831d69b561df1fb9ebbeade4d4929c"}, + {file = "tensorstore-0.1.28.tar.gz", hash = "sha256:cd8d8185136632c58edcd7cf4c43d301bf8ad61a197c632cb495d7d33b7be04e"}, +] tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, @@ -1316,26 +1382,27 @@ toolz = [ {file = "toolz-0.12.0.tar.gz", hash = "sha256:88c570861c440ee3f2f6037c4654613228ff40c93a6c25e0eba70d17282c6194"}, ] torch = [ - {file = "torch-1.12.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:9c038662db894a23e49e385df13d47b2a777ffd56d9bcd5b832593fab0a7e286"}, - {file = "torch-1.12.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:4e1b9c14cf13fd2ab8d769529050629a0e68a6fc5cb8e84b4a3cc1dd8c4fe541"}, - {file = "torch-1.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:e9c8f4a311ac29fc7e8e955cfb7733deb5dbe1bdaabf5d4af2765695824b7e0d"}, - {file = "torch-1.12.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:976c3f997cea38ee91a0dd3c3a42322785414748d1761ef926b789dfa97c6134"}, - {file = "torch-1.12.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:68104e4715a55c4bb29a85c6a8d57d820e0757da363be1ba680fa8cc5be17b52"}, - {file = "torch-1.12.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:743784ccea0dc8f2a3fe6a536bec8c4763bd82c1352f314937cb4008d4805de1"}, - {file = "torch-1.12.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b5dbcca369800ce99ba7ae6dee3466607a66958afca3b740690d88168752abcf"}, - {file = "torch-1.12.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f3b52a634e62821e747e872084ab32fbcb01b7fa7dbb7471b6218279f02a178a"}, - {file = "torch-1.12.1-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:8a34a2fbbaa07c921e1b203f59d3d6e00ed379f2b384445773bd14e328a5b6c8"}, - {file = "torch-1.12.1-cp37-none-macosx_11_0_arm64.whl", hash = "sha256:42f639501928caabb9d1d55ddd17f07cd694de146686c24489ab8c615c2871f2"}, - {file = "torch-1.12.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0b44601ec56f7dd44ad8afc00846051162ef9c26a8579dda0a02194327f2d55e"}, - {file = "torch-1.12.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:cd26d8c5640c3a28c526d41ccdca14cf1cbca0d0f2e14e8263a7ac17194ab1d2"}, - {file = "torch-1.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:42e115dab26f60c29e298559dbec88444175528b729ae994ec4c65d56fe267dd"}, - {file = "torch-1.12.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:a8320ba9ad87e80ca5a6a016e46ada4d1ba0c54626e135d99b2129a4541c509d"}, - {file = "torch-1.12.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:03e31c37711db2cd201e02de5826de875529e45a55631d317aadce2f1ed45aa8"}, - {file = "torch-1.12.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:9b356aea223772cd754edb4d9ecf2a025909b8615a7668ac7d5130f86e7ec421"}, - {file = "torch-1.12.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:6cf6f54b43c0c30335428195589bd00e764a6d27f3b9ba637aaa8c11aaf93073"}, - {file = "torch-1.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:f00c721f489089dc6364a01fd84906348fe02243d0af737f944fddb36003400d"}, - {file = "torch-1.12.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:bfec2843daa654f04fda23ba823af03e7b6f7650a873cdb726752d0e3718dada"}, - {file = "torch-1.12.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:69fe2cae7c39ccadd65a123793d30e0db881f1c1927945519c5c17323131437e"}, + {file = "torch-1.13.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:f68edfea71ade3862039ba66bcedf954190a2db03b0c41a9b79afd72210abd97"}, + {file = "torch-1.13.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:d2d2753519415d154de4d3e64d2eaaeefdba6b6fd7d69d5ffaef595988117700"}, + {file = "torch-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:6c227c16626e4ce766cca5351cc62a2358a11e8e466410a298487b9dff159eb1"}, + {file = "torch-1.13.0-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:49a949b8136b32b2ec0724cbf4c6678b54e974b7d68f19f1231eea21cde5c23b"}, + {file = "torch-1.13.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:0fdd38c96230947b1ed870fed4a560252f8d23c3a2bf4dab9d2d42b18f2e67c8"}, + {file = "torch-1.13.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:43db0723fc66ad6486f86dc4890c497937f7cd27429f28f73fb7e4d74b7482e2"}, + {file = "torch-1.13.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e643ac8d086706e82f77b5d4dfcf145a9dd37b69e03e64177fc23821754d2ed7"}, + {file = "torch-1.13.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:bb33a911460475d1594a8c8cb73f58c08293211760796d99cae8c2509b86d7f1"}, + {file = "torch-1.13.0-cp37-cp37m-win_amd64.whl", hash = "sha256:220325d0f4e69ee9edf00c04208244ef7cf22ebce083815ce272c7491f0603f5"}, + {file = "torch-1.13.0-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:cd1e67db6575e1b173a626077a54e4911133178557aac50683db03a34e2b636a"}, + {file = "torch-1.13.0-cp37-none-macosx_11_0_arm64.whl", hash = "sha256:9197ec216833b836b67e4d68e513d31fb38d9789d7cd998a08fba5b499c38454"}, + {file = "torch-1.13.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:fa768432ce4b8ffa29184c79a3376ab3de4a57b302cdf3c026a6be4c5a8ab75b"}, + {file = "torch-1.13.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:635dbb99d981a6483ca533b3dc7be18ef08dd9e1e96fb0bb0e6a99d79e85a130"}, + {file = "torch-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:857c7d5b1624c5fd979f66d2b074765733dba3f5e1cc97b7d6909155a2aae3ce"}, + {file = "torch-1.13.0-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:ef934a21da6f6a516d0a9c712a80d09c56128abdc6af8dc151bee5199b4c3b4e"}, + {file = "torch-1.13.0-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:f01a9ae0d4b69d2fc4145e8beab45b7877342dddbd4838a7d3c11ca7f6680745"}, + {file = "torch-1.13.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:9ac382cedaf2f70afea41380ad8e7c06acef6b5b7e2aef3971cdad666ca6e185"}, + {file = "torch-1.13.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:e20df14d874b024851c58e8bb3846249cb120e677f7463f60c986e3661f88680"}, + {file = "torch-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:4a378f5091307381abfb30eb821174e12986f39b1cf7c4522bf99155256819eb"}, + {file = "torch-1.13.0-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:922a4910613b310fbeb87707f00cb76fec328eb60cc1349ed2173e7c9b6edcd8"}, + {file = "torch-1.13.0-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:47fe6228386bff6d74319a2ffe9d4ed943e6e85473d78e80502518c607d644d2"}, ] typed-ast = [ {file = "typed_ast-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4"}, @@ -1364,10 +1431,14 @@ typed-ast = [ {file = "typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"}, ] typing-extensions = [ - {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, - {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, +] +wheel = [ + {file = "wheel-0.38.4-py3-none-any.whl", hash = "sha256:b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8"}, + {file = "wheel-0.38.4.tar.gz", hash = "sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac"}, ] zipp = [ - {file = "zipp-3.8.1-py3-none-any.whl", hash = "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009"}, - {file = "zipp-3.8.1.tar.gz", hash = "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2"}, + {file = "zipp-3.10.0-py3-none-any.whl", hash = "sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1"}, + {file = "zipp-3.10.0.tar.gz", hash = "sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8"}, ] diff --git a/tests/test_nested.py b/tests/test_nested.py index 358bcb8c5..884be2737 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -3,7 +3,7 @@ import pytest from frozendict import frozendict # type: ignore -from typing_extensions import Annotated +from typing_extensions import Annotated, Literal import tyro @@ -768,3 +768,58 @@ class Wrapper: tyro.cli(Wrapper, args="supertype:type-a --supertype.subtype.data 1".split(" ")) == Wrapper() ) + + +def test_nested_in_subparser_override_with_default(): + @dataclasses.dataclass + class Mnist: + binary: bool = False + """Set to load binary version of MNIST dataset.""" + + @dataclasses.dataclass + class ImageNet: + subset: Literal[50, 100, 1000] + """Choose between ImageNet-50, ImageNet-100, ImageNet-1000, etc.""" + + # Possible optimizer configurations. + + Selector = tyro.extras.subcommand_type_from_defaults( + { + "m": Mnist(), + "i": ImageNet(50), + } + ) + + @dataclasses.dataclass + class DatasetContainer: + dataset: Selector = Mnist() # type: ignore + + @dataclasses.dataclass + class Adam: + learning_rate: float = 1e-3 + betas: Tuple[float, float] = (0.9, 0.999) + container: DatasetContainer = DatasetContainer() + + @dataclasses.dataclass + class Sgd: + learning_rate: float = 3e-4 + container: DatasetContainer = DatasetContainer() + + # Train script. + + Optimizers = tyro.extras.subcommand_type_from_defaults( + {"adam": Adam(container=DatasetContainer(ImageNet(50))), "sgd": Sgd()}, + prefix_names=False, + ) + + @tyro.conf.configure(tyro.conf.OmitSubcommandPrefixes) + def train( + optimizer: Optimizers = Adam(container=DatasetContainer(ImageNet(50))), # type: ignore + ) -> Union[Adam, Sgd]: + return optimizer + + assert tyro.cli(train, args=[]) == Adam(container=DatasetContainer(ImageNet(50))) + assert tyro.cli(train, args=["adam"]) == Adam( + container=DatasetContainer(ImageNet(50)) + ) + assert tyro.cli(train, args=["sgd"]) == Sgd(container=DatasetContainer(Mnist())) diff --git a/tyro/_calling.py b/tyro/_calling.py index a390d1e45..e338601aa 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -24,7 +24,6 @@ def call_from_args( default_instance: Union[T, _fields.NonpropagatingMissingType], value_from_prefixed_field_name: Dict[str, Any], field_name_prefix: str, - subparser_def_from_prefixed_field_name: Dict[str, _parsers.SubparsersSpecification], ) -> Tuple[T, Set[str]]: """Call `f` with arguments specified by a dictionary of values from argparse. @@ -104,14 +103,13 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: field.default, value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, - subparser_def_from_prefixed_field_name=subparser_def_from_prefixed_field_name, ) consumed_keywords |= consumed_keywords_child else: # Unions over dataclasses (subparsers). This is the only other option. - assert parser_definition.subparsers is not None - subparser_def = subparser_def_from_prefixed_field_name[prefixed_field_name] - + subparser_def = parser_definition.subparsers_from_prefix[ + prefixed_field_name + ] subparser_dest = _strings.make_subparser_dest(name=prefixed_field_name) consumed_keywords.add(subparser_dest) if subparser_dest in value_from_prefixed_field_name: @@ -159,7 +157,6 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: field.default if type(field.default) is chosen_f else None, value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, - subparser_def_from_prefixed_field_name=subparser_def_from_prefixed_field_name, ) consumed_keywords |= consumed_keywords_child diff --git a/tyro/_cli.py b/tyro/_cli.py index 3f9fcfdad..e6f0a3e61 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -321,19 +321,6 @@ def fix_arg(arg: str) -> str: for k, v in value_from_prefixed_field_name.items() } - # Build a map from subparser names to definitions. - subparser_def_from_prefixed_field_name = {} - - def _cache_subparsers(parser_definition: _parsers.ParserSpecification) -> None: - subparsers = parser_definition.subparsers - if subparsers is None: - return - subparser_def_from_prefixed_field_name[subparsers.prefix] = subparsers - for p in subparsers.parser_from_name.values(): - _cache_subparsers(p) - - _cache_subparsers(parser_definition) - try: # Attempt to call `f` using whatever was passed in. out, consumed_keywords = _calling.call_from_args( @@ -342,7 +329,6 @@ def _cache_subparsers(parser_definition: _parsers.ParserSpecification) -> None: default_instance_internal, value_from_prefixed_field_name, field_name_prefix="", - subparser_def_from_prefixed_field_name=subparser_def_from_prefixed_field_name, ) except _calling.InstantiationError as e: # Emulate argparse's error behavior when invalid arguments are passed in. diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 7ca39daaf..14761e476 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -42,7 +42,14 @@ class ParserSpecification: description: str args: List[_arguments.ArgumentDefinition] helptext_from_nested_class_field_name: Dict[str, Optional[str]] + + # We have two mechanics for tracking subparser groups: + # - A single subparser group, which is what gets added in the tree structure built + # by the argparse parser. subparsers: Optional[SubparsersSpecification] + # - A set of subparser groups, which reflect the tree structure built by the + # hierarchy of a nested config structure. + subparsers_from_prefix: Dict[str, SubparsersSpecification] prefix: str has_required_args: bool consolidate_subcommand_args: bool @@ -87,7 +94,9 @@ def from_callable_or_type( has_required_args = False args = [] helptext_from_nested_class_field_name = {} + subparsers = None + subparsers_from_prefix = {} field_list = _fields.field_list_from_callable( f=f, default_instance=default_instance @@ -129,10 +138,10 @@ def from_callable_or_type( ): # Don't make a subparser. field = dataclasses.replace(field, typ=type(field.default)) - elif subparsers is None: - subparsers = subparsers_attempt - continue else: + subparsers_from_prefix[ + subparsers_attempt.prefix + ] = subparsers_attempt subparsers = add_subparsers_to_leaves( subparsers, subparsers_attempt ) @@ -166,12 +175,11 @@ def from_callable_or_type( # Include nested subparsers. if nested_parser.subparsers is not None: - subparsers = ( - nested_parser.subparsers - if subparsers is None - else add_subparsers_to_leaves( - subparsers, nested_parser.subparsers - ) + subparsers_from_prefix[ + nested_parser.subparsers.prefix + ] = nested_parser.subparsers + subparsers = add_subparsers_to_leaves( + subparsers, nested_parser.subparsers ) # Include nested strings. @@ -214,6 +222,7 @@ def from_callable_or_type( args=args, helptext_from_nested_class_field_name=helptext_from_nested_class_field_name, subparsers=subparsers, + subparsers_from_prefix=subparsers_from_prefix, prefix=prefix, has_required_args=has_required_args, consolidate_subcommand_args=consolidate_subcommand_args, @@ -228,8 +237,10 @@ def apply( parser.description = self.description # Create subparser tree. + subparser_group = None if self.subparsers is not None: leaves = self.subparsers.apply(parser) + subparser_group = parser._action_groups.pop() else: leaves = (parser,) @@ -241,8 +252,16 @@ def apply( else: self.apply_args(parser) + if subparser_group is not None: + parser._action_groups.append(subparser_group) + # Break some API boundaries to rename the "optional arguments" => "arguments". - assert parser._action_groups[1].title == "optional arguments" + assert parser._action_groups[1].title in ( + # python <= 3.9 + "optional arguments", + # python >= 3.10 + "options", + ) parser._action_groups[1].title = "arguments" return leaves @@ -431,7 +450,6 @@ def from_field( subcommand_config = dataclasses.replace( subcommand_config, default=field.default ) - subparser = ParserSpecification.from_callable_or_type( # Recursively apply markers. Annotated.__class_getitem__((option,) + tuple(field.markers)) # type: ignore @@ -493,8 +511,6 @@ def from_field( prefix=prefix, required=required, default_instance=field.default, - # if field.default not in _fields.MISSING_SINGLETONS - # else None, can_be_none=options != options_no_none, ) From 86406a348677a2a4a2f2bb73539f6b90fb6502b8 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 20 Nov 2022 17:23:27 -0800 Subject: [PATCH 235/491] Add workaround for flax typing error https://github.com/google/flax/issues/2636 --- docs/source/examples/04_additional/07_flax.rst | 3 ++- examples/04_additional/07_flax.py | 3 ++- tests/test_flax_ignore_py310.py | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/source/examples/04_additional/07_flax.rst b/docs/source/examples/04_additional/07_flax.rst index d9116016b..2d47b727d 100644 --- a/docs/source/examples/04_additional/07_flax.rst +++ b/docs/source/examples/04_additional/07_flax.rst @@ -15,12 +15,13 @@ directly from ``tyro.cli``. from flax import linen as nn + from flax.linen import Module # https://github.com/google/flax/issues/2636 from jax import numpy as jnp import tyro - class Classifier(nn.Module): + class Classifier(Module): layers: int """Layers in our network.""" units: int = 32 diff --git a/examples/04_additional/07_flax.py b/examples/04_additional/07_flax.py index 6707964ee..42364965d 100644 --- a/examples/04_additional/07_flax.py +++ b/examples/04_additional/07_flax.py @@ -9,12 +9,13 @@ """ from flax import linen as nn +from flax.linen import Module # https://github.com/google/flax/issues/2636 from jax import numpy as jnp import tyro -class Classifier(nn.Module): +class Classifier(Module): layers: int """Layers in our network.""" units: int = 32 diff --git a/tests/test_flax_ignore_py310.py b/tests/test_flax_ignore_py310.py index 9642692e2..7f057d3a0 100644 --- a/tests/test_flax_ignore_py310.py +++ b/tests/test_flax_ignore_py310.py @@ -2,6 +2,7 @@ import jax import pytest from flax import linen as nn +from flax.linen import Module # https://github.com/google/flax/issues/2636 from jax import numpy as jnp import tyro From f66c7737a51194fb2046e7789d017e014fe0b37e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 20 Nov 2022 23:13:31 -0800 Subject: [PATCH 236/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1d869fc0b..bf1c89d58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.34" +version = "0.3.35" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] From 9158c66532aa2b65b95d7be09f00c7722f4a83df Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Sun, 27 Nov 2022 11:14:44 +0800 Subject: [PATCH 237/491] Fix typos (#22) Found via `codespell -S .mypy_cache` --- docs/source/examples/02_nesting/04_nesting_in_containers.rst | 2 +- docs/source/examples/03_config_systems/01_base_configs.rst | 2 +- examples/02_nesting/04_nesting_in_containers.py | 2 +- examples/03_config_systems/01_base_configs.py | 2 +- tests/test_collections.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/examples/02_nesting/04_nesting_in_containers.rst b/docs/source/examples/02_nesting/04_nesting_in_containers.rst index da896fb95..8a8089376 100644 --- a/docs/source/examples/02_nesting/04_nesting_in_containers.rst +++ b/docs/source/examples/02_nesting/04_nesting_in_containers.rst @@ -7,7 +7,7 @@ Nesting in Containers Structures can be nested inside of standard containers. -Note that lengths must be inferrable, either via a fixed-length tuple annotation or by +Note that lengths must be inferable, either via a fixed-length tuple annotation or by parsing default values. diff --git a/docs/source/examples/03_config_systems/01_base_configs.rst b/docs/source/examples/03_config_systems/01_base_configs.rst index a5ad93898..0ae974006 100644 --- a/docs/source/examples/03_config_systems/01_base_configs.rst +++ b/docs/source/examples/03_config_systems/01_base_configs.rst @@ -11,7 +11,7 @@ use the CLI to either override (existing) or fill in (missing) values. Note that our interfaces don't prescribe any of the mechanics used for storing base configurations. A Hydra-style YAML approach could just as easily -be used for the config libary (although we generally prefer to avoid YAMLs; staying in +be used for the config library (although we generally prefer to avoid YAMLs; staying in Python is convenient for autocompletion and type checking). diff --git a/examples/02_nesting/04_nesting_in_containers.py b/examples/02_nesting/04_nesting_in_containers.py index 5b699d28c..bd8e02694 100644 --- a/examples/02_nesting/04_nesting_in_containers.py +++ b/examples/02_nesting/04_nesting_in_containers.py @@ -2,7 +2,7 @@ Structures can be nested inside of standard containers. -Note that lengths must be inferrable, either via a fixed-length tuple annotation or by +Note that lengths must be inferable, either via a fixed-length tuple annotation or by parsing default values. diff --git a/examples/03_config_systems/01_base_configs.py b/examples/03_config_systems/01_base_configs.py index 4a0830afe..8281b18c4 100644 --- a/examples/03_config_systems/01_base_configs.py +++ b/examples/03_config_systems/01_base_configs.py @@ -6,7 +6,7 @@ Note that our interfaces don't prescribe any of the mechanics used for storing base configurations. A Hydra-style YAML approach could just as easily -be used for the config libary (although we generally prefer to avoid YAMLs; staying in +be used for the config library (although we generally prefer to avoid YAMLs; staying in Python is convenient for autocompletion and type checking). Usage: diff --git a/tests/test_collections.py b/tests/test_collections.py index e4eab8637..41df3c2ec 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -311,7 +311,7 @@ class A: def test_nested_optional_types(): """We support "None" as a special-case keyword. (note: this is a bit weird because - Optional[str] might interprete "None" as either a string or an actual `None` + Optional[str] might interpret "None" as either a string or an actual `None` value)""" @dataclasses.dataclass From 5f2a231bc60c9a27a8a1e78af21f07fe8673dc03 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 24 Nov 2022 17:03:02 -0800 Subject: [PATCH 238/491] Disentangle dest prefix and name prefix --- tyro/_arguments.py | 15 +++++++++------ tyro/_calling.py | 2 +- tyro/_parsers.py | 15 ++++++++------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/tyro/_arguments.py b/tyro/_arguments.py index f381f3020..13655f1ac 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -39,7 +39,8 @@ class ArgumentDefinition: """Structure containing everything needed to define an argument.""" - prefix: str # Prefix for nesting. + dest_prefix: str # True prefix. (eg for the argument's dest field) + name_prefix: str # User-facing prefix. subcommand_prefix: str # Prefix for nesting. field: _fields.FieldDefinition type_from_typevar: Dict[TypeVar, Type] @@ -231,7 +232,7 @@ def _rule_recursive_instantiator_from_type( if arg.field.default in _fields.MISSING_SINGLETONS: raise _instantiators.UnsupportedTypeAnnotationError( "Unsupported type annotation for the field" - f" {_strings.make_field_name([arg.prefix, arg.field.name])}. To" + f" {_strings.make_field_name([arg.name_prefix, arg.field.name])}. To" " suppress this error, assign the field a default value." ) from e else: @@ -363,15 +364,17 @@ def _rule_set_name_or_flag_and_dest( ) -> LoweredArgumentDefinition: # Positional arguments: no -- prefix. if arg.field.is_positional(): - name_or_flag = _strings.make_field_name([arg.prefix, arg.field.name]) + name_or_flag = _strings.make_field_name([arg.name_prefix, arg.field.name]) # Negated booleans. elif lowered.action == "store_false": name_or_flag = "--" + _strings.make_field_name( - [arg.prefix, "no-" + arg.field.name] + [arg.name_prefix, "no-" + arg.field.name] ) # Prefix keyword arguments with --. else: - name_or_flag = "--" + _strings.make_field_name([arg.prefix, arg.field.name]) + name_or_flag = "--" + _strings.make_field_name( + [arg.name_prefix, arg.field.name] + ) # Strip. if name_or_flag.startswith("--") and arg.subcommand_prefix != "": @@ -384,7 +387,7 @@ def _rule_set_name_or_flag_and_dest( return dataclasses.replace( lowered, name_or_flag=name_or_flag, - dest=_strings.make_field_name([arg.prefix, arg.field.name]), + dest=_strings.make_field_name([arg.dest_prefix, arg.field.name]), ) diff --git a/tyro/_calling.py b/tyro/_calling.py index e338601aa..7dd2329f4 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -45,7 +45,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: arg_from_prefixed_field_name: Dict[str, _arguments.ArgumentDefinition] = {} for arg in parser_definition.args: arg_from_prefixed_field_name[ - _strings.make_field_name([arg.prefix, arg.field.name]) + _strings.make_field_name([arg.dest_prefix, arg.field.name]) ] = arg for field in _fields.field_list_from_callable( diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 14761e476..d6194a746 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -203,7 +203,8 @@ def from_callable_or_type( # (3) Handle primitive or fixed types. These produce a single argument! arg = _arguments.ArgumentDefinition( - prefix=prefix, + dest_prefix=prefix, + name_prefix=prefix, subcommand_prefix=subcommand_prefix, field=field, type_from_typevar=type_from_typevar, @@ -291,11 +292,11 @@ def format_group_name(prefix: str) -> str: for arg in self.args: if ( arg.lowered.help is not argparse.SUPPRESS - and arg.prefix not in group_from_prefix + and arg.dest_prefix not in group_from_prefix ): - description = self.helptext_from_nested_class_field_name.get(arg.prefix) - group_from_prefix[arg.prefix] = parser.add_argument_group( - format_group_name(arg.prefix), + description = self.helptext_from_nested_class_field_name.get(arg.dest_prefix) + group_from_prefix[arg.dest_prefix] = parser.add_argument_group( + format_group_name(arg.dest_prefix), description=description, ) @@ -305,8 +306,8 @@ def format_group_name(prefix: str) -> str: arg.add_argument(positional_group) continue - if arg.prefix in group_from_prefix: - arg.add_argument(group_from_prefix[arg.prefix]) + if arg.dest_prefix in group_from_prefix: + arg.add_argument(group_from_prefix[arg.dest_prefix]) else: # Suppressed argument: still need to add them, but they won't show up in # the helptext so it doesn't matter which group. From 77948db182c3a445395ab3f8eade9594c6aa776d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 28 Nov 2022 00:53:40 -0800 Subject: [PATCH 239/491] Cleanup --- poetry.lock | 68 +++++---- pyproject.toml | 2 +- tests/test_attrs.py | 8 +- tests/test_collections.py | 74 ++++----- tests/test_conf.py | 42 ++--- tests/test_dcargs.py | 91 +++++------ tests/test_dict_namedtuple.py | 34 ++--- tests/test_errors.py | 28 ++-- tests/test_generics_and_serialization.py | 44 +++--- tests/test_helptext.py | 64 ++++---- tests/test_initvar_ignore_py37.py | 2 +- tests/test_missing.py | 8 +- tests/test_nested.py | 54 +++---- tests/test_nested_in_containers.py | 48 +++--- tests/test_pydantic.py | 4 +- tyro/_arguments.py | 2 +- tyro/_fields.py | 185 +++++++++++------------ tyro/_instantiators.py | 22 +-- tyro/_parsers.py | 12 +- tyro/_resolver.py | 6 +- tyro/conf/_markers.py | 13 +- tyro/extras/_serialization.py | 14 +- 22 files changed, 421 insertions(+), 404 deletions(-) diff --git a/poetry.lock b/poetry.lock index a32170521..2563d063e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -273,11 +273,11 @@ python-versions = "*" [[package]] name = "mypy" -version = "0.971" +version = "0.991" description = "Optional static typing for Python" -category = "dev" +category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] mypy-extensions = ">=0.4.3" @@ -287,6 +287,7 @@ typing-extensions = ">=3.10" [package.extras] dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] python2 = ["typed-ast (>=1.4.0,<2)"] reports = ["lxml"] @@ -294,7 +295,7 @@ reports = ["lxml"] name = "mypy-extensions" version = "0.4.3" description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "dev" +category = "main" optional = false python-versions = "*" @@ -642,7 +643,7 @@ numpy = ">=1.16.0" name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" @@ -676,7 +677,7 @@ opt-einsum = ["opt-einsum (>=3.3)"] name = "typed-ast" version = "1.5.4" description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -714,7 +715,7 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "49867c0eda45ffab0a881ae1181315655a200d447d3ca4f85bc25713acb4a699" +content-hash = "08c358613cee83d6f1992fd715ed61144442e3bebdc746f79a1afe30fef7b58a" [metadata.files] absl-py = [ @@ -1048,29 +1049,36 @@ msgpack = [ {file = "msgpack-1.0.4.tar.gz", hash = "sha256:f5d869c18f030202eb412f08b28d2afeea553d6613aee89e200d7aca7ef01f5f"}, ] mypy = [ - {file = "mypy-0.971-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f2899a3cbd394da157194f913a931edfd4be5f274a88041c9dc2d9cdcb1c315c"}, - {file = "mypy-0.971-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:98e02d56ebe93981c41211c05adb630d1d26c14195d04d95e49cd97dbc046dc5"}, - {file = "mypy-0.971-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:19830b7dba7d5356d3e26e2427a2ec91c994cd92d983142cbd025ebe81d69cf3"}, - {file = "mypy-0.971-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:02ef476f6dcb86e6f502ae39a16b93285fef97e7f1ff22932b657d1ef1f28655"}, - {file = "mypy-0.971-cp310-cp310-win_amd64.whl", hash = "sha256:25c5750ba5609a0c7550b73a33deb314ecfb559c350bb050b655505e8aed4103"}, - {file = "mypy-0.971-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d3348e7eb2eea2472db611486846742d5d52d1290576de99d59edeb7cd4a42ca"}, - {file = "mypy-0.971-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3fa7a477b9900be9b7dd4bab30a12759e5abe9586574ceb944bc29cddf8f0417"}, - {file = "mypy-0.971-cp36-cp36m-win_amd64.whl", hash = "sha256:2ad53cf9c3adc43cf3bea0a7d01a2f2e86db9fe7596dfecb4496a5dda63cbb09"}, - {file = "mypy-0.971-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:855048b6feb6dfe09d3353466004490b1872887150c5bb5caad7838b57328cc8"}, - {file = "mypy-0.971-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:23488a14a83bca6e54402c2e6435467a4138785df93ec85aeff64c6170077fb0"}, - {file = "mypy-0.971-cp37-cp37m-win_amd64.whl", hash = "sha256:4b21e5b1a70dfb972490035128f305c39bc4bc253f34e96a4adf9127cf943eb2"}, - {file = "mypy-0.971-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:9796a2ba7b4b538649caa5cecd398d873f4022ed2333ffde58eaf604c4d2cb27"}, - {file = "mypy-0.971-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5a361d92635ad4ada1b1b2d3630fc2f53f2127d51cf2def9db83cba32e47c856"}, - {file = "mypy-0.971-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b793b899f7cf563b1e7044a5c97361196b938e92f0a4343a5d27966a53d2ec71"}, - {file = "mypy-0.971-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d1ea5d12c8e2d266b5fb8c7a5d2e9c0219fedfeb493b7ed60cd350322384ac27"}, - {file = "mypy-0.971-cp38-cp38-win_amd64.whl", hash = "sha256:23c7ff43fff4b0df93a186581885c8512bc50fc4d4910e0f838e35d6bb6b5e58"}, - {file = "mypy-0.971-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1f7656b69974a6933e987ee8ffb951d836272d6c0f81d727f1d0e2696074d9e6"}, - {file = "mypy-0.971-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d2022bfadb7a5c2ef410d6a7c9763188afdb7f3533f22a0a32be10d571ee4bbe"}, - {file = "mypy-0.971-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ef943c72a786b0f8d90fd76e9b39ce81fb7171172daf84bf43eaf937e9f220a9"}, - {file = "mypy-0.971-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d744f72eb39f69312bc6c2abf8ff6656973120e2eb3f3ec4f758ed47e414a4bf"}, - {file = "mypy-0.971-cp39-cp39-win_amd64.whl", hash = "sha256:77a514ea15d3007d33a9e2157b0ba9c267496acf12a7f2b9b9f8446337aac5b0"}, - {file = "mypy-0.971-py3-none-any.whl", hash = "sha256:0d054ef16b071149917085f51f89555a576e2618d5d9dd70bd6eea6410af3ac9"}, - {file = "mypy-0.971.tar.gz", hash = "sha256:40b0f21484238269ae6a57200c807d80debc6459d444c0489a102d7c6a75fa56"}, + {file = "mypy-0.991-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7d17e0a9707d0772f4a7b878f04b4fd11f6f5bcb9b3813975a9b13c9332153ab"}, + {file = "mypy-0.991-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0714258640194d75677e86c786e80ccf294972cc76885d3ebbb560f11db0003d"}, + {file = "mypy-0.991-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c8f3be99e8a8bd403caa8c03be619544bc2c77a7093685dcf308c6b109426c6"}, + {file = "mypy-0.991-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc9ec663ed6c8f15f4ae9d3c04c989b744436c16d26580eaa760ae9dd5d662eb"}, + {file = "mypy-0.991-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4307270436fd7694b41f913eb09210faff27ea4979ecbcd849e57d2da2f65305"}, + {file = "mypy-0.991-cp310-cp310-win_amd64.whl", hash = "sha256:901c2c269c616e6cb0998b33d4adbb4a6af0ac4ce5cd078afd7bc95830e62c1c"}, + {file = "mypy-0.991-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d13674f3fb73805ba0c45eb6c0c3053d218aa1f7abead6e446d474529aafc372"}, + {file = "mypy-0.991-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c8cd4fb70e8584ca1ed5805cbc7c017a3d1a29fb450621089ffed3e99d1857f"}, + {file = "mypy-0.991-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:209ee89fbb0deed518605edddd234af80506aec932ad28d73c08f1400ef80a33"}, + {file = "mypy-0.991-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37bd02ebf9d10e05b00d71302d2c2e6ca333e6c2a8584a98c00e038db8121f05"}, + {file = "mypy-0.991-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:26efb2fcc6b67e4d5a55561f39176821d2adf88f2745ddc72751b7890f3194ad"}, + {file = "mypy-0.991-cp311-cp311-win_amd64.whl", hash = "sha256:3a700330b567114b673cf8ee7388e949f843b356a73b5ab22dd7cff4742a5297"}, + {file = "mypy-0.991-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1f7d1a520373e2272b10796c3ff721ea1a0712288cafaa95931e66aa15798813"}, + {file = "mypy-0.991-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:641411733b127c3e0dab94c45af15fea99e4468f99ac88b39efb1ad677da5711"}, + {file = "mypy-0.991-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3d80e36b7d7a9259b740be6d8d906221789b0d836201af4234093cae89ced0cd"}, + {file = "mypy-0.991-cp37-cp37m-win_amd64.whl", hash = "sha256:e62ebaad93be3ad1a828a11e90f0e76f15449371ffeecca4a0a0b9adc99abcef"}, + {file = "mypy-0.991-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b86ce2c1866a748c0f6faca5232059f881cda6dda2a893b9a8373353cfe3715a"}, + {file = "mypy-0.991-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac6e503823143464538efda0e8e356d871557ef60ccd38f8824a4257acc18d93"}, + {file = "mypy-0.991-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0cca5adf694af539aeaa6ac633a7afe9bbd760df9d31be55ab780b77ab5ae8bf"}, + {file = "mypy-0.991-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a12c56bf73cdab116df96e4ff39610b92a348cc99a1307e1da3c3768bbb5b135"}, + {file = "mypy-0.991-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:652b651d42f155033a1967739788c436491b577b6a44e4c39fb340d0ee7f0d70"}, + {file = "mypy-0.991-cp38-cp38-win_amd64.whl", hash = "sha256:4175593dc25d9da12f7de8de873a33f9b2b8bdb4e827a7cae952e5b1a342e243"}, + {file = "mypy-0.991-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:98e781cd35c0acf33eb0295e8b9c55cdbef64fcb35f6d3aa2186f289bed6e80d"}, + {file = "mypy-0.991-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6d7464bac72a85cb3491c7e92b5b62f3dcccb8af26826257760a552a5e244aa5"}, + {file = "mypy-0.991-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c9166b3f81a10cdf9b49f2d594b21b31adadb3d5e9db9b834866c3258b695be3"}, + {file = "mypy-0.991-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8472f736a5bfb159a5e36740847808f6f5b659960115ff29c7cecec1741c648"}, + {file = "mypy-0.991-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e80e758243b97b618cdf22004beb09e8a2de1af481382e4d84bc52152d1c476"}, + {file = "mypy-0.991-cp39-cp39-win_amd64.whl", hash = "sha256:74e259b5c19f70d35fcc1ad3d56499065c601dfe94ff67ae48b85596b9ec1461"}, + {file = "mypy-0.991-py3-none-any.whl", hash = "sha256:de32edc9b0a7e67c2775e574cb061a537660e51210fbf6006b0b36ea695ae9bb"}, + {file = "mypy-0.991.tar.gz", hash = "sha256:3c0165ba8f354a6d9881809ef29f1a9318a236a6d81c690094c5df32107bde06"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, diff --git a/pyproject.toml b/pyproject.toml index bf1c89d58..84f261241 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ colorama = {version = "^0.4.0", platform = "win32"} frozendict = "^2.3.4" rich = ">=11.1.0" shtab = "^1.5.6" +mypy = "^0.991" [tool.poetry.group.dev.dependencies] pytest = "^7.1.2" @@ -26,7 +27,6 @@ pytest-cov = "^3.0.0" omegaconf = "^2.2.2" attrs = "^21.4.0" torch = "^1.10.0" -mypy = "^0.971" pyright = "^1.1.264" numpy = ">=1.20.0" flax = "^0.6.0" diff --git a/tests/test_attrs.py b/tests/test_attrs.py index 85a463358..459f63b8e 100644 --- a/tests/test_attrs.py +++ b/tests/test_attrs.py @@ -10,7 +10,7 @@ import tyro -def test_attrs_basic(): +def test_attrs_basic() -> None: @attr.s class ManyTypesA: i: int = attr.ib() @@ -34,7 +34,7 @@ class ManyTypesA: ) == ManyTypesA(i=5, s="5", f=5.0, p=pathlib.Path("~")) -def test_attrs_defaults(): +def test_attrs_defaults() -> None: @attr.s class ManyTypesB: i: int = attr.ib() @@ -53,7 +53,7 @@ class ManyTypesB: ) == ManyTypesB(i=5, s="5", f=1.0) -def test_attrs_helptext(): +def test_attrs_helptext() -> None: @attr.s class Helptext: """This docstring should be printed as a description.""" @@ -78,7 +78,7 @@ class Helptext: assert "Documentation 3" in helptext -def test_attrs_next_gen_and_factory(): +def test_attrs_next_gen_and_factory() -> None: @define class Helptext: """This docstring should be printed as a description.""" diff --git a/tests/test_collections.py b/tests/test_collections.py index 41df3c2ec..b63d09968 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -22,7 +22,7 @@ import tyro -def test_tuples_fixed(): +def test_tuples_fixed() -> None: @dataclasses.dataclass class A: x: Tuple[int, int, int] @@ -34,7 +34,7 @@ class A: tyro.cli(A, args=[]) -def test_tuples_fixed_mixed(): +def test_tuples_fixed_mixed() -> None: @dataclasses.dataclass class A: x: Tuple[int, str, int] @@ -46,7 +46,7 @@ class A: tyro.cli(A, args=[]) -def test_tuples_with_default(): +def test_tuples_with_default() -> None: @dataclasses.dataclass class A: x: Tuple[int, int, int] = (0, 1, 2) @@ -57,7 +57,7 @@ class A: tyro.cli(A, args=["--x"]) -def test_tuple_with_literal_and_default(): +def test_tuple_with_literal_and_default() -> None: @dataclasses.dataclass class A: x: Tuple[Literal[1, 2, 3], ...] = (1, 2) @@ -70,7 +70,7 @@ class A: tyro.cli(A, args=["--x"]) -def test_positional_tuple_with_literal_and_default(): +def test_positional_tuple_with_literal_and_default() -> None: @dataclasses.dataclass class A: x: tyro.conf.Positional[Tuple[Literal[1, 2, 3], ...]] = (1, 2) @@ -84,7 +84,7 @@ class A: assert "invalid choice" in target.getvalue() -def test_tuples_fixed_multitype(): +def test_tuples_fixed_multitype() -> None: @dataclasses.dataclass class A: x: Tuple[int, str, float] @@ -96,7 +96,7 @@ class A: tyro.cli(A, args=[]) -def test_tuples_fixed_bool(): +def test_tuples_fixed_bool() -> None: @dataclasses.dataclass class A: x: Tuple[bool, bool, bool] @@ -110,7 +110,7 @@ class A: tyro.cli(A, args=[]) -def test_tuples_variable(): +def test_tuples_variable() -> None: @dataclasses.dataclass class A: x: Tuple[int, ...] @@ -122,7 +122,7 @@ class A: tyro.cli(A, args=[]) -def test_tuples_variable_bool(): +def test_tuples_variable_bool() -> None: @dataclasses.dataclass class A: x: Tuple[bool, ...] @@ -136,7 +136,7 @@ class A: tyro.cli(A, args=[]) -def test_tuples_variable_optional(): +def test_tuples_variable_optional() -> None: @dataclasses.dataclass class A: x: Optional[Tuple[int, ...]] = None @@ -147,7 +147,7 @@ class A: assert tyro.cli(A, args=[]) == A(x=None) -def test_sequences(): +def test_sequences() -> None: @dataclasses.dataclass class A: x: Sequence[int] @@ -159,7 +159,7 @@ class A: tyro.cli(A, args=[]) -def test_lists(): +def test_lists() -> None: @dataclasses.dataclass class A: x: List[int] @@ -171,7 +171,7 @@ class A: tyro.cli(A, args=[]) -def test_list_with_literal(): +def test_list_with_literal() -> None: @dataclasses.dataclass class A: x: List[Literal[1, 2, 3]] @@ -185,7 +185,7 @@ class A: tyro.cli(A, args=[]) -def test_list_with_enums(): +def test_list_with_enums() -> None: class Color(enum.Enum): RED = enum.auto() GREEN = enum.auto() @@ -206,7 +206,7 @@ class A: tyro.cli(A, args=[]) -def test_lists_with_default(): +def test_lists_with_default() -> None: class Color(enum.Enum): RED = enum.auto() GREEN = enum.auto() @@ -224,7 +224,7 @@ class A: ) -def test_lists_bool(): +def test_lists_bool() -> None: @dataclasses.dataclass class A: x: List[bool] @@ -238,7 +238,7 @@ class A: tyro.cli(A, args=[]) -def test_sets(): +def test_sets() -> None: @dataclasses.dataclass class A: x: Set[int] @@ -250,7 +250,7 @@ class A: tyro.cli(A, args=[]) -def test_frozen_sets(): +def test_frozen_sets() -> None: @dataclasses.dataclass class A: x: FrozenSet[int] @@ -262,7 +262,7 @@ class A: tyro.cli(A, args=[]) -def test_deque(): +def test_deque() -> None: @dataclasses.dataclass class A: x: Deque[int] @@ -276,7 +276,7 @@ class A: tyro.cli(A, args=[]) -def test_sets_with_default(): +def test_sets_with_default() -> None: @dataclasses.dataclass class A: x: Set[int] = dataclasses.field(default_factory={0, 1, 2}.copy) @@ -287,7 +287,7 @@ class A: tyro.cli(A, args=["--x"]) -def test_optional_sequences(): +def test_optional_sequences() -> None: @dataclasses.dataclass class A: x: Optional[Sequence[int]] = None @@ -298,7 +298,7 @@ class A: assert tyro.cli(A, args=[]) == A(x=None) -def test_optional_lists(): +def test_optional_lists() -> None: @dataclasses.dataclass class A: x: Optional[List[int]] = None @@ -309,7 +309,7 @@ class A: assert tyro.cli(A, args=[]) == A(x=None) -def test_nested_optional_types(): +def test_nested_optional_types() -> None: """We support "None" as a special-case keyword. (note: this is a bit weird because Optional[str] might interpret "None" as either a string or an actual `None` value)""" @@ -322,7 +322,7 @@ class A: assert tyro.cli(A, args=["--x", "0", "None", "1"]) == A((0, None, 1)) -def test_union_over_collections(): +def test_union_over_collections() -> None: def main(a: Union[Tuple[float, ...], Tuple[int, ...]]) -> Any: return a @@ -330,7 +330,7 @@ def main(a: Union[Tuple[float, ...], Tuple[int, ...]]) -> Any: assert tyro.cli(main, args="--a 3 3 7".split(" ")) == (3, 3, 7) -def test_union_over_collections_2(): +def test_union_over_collections_2() -> None: def main(a: Union[Tuple[str, float, str], Tuple[str, str, float]]) -> Any: return a @@ -338,7 +338,7 @@ def main(a: Union[Tuple[str, float, str], Tuple[str, str, float]]) -> Any: assert tyro.cli(main, args="--a 3 3 hey".split(" ")) == ("3", 3.0, "hey") -def test_union_over_collections_3(): +def test_union_over_collections_3() -> None: def main(a: Union[Tuple[int, int], Tuple[int, int, int]]) -> Tuple[int, ...]: return a @@ -354,7 +354,7 @@ def main(a: Union[Tuple[int, int], Tuple[int, int, int]]) -> Tuple[int, ...]: tyro.cli(main, args=[]) -def test_choices_in_tuples_0(): +def test_choices_in_tuples_0() -> None: @dataclasses.dataclass class A: x: Tuple[bool, bool] @@ -362,7 +362,7 @@ class A: assert tyro.cli(A, args=["--x", "True", "False"]) == A((True, False)) -def test_choices_in_tuples_1(): +def test_choices_in_tuples_1() -> None: @dataclasses.dataclass class A: x: Tuple[bool, Literal["True", "False"]] @@ -370,7 +370,7 @@ class A: assert tyro.cli(A, args=["--x", "True", "False"]) == A((True, "False")) -def test_choices_in_tuples_2(): +def test_choices_in_tuples_2() -> None: @dataclasses.dataclass class A: x: Tuple[bool, Literal["True", "False", "None"]] @@ -381,7 +381,7 @@ class A: tyro.cli(A, args=["--x", "None", "False"]) -def test_nested_tuple_types(): +def test_nested_tuple_types() -> None: @dataclasses.dataclass class A: x: Tuple[Tuple[int, int], Tuple[str, str]] @@ -389,7 +389,7 @@ class A: assert tyro.cli(A, args="--x 5 5 5 5".split(" ")).x == ((5, 5), ("5", "5")) -def test_variable_nested_tuple(): +def test_variable_nested_tuple() -> None: def main(x: Tuple[Tuple[int, str], ...]) -> tuple: return x @@ -398,7 +398,7 @@ def main(x: Tuple[Tuple[int, str], ...]) -> tuple: tyro.cli(main, args="--x 1 1 2".split(" ")) -def test_super_nested(): +def test_super_nested() -> None: def main( x: Optional[ List[ @@ -424,7 +424,7 @@ def main( tyro.cli(main, args=["--help"]) -def test_dict_no_annotation(): +def test_dict_no_annotation() -> None: def main(x: Dict[str, Any] = {"int": 5, "str": "5"}): return x @@ -435,7 +435,7 @@ def main(x: Dict[str, Any] = {"int": 5, "str": "5"}): } -def test_double_dict_no_annotation(): +def test_double_dict_no_annotation() -> None: def main( x: Dict[str, Any] = { "wow": {"int": 5, "str": "5"}, @@ -452,21 +452,21 @@ def main( } -def test_list_narrowing(): +def test_list_narrowing() -> None: def main(x: list = [0, 1, 2, "hello"]) -> Any: return x assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", 5] -def test_set_narrowing(): +def test_set_narrowing() -> None: def main(x: set = {0, 1, 2, "hello"}) -> Any: return x assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", 5} -def test_tuple_narrowing(): +def test_tuple_narrowing() -> None: def main(x: tuple = (0, 1, 2, "hello")) -> Any: return x diff --git a/tests/test_conf.py b/tests/test_conf.py index c0ff1fa4d..57597b1e3 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -8,7 +8,7 @@ import tyro -def test_omit_subcommand_prefix(): +def test_omit_subcommand_prefix() -> None: @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 @@ -61,7 +61,7 @@ class DefaultInstanceSubparser: ) -def test_avoid_subparser_with_default(): +def test_avoid_subparser_with_default() -> None: @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 @@ -103,7 +103,7 @@ class DefaultInstanceSubparser: ) -def test_avoid_subparser_with_default_recursive(): +def test_avoid_subparser_with_default_recursive() -> None: @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 @@ -148,7 +148,7 @@ class DefaultInstanceSubparser: ) -def test_subparser_in_nested_with_metadata(): +def test_subparser_in_nested_with_metadata() -> None: @dataclasses.dataclass class A: a: int @@ -200,7 +200,7 @@ class Parent: ) == Parent(Nested1(Nested2(B(7)))) -def test_subparser_in_nested_with_metadata_generic(): +def test_subparser_in_nested_with_metadata_generic() -> None: @dataclasses.dataclass class A: a: int @@ -256,7 +256,7 @@ class Parent: ) == Parent(Nested1(Nested2(B(7)))) -def test_subparser_in_nested_with_metadata_generic_alt(): +def test_subparser_in_nested_with_metadata_generic_alt() -> None: @dataclasses.dataclass class A: a: int @@ -310,7 +310,7 @@ class Parent: ) == Parent(Nested1(Nested2(B(7)))) -def test_subparser_in_nested_with_metadata_default_matching(): +def test_subparser_in_nested_with_metadata_default_matching() -> None: @dataclasses.dataclass(frozen=True) class A: a: int @@ -351,7 +351,7 @@ def main_three(x: Nested = Nested(B(15))) -> None: assert "default: x.subcommand:three" in get_helptext(main_three) -def test_flag(): +def test_flag() -> None: """When boolean flags have no default value, they must be explicitly specified.""" @dataclasses.dataclass @@ -371,7 +371,7 @@ class A: ) == A(True) -def test_fixed(): +def test_fixed() -> None: """When boolean flags have no default value, they must be explicitly specified.""" @dataclasses.dataclass @@ -392,7 +392,7 @@ class A: ) == A(True) -def test_fixed_recursive(): +def test_fixed_recursive() -> None: """When boolean flags have no default value, they must be explicitly specified.""" @dataclasses.dataclass @@ -413,7 +413,7 @@ class A: ) == A(True) -def test_suppressed_group(): +def test_suppressed_group() -> None: """Reproduction of https://github.com/nerfstudio-project/nerfstudio/issues/882.""" @dataclasses.dataclass @@ -430,7 +430,7 @@ def main( assert tyro.cli(main, args=["--value", "5"]) == 8 -def test_fixed_group(): +def test_fixed_group() -> None: """Inspired by https://github.com/nerfstudio-project/nerfstudio/issues/882.""" @dataclasses.dataclass @@ -447,7 +447,7 @@ def main( assert tyro.cli(main, args=["--value", "5"]) == 8 -def test_fixed_suppressed_group(): +def test_fixed_suppressed_group() -> None: """Reproduction of https://github.com/nerfstudio-project/nerfstudio/issues/882.""" @dataclasses.dataclass @@ -464,7 +464,7 @@ def main( assert tyro.cli(main, args=["--value", "5"]) == 8 -def test_suppressed(): +def test_suppressed() -> None: @dataclasses.dataclass class Struct: a: int = 5 @@ -478,7 +478,7 @@ def main(x: Any = Struct()): assert "--x.b" not in helptext -def test_suppress_manual_fixed(): +def test_suppress_manual_fixed() -> None: @dataclasses.dataclass class Struct: a: int = 5 @@ -492,7 +492,7 @@ def main(x: Any = Struct()): assert "--x.b" not in helptext -def test_suppress_auto_fixed(): +def test_suppress_auto_fixed() -> None: @dataclasses.dataclass class Struct: a: int = 5 @@ -506,7 +506,7 @@ def main(x: tyro.conf.SuppressFixed[Any] = Struct()): assert "--x.b" not in helptext -def test_argconf_help(): +def test_argconf_help() -> None: @dataclasses.dataclass class Struct: a: Annotated[ @@ -529,7 +529,7 @@ def main(x: Any = Struct()) -> int: assert tyro.cli(main, args=["--x.nice", "3"]) == 3 -def test_positional(): +def test_positional() -> None: def main(x: tyro.conf.Positional[int], y: int) -> int: return x + y @@ -537,7 +537,7 @@ def main(x: tyro.conf.Positional[int], y: int) -> int: assert tyro.cli(main, args="--y 3 5".split(" ")) == 8 -def test_positional_order_swap(): +def test_positional_order_swap() -> None: def main(x: int, y: tyro.conf.Positional[int]) -> int: return x + y @@ -545,7 +545,7 @@ def main(x: int, y: tyro.conf.Positional[int]) -> int: assert tyro.cli(main, args="--x 3 5".split(" ")) == 8 -def test_omit_subcommand_prefix_and_consolidate_subcommand_args(): +def test_omit_subcommand_prefix_and_consolidate_subcommand_args() -> None: @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 @@ -616,7 +616,7 @@ class DefaultInstanceSubparser: ) -def test_omit_subcommand_prefix_and_consolidate_subcommand_args_in_function(): +def test_omit_subcommand_prefix_and_consolidate_subcommand_args_in_function() -> None: @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 346b64a4f..1e20975ed 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -23,7 +23,7 @@ import tyro -def test_no_args(): +def test_no_args() -> None: def main() -> int: return 5 @@ -32,7 +32,7 @@ def main() -> int: tyro.cli(main, args=["3"]) -def test_basic(): +def test_basic() -> None: @dataclasses.dataclass class ManyTypes: i: int @@ -93,7 +93,7 @@ def __init__(self, i: int, s: str, f: float, p: pathlib.Path): ).inner == ManyTypes(i=5, s="5", f=5.0, p=pathlib.Path("~")) -def test_init_false(): +def test_init_false() -> None: @dataclasses.dataclass class InitFalseDataclass: i: int @@ -134,7 +134,7 @@ class InitFalseDataclass: ) -def test_required(): +def test_required() -> None: @dataclasses.dataclass class A: x: int @@ -143,7 +143,7 @@ class A: tyro.cli(A, args=[]) -def test_flag(): +def test_flag() -> None: """When boolean flags have no default value, they must be explicitly specified.""" @dataclasses.dataclass @@ -166,7 +166,7 @@ class A: assert tyro.cli(A, args=["--x", "False"]) == A(False) -def test_flag_default_false(): +def test_flag_default_false() -> None: """When boolean flags default to False, a --flag-name flag must be passed in to flip it to True.""" @dataclasses.dataclass @@ -177,7 +177,7 @@ class A: assert tyro.cli(A, args=["--x"]) == A(True) -def test_flag_default_true(): +def test_flag_default_true() -> None: """When boolean flags default to True, a --no-flag-name flag must be passed in to flip it to False.""" @dataclasses.dataclass @@ -188,7 +188,7 @@ class A: assert tyro.cli(A, args=["--no-x"]) == A(False) -def test_flag_default_true_nested(): +def test_flag_default_true_nested() -> None: """When boolean flags default to True, a --no-flag-name flag must be passed in to flip it to False.""" @dataclasses.dataclass @@ -203,7 +203,7 @@ class A: assert tyro.cli(A, args=["--x.no-x"]) == A(NestedDefaultTrue(False)) -def test_default(): +def test_default() -> None: @dataclasses.dataclass class A: x: int = 5 @@ -211,7 +211,7 @@ class A: assert tyro.cli(A, args=[]) == A() -def test_default_factory(): +def test_default_factory() -> None: @dataclasses.dataclass class A: x: int = dataclasses.field(default_factory=lambda: 5) @@ -219,7 +219,7 @@ class A: assert tyro.cli(A, args=[]) == A() -def test_optional(): +def test_optional() -> None: @dataclasses.dataclass class A: x: Optional[int] = None @@ -227,7 +227,7 @@ class A: assert tyro.cli(A, args=[]) == A(x=None) -def test_union_basic(): +def test_union_basic() -> None: def main(x: Union[int, str]) -> Union[int, str]: return x @@ -236,7 +236,7 @@ def main(x: Union[int, str]) -> Union[int, str]: assert tyro.cli(main, args=["--x", "five"]) == "five" -def test_union_with_list(): +def test_union_with_list() -> None: def main(x: Union[int, str, List[bool]]) -> Any: return x @@ -247,7 +247,7 @@ def main(x: Union[int, str, List[bool]]) -> Any: assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] -def test_union_literal(): +def test_union_literal() -> None: def main(x: Union[Literal[1, 2], Literal[3, 4, 5], str]) -> Union[int, str]: return x @@ -256,7 +256,7 @@ def main(x: Union[Literal[1, 2], Literal[3, 4, 5], str]) -> Union[int, str]: assert tyro.cli(main, args=["--x", "five"]) == "five" -def test_func_typevar(): +def test_func_typevar() -> None: T = TypeVar("T", int, str) def main(x: T) -> T: @@ -266,7 +266,7 @@ def main(x: T) -> T: assert tyro.cli(main, args=["--x", "five"]) == "five" -def test_func_typevar_bound(): +def test_func_typevar_bound() -> None: T = TypeVar("T", bound=int) def main(x: T) -> T: @@ -277,7 +277,7 @@ def main(x: T) -> T: tyro.cli(main, args=["--x", "five"]) -def test_enum(): +def test_enum() -> None: class Color(enum.Enum): RED = enum.auto() GREEN = enum.auto() @@ -295,7 +295,7 @@ class EnumClassB: assert tyro.cli(EnumClassB, args=[]) == EnumClassB() -def test_literal(): +def test_literal() -> None: @dataclasses.dataclass class A: x: Literal[0, 1, 2] @@ -310,7 +310,7 @@ class A: Choices = tyro.extras.literal_type_from_choices([0, 1, 2]) # type: ignore -def test_dynamic_literal(): +def test_dynamic_literal() -> None: @dataclasses.dataclass class A: x: Choices @@ -320,7 +320,7 @@ class A: assert tyro.cli(A, args=["--x", "3"]) -def test_literal_bool(): +def test_literal_bool() -> None: def main(x: Literal[True]) -> bool: return x @@ -337,7 +337,7 @@ def main2(x: Literal[True, False]) -> bool: tyro.cli(main2, args=["--x", "Tru"]) -def test_literal_enum(): +def test_literal_enum() -> None: class Color(enum.Enum): RED = enum.auto() GREEN = enum.auto() @@ -353,7 +353,7 @@ class A: assert tyro.cli(A, args=["--x", "BLUE"]) -def test_optional_literal(): +def test_optional_literal() -> None: @dataclasses.dataclass class A: x: Optional[Literal[0, 1, 2]] = None @@ -364,7 +364,7 @@ class A: assert tyro.cli(A, args=[]) == A(x=None) -def test_multitype_literal(): +def test_multitype_literal() -> None: def main(x: Literal[0, "5"]) -> Any: return x @@ -374,7 +374,7 @@ def main(x: Literal[0, "5"]) -> Any: tyro.cli(main, args=["--x", "6"]) -def test_annotated(): +def test_annotated() -> None: """Annotated[] is a no-op.""" @dataclasses.dataclass @@ -384,7 +384,7 @@ class A: assert tyro.cli(A, args=["--x", "5"]) == A(x=5) -def test_annotated_optional(): +def test_annotated_optional() -> None: """Annotated[] is a no-op.""" @dataclasses.dataclass @@ -395,7 +395,7 @@ class A: assert tyro.cli(A, args=["--x", "5"]) == A(x=5) -def test_optional_annotated(): +def test_optional_annotated() -> None: """Annotated[] is a no-op.""" @dataclasses.dataclass @@ -406,7 +406,7 @@ class A: assert tyro.cli(A, args=["--x", "5"]) == A(x=5) -def test_final(): +def test_final() -> None: """Final[] is a no-op.""" @dataclasses.dataclass @@ -416,7 +416,7 @@ class A: assert tyro.cli(A, args=["--x", "5"]) == A(x=5) -def test_final_optional(): +def test_final_optional() -> None: @dataclasses.dataclass class A: x: Final[Optional[int]] = 3 @@ -425,7 +425,7 @@ class A: assert tyro.cli(A, args=["--x", "5"]) == A(x=5) -def test_classvar(): +def test_classvar() -> None: """ClassVar[] types should be skipped.""" @dataclasses.dataclass @@ -437,7 +437,7 @@ class A: assert tyro.cli(A, args=[]) == A() -def test_parse_empty_description(): +def test_parse_empty_description() -> None: """If the file has no dosctring, it should be treated as an empty string.""" @dataclasses.dataclass @@ -450,7 +450,7 @@ class A: SomeTypeAlias: TypeAlias = int -def test_type_alias(): +def test_type_alias() -> None: def add(a: SomeTypeAlias, b: SomeTypeAlias) -> SomeTypeAlias: return a + b @@ -458,21 +458,21 @@ def add(a: SomeTypeAlias, b: SomeTypeAlias) -> SomeTypeAlias: @pytest.mark.filterwarnings("ignore::Warning") -def test_any(): +def test_any() -> None: def main(x: Any = 5) -> Any: return x assert tyro.cli(main, args=[]) == 5 -def test_bytes(): +def test_bytes() -> None: def main(x: bytes) -> bytes: return x assert tyro.cli(main, args=["--x", "hello"]) == b"hello" -def test_any_str(): +def test_any_str() -> None: def main(x: AnyStr) -> AnyStr: return x @@ -481,7 +481,7 @@ def main(x: AnyStr) -> AnyStr: assert tyro.cli(main, args=["--x", "hello„"]) == "hello„" -def test_fixed(): +def test_fixed() -> None: def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]: return x @@ -490,7 +490,7 @@ def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]: tyro.cli(main, args=["--x", "something"]) -def test_fixed_dataclass_type(): +def test_fixed_dataclass_type() -> None: def dummy(): return 5 # noqa @@ -502,38 +502,39 @@ def main(x: Callable = dummy) -> Callable: tyro.cli(main, args=["--x", "something"]) -def test_missing_singleton(): +def test_missing_singleton() -> None: assert tyro.MISSING is copy.deepcopy(tyro.MISSING) -def test_torch_device(): +def test_torch_device() -> None: def main(device: torch.device) -> torch.device: return device assert tyro.cli(main, args=["--device", "cpu"]) == torch.device("cpu") -def test_torch_device_2(): +def test_torch_device_2() -> None: assert tyro.cli(torch.device, args=["cpu"]) == torch.device("cpu") -def test_just_int(): +def test_just_int() -> None: assert tyro.cli(int, args=["123"]) == 123 -def test_just_dict(): +def test_just_dict() -> None: assert tyro.cli(Dict[str, str], args="key value key2 value2".split(" ")) == { "key": "value", "key2": "value2", } -def test_just_list(): +def test_just_list() -> None: assert tyro.cli(List[int], args="1 2 3 4".split(" ")) == [1, 2, 3, 4] -def test_just_tuple(): - assert tyro.cli(Tuple[int, int, int, int], args="1 2 3 4".split(" ")) == ( +def test_just_tuple() -> None: + # Need a type: ignore for mypy. Seems like a mypy bug. + assert tyro.cli(Tuple[int, int, int, int], args="1 2 3 4".split(" ")) == ( # type: ignore 1, 2, 3, @@ -541,7 +542,7 @@ def test_just_tuple(): ) -def test_return_parser(): +def test_return_parser() -> None: def main() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() return parser diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index c6a2855b6..88c47cf4f 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -12,7 +12,7 @@ import tyro._strings -def test_basic_dict(): +def test_basic_dict() -> None: def main(params: Dict[str, int]) -> Dict[str, int]: return params @@ -32,7 +32,7 @@ def main(params: Dict[str, int]) -> Dict[str, int]: tyro.cli(main, args="--params".split(" ")) -def test_dict_with_default(): +def test_dict_with_default() -> None: def main(params: Mapping[Literal[1, 3, 5, 7], bool] = {5: False, 1: True}) -> Any: return params @@ -45,7 +45,7 @@ def main(params: Mapping[Literal[1, 3, 5, 7], bool] = {5: False, 1: True}) -> An tyro.cli(main, args="--params".split(" ")) -def test_tuple_in_dict(): +def test_tuple_in_dict() -> None: def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> dict: return x @@ -55,7 +55,7 @@ def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> d } -def test_basic_typeddict(): +def test_basic_typeddict() -> None: class ManyTypesTypedDict(TypedDict): i: int s: str @@ -72,7 +72,7 @@ class ManyTypesTypedDict(TypedDict): tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) -def test_total_false_typeddict(): +def test_total_false_typeddict() -> None: class ManyTypesTypedDict(TypedDict, total=False): i: int s: str @@ -86,7 +86,7 @@ class ManyTypesTypedDict(TypedDict, total=False): assert tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) == dict(s="5") -def test_total_false_nested_typeddict(): +def test_total_false_nested_typeddict() -> None: class ChildTypedDict(TypedDict, total=False): i: int s: str @@ -110,7 +110,7 @@ class ParentTypedDict(TypedDict, total=False): ) -def test_total_false_typeddict_with_nested(): +def test_total_false_typeddict_with_nested() -> None: @dataclasses.dataclass class Inner: j: float @@ -132,7 +132,7 @@ class ManyTypesTypedDict(TypedDict, total=False): ) -def test_total_false_typeddict_with_tuple(): +def test_total_false_typeddict_with_tuple() -> None: class ManyTypesTypedDict(TypedDict, total=False): i: int s: Tuple[str, str] @@ -151,7 +151,7 @@ class ManyTypesTypedDict(TypedDict, total=False): ) == dict(i=5, s=("5", "5")) -def test_nested_typeddict(): +def test_nested_typeddict() -> None: class ChildTypedDict(TypedDict): y: int @@ -166,7 +166,7 @@ class NestedTypedDict(TypedDict): tyro.cli(NestedTypedDict, args=["--x", "1"]) -def test_helptext_and_default_typeddict(): +def test_helptext_and_default_typeddict() -> None: class HelptextTypedDict(TypedDict): """This docstring should be printed as a description.""" @@ -192,7 +192,7 @@ class HelptextTypedDict(TypedDict): assert "Documentation 3 (default: 3)" in helptext -def test_basic_namedtuple(): +def test_basic_namedtuple() -> None: class ManyTypesNamedTuple(NamedTuple): i: int s: str @@ -214,7 +214,7 @@ class ManyTypesNamedTuple(NamedTuple): ) == ManyTypesNamedTuple(i=5, s="5", f=5.0, p=pathlib.Path("~")) -def test_nested_namedtuple(): +def test_nested_namedtuple() -> None: class ChildNamedTuple(NamedTuple): y: int @@ -229,7 +229,7 @@ class NestedNamedTuple(NamedTuple): tyro.cli(NestedNamedTuple, args=["--x", "1"]) -def test_helptext_and_default_namedtuple(): +def test_helptext_and_default_namedtuple() -> None: class HelptextNamedTupleDefault(NamedTuple): """This docstring should be printed as a description.""" @@ -255,7 +255,7 @@ class HelptextNamedTupleDefault(NamedTuple): assert "Documentation 3 (default: 3)" in helptext -def test_helptext_and_default_namedtuple_alternate(): +def test_helptext_and_default_namedtuple_alternate() -> None: class HelptextNamedTuple(NamedTuple): """This docstring should be printed as a description.""" @@ -294,7 +294,7 @@ class HelptextNamedTuple(NamedTuple): assert "(default: 3)" in helptext -def test_nested_dict(): +def test_nested_dict() -> None: loaded_config = { "batch_size": 32, "optimizer": { @@ -324,7 +324,7 @@ def test_nested_dict(): assert loaded_config == backup_config -def test_nested_dict_hyphen(): +def test_nested_dict_hyphen() -> None: # We do a lot of underscore <=> conversion in the code; this is just to make sure it # doesn't break anything! loaded_config = { @@ -356,7 +356,7 @@ def test_nested_dict_hyphen(): assert loaded_config == backup_config -def test_nested_dict_annotations(): +def test_nested_dict_annotations() -> None: loaded_config = { "optimizer": { "scheduler": {"schedule-type": "constant"}, diff --git a/tests/test_errors.py b/tests/test_errors.py index 405d25275..97b5bba48 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -6,7 +6,7 @@ import tyro -def test_ambiguous_collection_0(): +def test_ambiguous_collection_0() -> None: @dataclasses.dataclass class A: x: Tuple[Tuple[int, ...], ...] @@ -15,7 +15,7 @@ class A: tyro.cli(A, args=["--x", "0", "1"]) -def test_ambiguous_collection_1(): +def test_ambiguous_collection_1() -> None: def main(x: Tuple[List[str], List[str]]) -> None: pass @@ -23,7 +23,7 @@ def main(x: Tuple[List[str], List[str]]) -> None: tyro.cli(main, args=["--help"]) -def test_ambiguous_collection_2(): +def test_ambiguous_collection_2() -> None: def main(x: List[Union[Tuple[int, int], Tuple[int, int, int]]]) -> None: pass @@ -37,12 +37,12 @@ class _CycleDataclass: x: "_CycleDataclass" -def test_cycle(): +def test_cycle() -> None: with pytest.raises(tyro.UnsupportedTypeAnnotationError): tyro.cli(_CycleDataclass, args=[]) -def test_uncallable_annotation(): +def test_uncallable_annotation() -> None: def main(arg: 5) -> None: # type: ignore pass @@ -50,7 +50,7 @@ def main(arg: 5) -> None: # type: ignore tyro.cli(main, args=[]) -def test_nested_annotation(): +def test_nested_annotation() -> None: @dataclasses.dataclass class OneIntArg: x: int @@ -76,14 +76,14 @@ class TwoStringArg: x: str y: str - def main(arg: List[TwoStringArg]) -> None: + def main2(arg: List[TwoStringArg]) -> None: pass with pytest.raises(tyro.UnsupportedTypeAnnotationError): - tyro.cli(main, args=[]) + tyro.cli(main2, args=[]) -def test_missing_annotation_1(): +def test_missing_annotation_1() -> None: def main(a, b) -> None: pass @@ -91,7 +91,7 @@ def main(a, b) -> None: tyro.cli(main, args=["--help"]) -def test_missing_annotation_2(): +def test_missing_annotation_2() -> None: def main(*, a) -> None: pass @@ -99,7 +99,7 @@ def main(*, a) -> None: tyro.cli(main, args=["--help"]) -def test_tuple_needs_default(): +def test_tuple_needs_default() -> None: def main(arg: tuple) -> None: # type: ignore pass @@ -107,7 +107,7 @@ def main(arg: tuple) -> None: # type: ignore tyro.cli(main, args=["--help"]) -def test_unbound_typevar(): +def test_unbound_typevar() -> None: T = TypeVar("T") def main(arg: T) -> None: # type: ignore @@ -117,7 +117,7 @@ def main(arg: T) -> None: # type: ignore tyro.cli(main, args=["--help"]) -def test_missing_default_fixed(): +def test_missing_default_fixed() -> None: def main(value: tyro.conf.SuppressFixed[tyro.conf.Fixed[int]]) -> int: return value @@ -125,7 +125,7 @@ def main(value: tyro.conf.SuppressFixed[tyro.conf.Fixed[int]]) -> int: tyro.cli(main, args=["--help"]) -def test_missing_default_suppressed(): +def test_missing_default_suppressed() -> None: def main(value: tyro.conf.Suppress[int]) -> int: return value diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index af5a2679e..7fc3e4a1a 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -20,7 +20,7 @@ def _check_serialization_identity(cls: Type[T], instance: T) -> None: ScalarType = TypeVar("ScalarType") -def test_tuple_generic_variable(): +def test_tuple_generic_variable() -> None: @dataclasses.dataclass class TupleGenericVariable(Generic[ScalarType]): xyz: Tuple[ScalarType, ...] @@ -30,7 +30,7 @@ class TupleGenericVariable(Generic[ScalarType]): ) == TupleGenericVariable((1, 2, 3)) -def test_tuple_generic_helptext(): +def test_tuple_generic_helptext() -> None: @dataclasses.dataclass class TupleGenericVariableHelptext(Generic[ScalarType]): """Helptext!""" @@ -45,7 +45,7 @@ class TupleGenericVariableHelptext(Generic[ScalarType]): assert "Helptext!" in helptext -def test_tuple_generic_no_helptext(): +def test_tuple_generic_no_helptext() -> None: @dataclasses.dataclass class TupleGenericVariableNoHelptext(Generic[ScalarType]): xyz: Tuple[ScalarType, ...] @@ -61,7 +61,7 @@ class TupleGenericVariableNoHelptext(Generic[ScalarType]): assert "The central part of internal API" not in helptext -def test_tuple_generic_fixed(): +def test_tuple_generic_fixed() -> None: @dataclasses.dataclass class TupleGenericFixed(Generic[ScalarType]): xyz: Tuple[ScalarType, ScalarType, ScalarType] @@ -84,7 +84,7 @@ class Point3(Generic[ScalarType]): frame: CoordinateFrame -def test_simple_generic(): +def test_simple_generic() -> None: @dataclasses.dataclass class SimpleGeneric: point_continuous: Point3[float] @@ -142,7 +142,7 @@ class SimpleGeneric: ) -def test_multilevel_generic(): +def test_multilevel_generic() -> None: @dataclasses.dataclass class Triangle(Generic[ScalarType]): a: Point3[ScalarType] @@ -186,7 +186,7 @@ class Triangle(Generic[ScalarType]): _check_serialization_identity(Triangle[float], parsed_instance) -def test_multilevel_generic_no_helptext(): +def test_multilevel_generic_no_helptext() -> None: @dataclasses.dataclass class LineSegment(Generic[ScalarType]): a: Point3[ScalarType] @@ -202,7 +202,7 @@ class LineSegment(Generic[ScalarType]): assert "The central part of internal API" not in helptext -def test_generic_nested_dataclass(): +def test_generic_nested_dataclass() -> None: @dataclasses.dataclass class Child: a: int @@ -224,7 +224,7 @@ class DataclassGeneric(Generic[T]): _check_serialization_identity(DataclassGeneric[Child], parsed_instance) -def test_generic_nested_dataclass_helptext(): +def test_generic_nested_dataclass_helptext() -> None: @dataclasses.dataclass class Child: a: int @@ -246,7 +246,7 @@ class DataclassGeneric(Generic[T]): assert "The central part of internal API" not in helptext -def test_generic_subparsers(): +def test_generic_subparsers() -> None: @dataclasses.dataclass class CommandOne: a: int @@ -266,7 +266,8 @@ class Subparser(Generic[T1, T2]): Subparser[CommandOne, CommandTwo], args="command:command-one --command.a 5".split(" "), ) - assert parsed_instance == Subparser(CommandOne(5)) + # Not supported in mypy. + assert parsed_instance == Subparser(CommandOne(5)) # type: ignore # Local generics will break. with pytest.raises(yaml.constructor.ConstructorError): _check_serialization_identity( @@ -277,7 +278,8 @@ class Subparser(Generic[T1, T2]): Subparser[CommandOne, CommandTwo], args="command:command-two --command.b 7".split(" "), ) - assert parsed_instance == Subparser(CommandTwo(7)) + # Not supported in mypy. + assert parsed_instance == Subparser(CommandTwo(7)) # type: ignore # Local generics will break. with pytest.raises(yaml.constructor.ConstructorError): _check_serialization_identity( @@ -285,7 +287,7 @@ class Subparser(Generic[T1, T2]): ) -def test_generic_subparsers_in_container(): +def test_generic_subparsers_in_container() -> None: T = TypeVar("T") @dataclasses.dataclass @@ -303,7 +305,8 @@ class Subparser(Generic[T1, T2]): Subparser[Command[int], Command[float]], args="command:command-int --command.a 5 3".split(" "), ) - assert parsed_instance == Subparser(Command([5, 3])) and isinstance( + # Not supported in mypy. + assert parsed_instance == Subparser(Command([5, 3])) and isinstance( # type: ignore parsed_instance.command.a[0], int ) # Local generics will break. @@ -316,7 +319,8 @@ class Subparser(Generic[T1, T2]): Subparser[Command[int], Command[float]], args="command:command-float --command.a 7 2".split(" "), ) - assert parsed_instance == Subparser(Command([7.0, 2.0])) and isinstance( + # Not supported in mypy. + assert parsed_instance == Subparser(Command([7.0, 2.0])) and isinstance( # type: ignore parsed_instance.command.a[0], float ) # Local generics will break. @@ -326,7 +330,7 @@ class Subparser(Generic[T1, T2]): ) -def test_serialize_missing(): +def test_serialize_missing() -> None: @dataclasses.dataclass class TupleGenericVariableMissing(Generic[ScalarType]): xyz: Tuple[ScalarType, ...] @@ -342,7 +346,7 @@ class TupleGenericVariableMissing(Generic[ScalarType]): ) -def test_generic_inherited_type_narrowing(): +def test_generic_inherited_type_narrowing() -> None: T = TypeVar("T") @dataclasses.dataclass @@ -365,7 +369,7 @@ def main(x: ActualParentClass[int] = ChildClass(5, 5)) -> ActualParentClass: assert tyro.cli(main, args="--x.x 3".split(" ")) == ChildClass(3, 5, 3) -def test_pculbertson(): +def test_pculbertson() -> None: # https://github.com/brentyi/tyro/issues/7 from typing import Union @@ -385,7 +389,7 @@ class Wrapper: assert wrapper1 == tyro.extras.from_yaml(Wrapper, tyro.extras.to_yaml(wrapper1)) -def test_annotated(): +def test_annotated() -> None: # https://github.com/brentyi/tyro/issues/7 @dataclasses.dataclass @@ -400,7 +404,7 @@ class Wrapper: assert wrapper1 == tyro.extras.from_yaml(Wrapper, tyro.extras.to_yaml(wrapper1)) -def test_superclass(): +def test_superclass() -> None: # https://github.com/brentyi/tyro/issues/7 @dataclasses.dataclass diff --git a/tests/test_helptext.py b/tests/test_helptext.py index c572746f4..af8dbaaf7 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -9,7 +9,7 @@ from typing_extensions import Annotated, Literal -def test_helptext(): +def test_helptext() -> None: @dataclasses.dataclass class Helptext: """This docstring should be printed as a description.""" @@ -32,7 +32,7 @@ class Helptext: assert "Documentation 3 (default: 3)" in helptext -def test_helptext_from_class_docstring(): +def test_helptext_from_class_docstring() -> None: @dataclasses.dataclass class Helptext2: """This docstring should be printed as a description. @@ -58,7 +58,7 @@ class Helptext2: assert "Documentation 3 (default: 3)" in helptext -def test_helptext_from_class_docstring_args(): +def test_helptext_from_class_docstring_args() -> None: @dataclasses.dataclass class Helptext3: """This docstring should be printed as a description. @@ -84,7 +84,7 @@ class Helptext3: assert "Documentation 3 (default: 3)" in helptext -def test_helptext_inherited(): +def test_helptext_inherited() -> None: class UnrelatedParentClass: pass @@ -113,7 +113,7 @@ class ChildClass(UnrelatedParentClass, ActualParentClass): assert "Documentation 2" in helptext -def test_helptext_inherited_default_override(): +def test_helptext_inherited_default_override() -> None: @dataclasses.dataclass class ParentClass: """This docstring should __not__ be printed as a description.""" @@ -144,7 +144,7 @@ def main(x: ParentClass = ChildClass(x=5, y=5)) -> Any: assert "should be printed" in helptext -def test_helptext_nested(): +def test_helptext_nested() -> None: """For nested classes, we should pull helptext from the outermost docstring if possible. The class docstring can be used as a fallback.""" @@ -177,7 +177,7 @@ def main_no_docstring(a: Inner) -> None: assert "Hello world!" in helptext -def test_helptext_defaults(): +def test_helptext_defaults() -> None: class Color(enum.Enum): RED = enum.auto() GREEN = enum.auto() @@ -196,7 +196,7 @@ class HelptextWithVariousDefaults: assert "(default: RED)" in helptext -def test_multiline_helptext(): +def test_multiline_helptext() -> None: @dataclasses.dataclass class HelptextMultiline: x: int # Documentation 1 @@ -219,7 +219,7 @@ class HelptextMultiline: assert "documentation 3" in helptext -def test_grouped_helptext(): +def test_grouped_helptext() -> None: @dataclasses.dataclass class HelptextGrouped: x: int # Documentation 1 @@ -233,7 +233,7 @@ class HelptextGrouped: assert "Description of both y and z. (default: 3)" in helptext -def test_none_default_value_helptext(): +def test_none_default_value_helptext() -> None: @dataclasses.dataclass class Config: x: Optional[int] = None @@ -244,7 +244,7 @@ class Config: assert "An optional variable. (default: None)" in helptext -def test_helptext_hard_bool(): +def test_helptext_hard_bool() -> None: @dataclasses.dataclass class HelptextHardString: # fmt: off @@ -262,7 +262,7 @@ class HelptextHardString: assert "2% milk." in helptext -def test_helptext_with_inheritance(): +def test_helptext_with_inheritance() -> None: @dataclasses.dataclass class Parent: # fmt: off @@ -282,7 +282,7 @@ class Child(Parent): assert "(default: 'This docstring" in helptext -def test_helptext_with_inheritance_overriden(): +def test_helptext_with_inheritance_overriden() -> None: @dataclasses.dataclass class Parent2: # fmt: off @@ -306,7 +306,7 @@ class Child2(Parent2): assert "Helptext! (default: 'This" in helptext -def test_tuple_helptext(): +def test_tuple_helptext() -> None: @dataclasses.dataclass class TupleHelptext: x: Tuple[int, str, float] @@ -315,7 +315,7 @@ class TupleHelptext: assert "--x INT STR FLOAT" in helptext -def test_tuple_helptext_defaults(): +def test_tuple_helptext_defaults() -> None: @dataclasses.dataclass class TupleHelptextDefaults: x: Tuple[int, str, str] = (5, "hello world", "hello") @@ -325,7 +325,7 @@ class TupleHelptextDefaults: assert "(default: 5 'hello world' hello)" in helptext -def test_generic_helptext(): +def test_generic_helptext() -> None: T = TypeVar("T") @dataclasses.dataclass @@ -336,7 +336,7 @@ class GenericTupleHelptext(Generic[T]): assert "--x INT" in helptext -def test_generic_tuple_helptext(): +def test_generic_tuple_helptext() -> None: T = TypeVar("T") @dataclasses.dataclass @@ -347,7 +347,7 @@ class GenericTupleHelptext(Generic[T]): assert "--x INT INT INT" in helptext -def test_generic_list_helptext(): +def test_generic_list_helptext() -> None: T = TypeVar("T") @dataclasses.dataclass @@ -358,7 +358,7 @@ class GenericTupleHelptext(Generic[T]): assert "--x INT [INT ...]" in helptext -def test_literal_helptext(): +def test_literal_helptext() -> None: @dataclasses.dataclass class LiteralHelptext: x: Literal[1, 2, 3] @@ -369,7 +369,7 @@ class LiteralHelptext: assert "A number. (required)" in helptext -def test_optional_literal_helptext(): +def test_optional_literal_helptext() -> None: @dataclasses.dataclass class OptionalLiteralHelptext: x: Optional[Literal[1, 2, 3]] = None @@ -380,7 +380,7 @@ class OptionalLiteralHelptext: assert "A number. (default: None)" in helptext -def test_multiple_subparsers_helptext(): +def test_multiple_subparsers_helptext() -> None: @dataclasses.dataclass class Subcommand1: """2% milk.""" # % symbol is prone to bugs in argparse. @@ -424,7 +424,7 @@ class MultipleSubparsers: assert "(default: c:subcommand3)" in helptext -def test_optional_helptext(): +def test_optional_helptext() -> None: @dataclasses.dataclass class OptionalHelptext: """This docstring should be printed as a description. 2% milk.""" @@ -438,14 +438,14 @@ class OptionalHelptext: """Documentation 3""" helptext = get_helptext(OptionalHelptext) - assert cast(str, OptionalHelptext.__doc__[:20]) in helptext + assert cast(str, cast(str, OptionalHelptext.__doc__)[:20]) in helptext assert "2% milk" in helptext assert "--x {None}|INT" in helptext assert "--y {None}|INT [{None}|INT ...]" in helptext assert "[--z {None}|INT]" in helptext -def test_metavar_0(): +def test_metavar_0() -> None: def main(x: Union[Literal[0, 1, 2, 3], Tuple[int, int]]) -> None: pass @@ -453,7 +453,7 @@ def main(x: Union[Literal[0, 1, 2, 3], Tuple[int, int]]) -> None: assert "--x {0,1,2,3}|{INT INT}" in helptext -def test_metavar_1(): +def test_metavar_1() -> None: def main( x: Union[ Literal[0, 1, 2, 3], @@ -468,7 +468,7 @@ def main( assert "--x {0,1,2,3,hey,there,hello}|{INT [INT ...]}" in helptext -def test_metavar_2(): +def test_metavar_2() -> None: def main( x: Tuple[ Literal[0, 1, 2, 3], @@ -481,7 +481,7 @@ def main( assert "--x {0,1,2,3} INT|STR" in helptext -def test_metavar_3(): +def test_metavar_3() -> None: def main( x: Union[ Literal[0, 1, 2, 3], @@ -494,7 +494,7 @@ def main( assert "--x {0,1,2,3}|{INT INT}|STR" in helptext -def test_metavar_4(): +def test_metavar_4() -> None: def main( x: Union[ Literal[0, 1, 2, 3], @@ -508,7 +508,7 @@ def main( assert "--x {0,1,2,3}|{INT INT}|{STR STR STR}|{True}" in helptext -def test_metavar_5(): +def test_metavar_5() -> None: def main( x: List[ Union[Tuple[int, int], Tuple[str, str]], @@ -520,7 +520,7 @@ def main( assert "[--x {INT INT}|{STR STR} [{INT INT}|{STR STR} ...]]" in helptext -def test_metavar_6(): +def test_metavar_6() -> None: def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> dict: return x @@ -530,7 +530,7 @@ def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> d ) -def test_comment_in_subclass_list(): +def test_comment_in_subclass_list() -> None: @dataclasses.dataclass class Something( # This text should not show up in the helptext anywhere. @@ -546,7 +546,7 @@ class Something( assert "But this text should!" in helptext -def test_unparsable(): +def test_unparsable() -> None: class Struct: a: int = 5 b: str = "7" diff --git a/tests/test_initvar_ignore_py37.py b/tests/test_initvar_ignore_py37.py index 6082ec1da..721462716 100644 --- a/tests/test_initvar_ignore_py37.py +++ b/tests/test_initvar_ignore_py37.py @@ -3,7 +3,7 @@ import tyro -def test_dataclass_init_var(): +def test_dataclass_init_var() -> None: @dataclasses.dataclass class DataclassWithInitVar: i: dataclasses.InitVar[int] diff --git a/tests/test_missing.py b/tests/test_missing.py index 75d6fa632..3b8b8a47c 100644 --- a/tests/test_missing.py +++ b/tests/test_missing.py @@ -8,7 +8,7 @@ import tyro -def test_missing(): +def test_missing() -> None: def main(a: int = 5, b: int = tyro.MISSING, c: int = 3) -> Tuple[int, int, int]: return a, b, c @@ -17,7 +17,7 @@ def main(a: int = 5, b: int = tyro.MISSING, c: int = 3) -> Tuple[int, int, int]: assert tyro.cli(main, args=["--b", "7"]) == (5, 7, 3) -def test_missing_dataclass(): +def test_missing_dataclass() -> None: @dataclasses.dataclass class Args2: a: int = 5 @@ -29,7 +29,7 @@ class Args2: assert tyro.cli(Args2, args=["--b", "7"]) == Args2(5, 7, 3) -def test_missing_default(): +def test_missing_default() -> None: @dataclasses.dataclass class Args2: a: int @@ -49,7 +49,7 @@ class Args2: ) == Args2(5, 7, 3) -def test_missing_nested_default(): +def test_missing_nested_default() -> None: @dataclasses.dataclass class Child: a: int = 5 diff --git a/tests/test_nested.py b/tests/test_nested.py index 884be2737..69cefff09 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -8,7 +8,7 @@ import tyro -def test_nested(): +def test_nested() -> None: @dataclasses.dataclass class B: y: int @@ -23,7 +23,7 @@ class Nested: tyro.cli(Nested, args=["--x", "1"]) -def test_nested_annotated(): +def test_nested_annotated() -> None: @dataclasses.dataclass class B: y: int @@ -38,7 +38,7 @@ class Nested: tyro.cli(Nested, args=["--x", "1"]) -def test_nested_accidental_underscores(): +def test_nested_accidental_underscores() -> None: @dataclasses.dataclass class B: arg_name: str @@ -59,7 +59,7 @@ class Nested: tyro.cli(Nested, args=["--x", "1"]) -def test_nested_default(): +def test_nested_default() -> None: @dataclasses.dataclass class B: y: int = 1 @@ -74,7 +74,7 @@ class Nested: ) -def test_nested_default_alternate(): +def test_nested_default_alternate() -> None: @dataclasses.dataclass class B: y: int = 3 @@ -92,7 +92,7 @@ class Nested: assert tyro.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=3)) -def test_default_nested(): +def test_default_nested() -> None: @dataclasses.dataclass(frozen=True) class B: y: int = 3 @@ -106,7 +106,7 @@ class Nested: assert tyro.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) -def test_double_default_nested(): +def test_double_default_nested() -> None: @dataclasses.dataclass(frozen=True) class Child: y: int @@ -128,7 +128,7 @@ class Grandparent: ) -def test_default_factory_nested(): +def test_default_factory_nested() -> None: @dataclasses.dataclass class B: y: int = 3 @@ -142,7 +142,7 @@ class Nested: assert tyro.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) -def test_optional_nested(): +def test_optional_nested() -> None: @dataclasses.dataclass class OptionalNestedChild: y: int @@ -169,7 +169,7 @@ class OptionalNested: ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) -def test_subparser(): +def test_subparser() -> None: @dataclasses.dataclass class HTTPServer: y: int @@ -201,7 +201,7 @@ class Subparser: tyro.cli(Subparser, args=["--x", "1", "bc:smtp-server", "--bc.y", "3"]) -def test_subparser_root(): +def test_subparser_root() -> None: @dataclasses.dataclass class HTTPServer: y: int @@ -220,7 +220,7 @@ class Subparser: ) == HTTPServer(y=3) -def test_subparser_with_default(): +def test_subparser_with_default() -> None: @dataclasses.dataclass class DefaultHTTPServer: y: int @@ -264,7 +264,7 @@ class DefaultSubparser: tyro.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) -def test_subparser_with_default_alternate(): +def test_subparser_with_default_alternate() -> None: @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 @@ -319,7 +319,7 @@ class DefaultInstanceSubparser: tyro.cli(DefaultInstanceSubparser, args=["--x", "1", "c", "--bc.y", "3"]) -def test_subparser_with_default_bad(): +def test_subparser_with_default_bad() -> None: @dataclasses.dataclass class DefaultHTTPServer: y: int @@ -341,7 +341,7 @@ class DefaultSubparser: ) -def test_optional_subparser(): +def test_optional_subparser() -> None: @dataclasses.dataclass class OptionalHTTPServer: y: int @@ -379,7 +379,7 @@ class OptionalSubparser: ) -def test_post_init_default(): +def test_post_init_default() -> None: @dataclasses.dataclass class DataclassWithDynamicDefault: x: int = 3 @@ -407,7 +407,7 @@ class DefaultFactoryPostInitArgs: ) -def test_multiple_subparsers(): +def test_multiple_subparsers() -> None: @dataclasses.dataclass class Subcommand1: x: int = 0 @@ -455,7 +455,7 @@ class MultipleSubparsers: ) == MultipleSubparsers(Subcommand3(z=5), Subcommand1(x=7), Subcommand3(z=3)) -def test_multiple_subparsers_with_default(): +def test_multiple_subparsers_with_default() -> None: @dataclasses.dataclass(frozen=True) class Subcommand1: x: int = 0 @@ -552,7 +552,7 @@ class MultipleSubparsers: ) == MultipleSubparsers(Subcommand1(x=0), Subcommand2(y=1), Subcommand2(y=1)) -def test_nested_subparsers_with_default(): +def test_nested_subparsers_with_default() -> None: @dataclasses.dataclass(frozen=True) class Subcommand1: x: int = 0 @@ -588,7 +588,7 @@ class MultipleSubparsers: ) == MultipleSubparsers(Subcommand2(Subcommand1(7))) -def test_nested_subparsers_multiple(): +def test_nested_subparsers_multiple() -> None: @dataclasses.dataclass(frozen=True) class Subcommand1: x: int = 0 @@ -628,7 +628,7 @@ class MultipleSubparsers: ) == MultipleSubparsers(Subcommand2(Subcommand1(3)), Subcommand2(Subcommand1(7))) -def test_tuple_nesting(): +def test_tuple_nesting() -> None: @dataclasses.dataclass(frozen=True) class Color: r: int @@ -653,7 +653,7 @@ def main(x: Tuple[Tuple[Color], Location, float]): ) == ((Color(255, 0, 0),), Location(5.0, 0.0, 2.0), 4.0) -def test_generic_subparsers(): +def test_generic_subparsers() -> None: T = TypeVar("T") @dataclasses.dataclass @@ -673,7 +673,7 @@ def main_with_default(x: Union[A[int], A[float]] = A(5)) -> Any: tyro.cli(main_with_default, args=[]) -def test_generic_inherited(): +def test_generic_inherited() -> None: class UnrelatedParentClass: pass @@ -698,7 +698,7 @@ class ChildClass(UnrelatedParentClass, ActualParentClass[int]): ) == ChildClass(x=1, y=2, z=3) -def test_subparser_in_nested(): +def test_subparser_in_nested() -> None: @dataclasses.dataclass class A: a: int @@ -729,7 +729,7 @@ class Parent: ) == Parent(Nested1(Nested2(B(7)))) -def test_frozen_dict(): +def test_frozen_dict() -> None: def main( x: Mapping[str, float] = frozendict( { @@ -745,7 +745,7 @@ def main( ) -def test_nested_in_subparser(): +def test_nested_in_subparser() -> None: # https://github.com/brentyi/tyro/issues/9 @dataclasses.dataclass(frozen=True) class Subtype: @@ -770,7 +770,7 @@ class Wrapper: ) -def test_nested_in_subparser_override_with_default(): +def test_nested_in_subparser_override_with_default() -> None: @dataclasses.dataclass class Mnist: binary: bool = False diff --git a/tests/test_nested_in_containers.py b/tests/test_nested_in_containers.py index bf0f1bf5e..91e095aac 100644 --- a/tests/test_nested_in_containers.py +++ b/tests/test_nested_in_containers.py @@ -14,7 +14,7 @@ class Color: b: int -def test_nested_tuple_fixed_single(): +def test_nested_tuple_fixed_single() -> None: def main(x: Tuple[Color]) -> Any: return x @@ -23,7 +23,7 @@ def main(x: Tuple[Color]) -> Any: ) -def test_nested_tuple_fixed_two(): +def test_nested_tuple_fixed_two() -> None: def main(x: Tuple[Color, Color]) -> Any: return x @@ -38,7 +38,7 @@ def main(x: Tuple[Color, Color]) -> Any: ) -def test_nested_tuple_fixed_three(): +def test_nested_tuple_fixed_three() -> None: def main(x: Tuple[Color, int, Color]) -> Any: return x @@ -55,7 +55,7 @@ def main(x: Tuple[Color, int, Color]) -> Any: ) -def test_nested_tuple_recursive(): +def test_nested_tuple_recursive() -> None: def main(x: Tuple[Color, Tuple[Color, Color]]) -> Any: return x @@ -74,7 +74,7 @@ def main(x: Tuple[Color, Tuple[Color, Color]]) -> Any: ) -def test_tuple_bad(): +def test_tuple_bad() -> None: # Unable to infer input length. def main(x: Tuple[Color, ...]) -> None: pass @@ -83,7 +83,7 @@ def main(x: Tuple[Color, ...]) -> None: tyro.cli(main, args=[]) -def test_set_bad(): +def test_set_bad() -> None: # Unable to infer input length. def main(x: Set[Color]) -> None: pass @@ -92,7 +92,7 @@ def main(x: Set[Color]) -> None: tyro.cli(main, args=[]) -def test_set_ok(): +def test_set_ok() -> None: def main(x: Set[Color] = {Color(255, 0, 0)}) -> Any: return x @@ -100,7 +100,7 @@ def main(x: Set[Color] = {Color(255, 0, 0)}) -> Any: assert tyro.cli(main, args="--x.0.r 127".split(" ")) == {Color(127, 0, 0)} -def test_list_bad(): +def test_list_bad() -> None: # Unable to infer input length. def main(x: List[Color]) -> None: pass @@ -109,7 +109,7 @@ def main(x: List[Color]) -> None: tyro.cli(main, args=[]) -def test_list_ok(): +def test_list_ok() -> None: def main(x: List[Color] = [Color(255, 0, 0)]) -> Any: return x @@ -117,7 +117,7 @@ def main(x: List[Color] = [Color(255, 0, 0)]) -> Any: assert tyro.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] -def test_list_object(): +def test_list_object() -> None: def main(x: List[object] = [Color(255, 0, 0)]) -> Any: return x @@ -125,7 +125,7 @@ def main(x: List[object] = [Color(255, 0, 0)]) -> Any: assert tyro.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] -def test_list_any(): +def test_list_any() -> None: def main(x: List[Any] = [Color(255, 0, 0)]) -> Any: return x @@ -133,7 +133,7 @@ def main(x: List[Any] = [Color(255, 0, 0)]) -> Any: assert tyro.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] -def test_tuple_in_list(): +def test_tuple_in_list() -> None: def main(x: List[Tuple[Color]] = [(Color(255, 0, 0),)]) -> Any: return x @@ -141,7 +141,7 @@ def main(x: List[Tuple[Color]] = [(Color(255, 0, 0),)]) -> Any: assert tyro.cli(main, args="--x.0.0.r 127".split(" ")) == [(Color(127, 0, 0),)] -def test_tuple_variable(): +def test_tuple_variable() -> None: def main(x: Tuple[Color, ...] = (Color(255, 0, 0), Color(255, 0, 127))) -> Any: return x @@ -152,7 +152,7 @@ def main(x: Tuple[Color, ...] = (Color(255, 0, 0), Color(255, 0, 127))) -> Any: ) -def test_dict_bad(): +def test_dict_bad() -> None: def main(x: Dict[str, Color]) -> Any: return x @@ -160,7 +160,7 @@ def main(x: Dict[str, Color]) -> Any: tyro.cli(main, args=[]) -def test_dict_ok(): +def test_dict_ok() -> None: def main( x: Dict[str, Color] = { "red": Color(255, 0, 0), @@ -176,7 +176,7 @@ def main( ) -def test_dict_key_int(): +def test_dict_key_int() -> None: def main( x: Dict[int, Color] = { 0: Color(255, 0, 0), @@ -190,7 +190,7 @@ def main( assert tyro.cli(main, args="--x.1.g 127".split(" "))[1] == Color(0, 127, 0) -def test_dict_key_enum(): +def test_dict_key_enum() -> None: class ColorType(enum.Enum): RED = enum.auto() GREEN = enum.auto() @@ -211,7 +211,7 @@ def main( ) -def test_dict_nested(): +def test_dict_nested() -> None: def main( x: Dict[str, Tuple[Color, int]] = { # For each color: RGB and xterm color code. @@ -228,7 +228,7 @@ def main( ] == (Color(0, 127, 0), 2) -def test_generic_in_tuple(): +def test_generic_in_tuple() -> None: ScalarType = TypeVar("ScalarType", int, float) @dataclasses.dataclass @@ -250,7 +250,7 @@ def main(x: Tuple[GenericColor[float], GenericColor[int]]) -> Any: ) == (GenericColor(0.5, 0.2, 0.3), GenericColor(25, 2, 3)) -def test_generic_in_tuple_with_default(): +def test_generic_in_tuple_with_default() -> None: ScalarType = TypeVar("ScalarType", int, float) @dataclasses.dataclass @@ -277,7 +277,7 @@ def main( ) == (GenericColor(0.5, 0.2, 0.3), GenericColor(25, 2, 3)) -def test_generic_in_variable_tuple_with_default(): +def test_generic_in_variable_tuple_with_default() -> None: ScalarType = TypeVar("ScalarType", int, float) @dataclasses.dataclass @@ -304,7 +304,7 @@ def main( ) == (GenericColor(0.5, 0.9, 0.3), GenericColor(25, 2, 3)) -def test_generic_in_dict_with_default(): +def test_generic_in_dict_with_default() -> None: ScalarType = TypeVar("ScalarType", int, float) @dataclasses.dataclass @@ -330,7 +330,7 @@ def main( ) == {"float": GenericColor(0.5, 0.2, 0.3), "int": GenericColor(25, 0, 3)} -def test_generic_in_double_nested_dict_with_default(): +def test_generic_in_double_nested_dict_with_default() -> None: ScalarType = TypeVar("ScalarType", int, float) @dataclasses.dataclass @@ -357,7 +357,7 @@ def main( } -def test_double_nested_dict_with_inferred_type(): +def test_double_nested_dict_with_inferred_type() -> None: def main( x: Dict[str, Any] = { "hello": { diff --git a/tests/test_pydantic.py b/tests/test_pydantic.py index 50e1f3491..da9621e7a 100644 --- a/tests/test_pydantic.py +++ b/tests/test_pydantic.py @@ -9,7 +9,7 @@ import tyro -def test_pydantic(): +def test_pydantic() -> None: class ManyTypesA(BaseModel): i: int s: str = "hello" @@ -28,7 +28,7 @@ class ManyTypesA(BaseModel): ) == ManyTypesA(i=5, s="hello", f=3.0, p=pathlib.Path("~")) -def test_pydantic_helptext(): +def test_pydantic_helptext() -> None: class Helptext(BaseModel): """This docstring should be printed as a description.""" diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 13655f1ac..bc15229af 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -43,7 +43,7 @@ class ArgumentDefinition: name_prefix: str # User-facing prefix. subcommand_prefix: str # Prefix for nesting. field: _fields.FieldDefinition - type_from_typevar: Dict[TypeVar, Type] + type_from_typevar: Dict[TypeVar, Type[Any]] def add_argument( self, parser: Union[argparse.ArgumentParser, argparse._ArgumentGroup] diff --git a/tyro/_fields.py b/tyro/_fields.py index f96d7367c..23623cacf 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -33,24 +33,14 @@ from . import _docstrings, _instantiators, _resolver, _singleton, _strings from .conf import _confstruct, _markers -# Support attrs and pydantic if they're installed. -try: - import attr -except ImportError: - attr = None # type: ignore -try: - import pydantic -except ImportError: - pydantic = None # type: ignore - @dataclasses.dataclass(frozen=True) class FieldDefinition: name: str - typ: Type + typ: Type[Any] default: Any helptext: Optional[str] - markers: FrozenSet[_markers.Marker] + markers: FrozenSet[_markers._Marker] argconf: _confstruct._ArgConfiguration @@ -69,12 +59,12 @@ def __post_init__(self): @staticmethod def make( name: str, - typ: Type, + typ: Type[Any], default: Any, helptext: Optional[str], call_argname_override: Optional[Any] = None, *, - markers: Tuple[_markers.Marker, ...] = (), + markers: Tuple[_markers._Marker, ...] = (), ): # Try to extract argconf overrides from type. _, argconfs = _resolver.unwrap_annotated(typ, _confstruct._ArgConfiguration) @@ -85,7 +75,7 @@ def make( (argconf,) = argconfs helptext = argconf.help - typ, inferred_markers = _resolver.unwrap_annotated(typ, _markers.Marker) + typ, inferred_markers = _resolver.unwrap_annotated(typ, _markers._Marker) return FieldDefinition( name if argconf.name is None else argconf.name, typ, @@ -96,7 +86,7 @@ def make( call_argname_override if call_argname_override is not None else name, ) - def add_markers(self, markers: Tuple[_markers.Marker, ...]) -> FieldDefinition: + def add_markers(self, markers: Tuple[_markers._Marker, ...]) -> FieldDefinition: return dataclasses.replace( self, markers=self.markers.union(markers), @@ -170,7 +160,7 @@ class UnsupportedNestedTypeMessage: message: str -def is_nested_type(typ: Type, default_instance: _DefaultInstance) -> bool: +def is_nested_type(typ: Type[Any], default_instance: _DefaultInstance) -> bool: """Determine whether a type should be treated as a 'nested type', where a single type can be broken down into multiple fields (eg for nested dataclasses or classes).""" @@ -181,7 +171,7 @@ def is_nested_type(typ: Type, default_instance: _DefaultInstance) -> bool: def field_list_from_callable( - f: Union[Callable, Type], + f: Union[Callable, Type[Any]], default_instance: _DefaultInstance, ) -> List[FieldDefinition]: """Generate a list of generic 'field' objects corresponding to the inputs of some @@ -192,7 +182,7 @@ def field_list_from_callable( raise _instantiators.UnsupportedTypeAnnotationError(out.message) # Recursively apply markers. - _, parent_markers = _resolver.unwrap_annotated(f, _markers.Marker) + _, parent_markers = _resolver.unwrap_annotated(f, _markers._Marker) out = list(map(lambda field: field.add_markers(parent_markers), out)) return out @@ -218,7 +208,7 @@ def field_list_from_callable( def _try_field_list_from_callable( - f: Union[Callable, Type], + f: Union[Callable, Type[Any]], default_instance: _DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: f, found_subcommand_configs = _resolver.unwrap_annotated( @@ -230,79 +220,48 @@ def _try_field_list_from_callable( # Unwrap generics. f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) + f_origin = _resolver.unwrap_origin_strip_extras(cast(Type, f)) # If `f` is a type: # 1. Set cls to the type. # 2. Consider `f` to be `cls.__init__`. - cls: Optional[Type] = None + cls: Optional[Type[Any]] = None if isinstance(f, type): cls = f f = cls.__init__ # type: ignore - f_origin: Callable = cls # type: ignore - f_origin = _resolver.unwrap_origin_strip_extras(f) + f_origin = cls # type: ignore - # Try special cases. + # Try field generation from class inputs. if cls is not None: - if is_typeddict(cls): - return _try_field_list_from_typeddict(cls, default_instance) - - if _resolver.is_namedtuple(cls): - return _try_field_list_from_namedtuple(cls, default_instance) - - if _resolver.is_dataclass(cls): - return _try_field_list_from_dataclass(cls, default_instance) - - if pydantic is not None and issubclass(cls, pydantic.BaseModel): - return _try_field_list_from_pydantic(cls, default_instance) - - if attr is not None and attr.has(cls): - return _try_field_list_from_attrs(cls, default_instance) - - # Standard container types. These are special because they can be nested structures + for match, field_list_from_class in ( + (is_typeddict, _field_list_from_typeddict), + (_resolver.is_namedtuple, _field_list_from_namedtuple), + (_resolver.is_dataclass, _field_list_from_dataclass), + (_is_attrs, _field_list_from_attrs), + (_is_pydantic, _field_list_from_pydantic), + ): + if match(cls): + return field_list_from_class(cls, default_instance) + + # Standard container types. These are different because they can be nested structures # if they contain other nested types (eg Tuple[Struct, Struct]), or treated as # single arguments otherwise (eg Tuple[int, int]). # # Note that f_origin will be populated if we annotate as `Tuple[..]`, and cls will - # be populated if we annotated as just `tuple`. - container_fields = None + # be populated if we annotate as just `tuple`. if f_origin is tuple or cls is tuple: - container_fields = _field_list_from_tuple(f, default_instance) + return _field_list_from_tuple(f, default_instance) + elif f_origin in (collections.abc.Mapping, dict) or cls in ( + collections.abc.Mapping, + dict, + ): + return _field_list_from_dict(f, default_instance) elif f_origin in (list, set, typing.Sequence) or cls in ( list, set, typing.Sequence, ): - contained_type: Any - if len(get_args(f)) == 0: - if default_instance in MISSING_SINGLETONS: - raise _instantiators.UnsupportedTypeAnnotationError( - f"Sequence type {cls} needs either an explicit type or a" - " default to infer from." - ) - assert isinstance(default_instance, Iterable) - contained_type = next(iter(default_instance)) - else: - (contained_type,) = get_args(f) - f_origin = list if f_origin is typing.Sequence else f_origin # type: ignore - - container_fields = _try_field_list_from_sequence( - contained_type, default_instance - ) - elif f_origin in (collections.abc.Mapping, dict) or cls in ( - collections.abc.Mapping, - dict, - ): - container_fields = _try_field_list_from_dict(f, default_instance) - - # Check if one of the container types matched. - if container_fields is not None: - # Found fields! - if not isinstance(container_fields, UnsupportedNestedTypeMessage): - return container_fields - # Got an error, - else: - assert isinstance(container_fields, UnsupportedNestedTypeMessage) - return container_fields + return _field_list_from_sequence_checked(f, default_instance) # General cases. if ( @@ -313,8 +272,8 @@ def _try_field_list_from_callable( return _try_field_list_from_general_callable(f, cls, default_instance) -def _try_field_list_from_typeddict( - cls: Type, default_instance: _DefaultInstance +def _field_list_from_typeddict( + cls: Type[Any], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: field_list = [] valid_default_instance = ( @@ -345,8 +304,8 @@ def _try_field_list_from_typeddict( return field_list -def _try_field_list_from_namedtuple( - cls: Type, default_instance: _DefaultInstance +def _field_list_from_namedtuple( + cls: Type[Any], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Handle NamedTuples. # @@ -376,8 +335,8 @@ def _try_field_list_from_namedtuple( return field_list -def _try_field_list_from_dataclass( - cls: Type, default_instance: _DefaultInstance +def _field_list_from_dataclass( + cls: Type[Any], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Handle dataclasses. field_list = [] @@ -405,8 +364,19 @@ def _try_field_list_from_dataclass( return field_list -def _try_field_list_from_pydantic( - cls: Type, default_instance: _DefaultInstance +# Support attrs and pydantic if they're installed. +try: + import pydantic +except ImportError: + pass + + +def _is_pydantic(cls: Type[Any]) -> bool: + return pydantic is not None and issubclass(cls, pydantic.BaseModel) + + +def _field_list_from_pydantic( + cls: Type[Any], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: assert pydantic is not None @@ -426,8 +396,18 @@ def _try_field_list_from_pydantic( return field_list -def _try_field_list_from_attrs( - cls: Type, default_instance: _DefaultInstance +try: + import attr +except ImportError: + attr = None # type: ignore + + +def _is_attrs(cls: Type[Any]) -> bool: + return attr is not None and attr.has(cls) + + +def _field_list_from_attrs( + cls: Type[Any], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: assert attr is not None @@ -454,13 +434,13 @@ def _try_field_list_from_attrs( def _field_list_from_tuple( - f: Union[Callable, Type], default_instance: _DefaultInstance + f: Union[Callable, Type[Any]], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Fixed-length tuples. field_list = [] children = get_args(f) if Ellipsis in children: - return _try_field_list_from_sequence( + return _try_field_list_from_sequence_inner( next(iter(set(children) - {Ellipsis})), default_instance ) @@ -513,8 +493,25 @@ def _field_list_from_tuple( return field_list -def _try_field_list_from_sequence( - contained_type: Type, +def _field_list_from_sequence_checked( + f: Union[Callable, Type[Any]], default_instance: _DefaultInstance +) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: + contained_type: Any + if len(get_args(f)) == 0: + if default_instance in MISSING_SINGLETONS: + raise _instantiators.UnsupportedTypeAnnotationError( + f"Sequence type {f} needs either an explicit type or a" + " default to infer from." + ) + assert isinstance(default_instance, Iterable) + contained_type = next(iter(default_instance)) + else: + (contained_type,) = get_args(f) + return _try_field_list_from_sequence_inner(contained_type, default_instance) + + +def _try_field_list_from_sequence_inner( + contained_type: Type[Any], default_instance: _DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # When no default instance is specified: @@ -557,8 +554,8 @@ def _try_field_list_from_sequence( return field_list -def _try_field_list_from_dict( - f: Union[Callable, Type], +def _field_list_from_dict( + f: Union[Callable, Type[Any]], default_instance: _DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: if default_instance in MISSING_SINGLETONS: @@ -581,8 +578,8 @@ def _try_field_list_from_dict( def _try_field_list_from_general_callable( - f: Union[Callable, Type], - cls: Optional[Type], + f: Union[Callable, Type[Any]], + cls: Optional[Type[Any]], default_instance: _DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Handle general callables. @@ -612,7 +609,9 @@ def _try_field_list_from_general_callable( def _field_list_from_params( - f: Union[Callable, Type], cls: Optional[Type], params: List[inspect.Parameter] + f: Union[Callable, Type[Any]], + cls: Optional[Type[Any]], + params: List[inspect.Parameter], ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Get type annotations, docstrings. docstring = inspect.getdoc(f) diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 4104bbaaa..5e86837a5 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -94,7 +94,7 @@ class UnsupportedTypeAnnotationError(Exception): def instantiator_from_type( - typ: Type, type_from_typevar: Dict[TypeVar, Type] + typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] ) -> Tuple[Instantiator, InstantiatorMetadata]: """Recursive helper for parsing type annotations. @@ -216,7 +216,7 @@ def instantiator_base_case(strings: List[str]) -> Any: @overload def _instantiator_from_type_inner( typ: Type, - type_from_typevar: Dict[TypeVar, Type], + type_from_typevar: Dict[TypeVar, Type[Any]], allow_sequences: Literal["fixed_length"], ) -> Tuple[Instantiator, InstantiatorMetadata]: ... @@ -225,7 +225,7 @@ def _instantiator_from_type_inner( @overload def _instantiator_from_type_inner( typ: Type, - type_from_typevar: Dict[TypeVar, Type], + type_from_typevar: Dict[TypeVar, Type[Any]], allow_sequences: Literal[False], ) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: ... @@ -234,7 +234,7 @@ def _instantiator_from_type_inner( @overload def _instantiator_from_type_inner( typ: Type, - type_from_typevar: Dict[TypeVar, Type], + type_from_typevar: Dict[TypeVar, Type[Any]], allow_sequences: Literal[True], ) -> Tuple[Instantiator, InstantiatorMetadata]: ... @@ -242,7 +242,7 @@ def _instantiator_from_type_inner( def _instantiator_from_type_inner( typ: Type, - type_from_typevar: Dict[TypeVar, Type], + type_from_typevar: Dict[TypeVar, Type[Any]], allow_sequences: Literal["fixed_length", True, False], ) -> Tuple[Instantiator, InstantiatorMetadata]: """Thin wrapper over instantiator_from_type, with some extra asserts for catching @@ -261,7 +261,7 @@ def _instantiator_from_type_inner( def _instantiator_from_container_type( - typ: Type, type_from_typevar: Dict[TypeVar, Type] + typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] ) -> Optional[Tuple[Instantiator, InstantiatorMetadata]]: """Attempt to create an instantiator from a container type. Returns `None` is no container type is found.""" @@ -297,7 +297,7 @@ def _instantiator_from_container_type( def _instantiator_from_tuple( - typ: Type, type_from_typevar: Dict[TypeVar, Type] + typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] ) -> Tuple[Instantiator, InstantiatorMetadata]: types = get_args(typ) typeset = set(types) # Note that sets are unordered. @@ -381,7 +381,7 @@ def _join_union_metavars(metavars: Iterable[str]) -> str: def _instantiator_from_union( - typ: Type, type_from_typevar: Dict[TypeVar, Type] + typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] ) -> Tuple[Instantiator, InstantiatorMetadata]: options = list(get_args(typ)) if NoneType in options: @@ -457,7 +457,7 @@ def union_instantiator(strings: List[str]) -> Any: def _instantiator_from_dict( - typ: Type, type_from_typevar: Dict[TypeVar, Type] + typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] ) -> Tuple[Instantiator, InstantiatorMetadata]: key_type, val_type = get_args(typ) key_instantiator, key_meta = _instantiator_from_type_inner( @@ -513,7 +513,7 @@ def dict_instantiator(strings: List[str]) -> Any: def _instantiator_from_sequence( - typ: Type, type_from_typevar: Dict[TypeVar, Type] + typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] ) -> Tuple[Instantiator, InstantiatorMetadata]: """Instantiator for variable-length sequences: list, sets, Tuple[T, ...], etc.""" container_type = get_origin(typ) @@ -557,7 +557,7 @@ def sequence_instantiator(strings: List[str]) -> Any: def _instantiator_from_literal( - typ: Type, type_from_typevar: Dict[TypeVar, Type] + typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] ) -> Tuple[Instantiator, InstantiatorMetadata]: choices = get_args(typ) str_choices = tuple(x.name if isinstance(x, enum.Enum) else str(x) for x in choices) diff --git a/tyro/_parsers.py b/tyro/_parsers.py index d6194a746..d75bedacb 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -58,7 +58,7 @@ class ParserSpecification: def from_callable_or_type( f: Callable[..., T], description: Optional[str], - parent_classes: Set[Type], + parent_classes: Set[Type[Any]], default_instance: Union[ T, _fields.PropagatingMissingType, _fields.NonpropagatingMissingType ], @@ -70,7 +70,7 @@ def from_callable_or_type( # Consolidate subcommand types. consolidate_subcommand_args = ( _markers.ConsolidateSubcommandArgs - in _resolver.unwrap_annotated(f, _markers.Marker)[1] + in _resolver.unwrap_annotated(f, _markers._Marker)[1] ) # Resolve generic types. @@ -294,7 +294,9 @@ def format_group_name(prefix: str) -> str: arg.lowered.help is not argparse.SUPPRESS and arg.dest_prefix not in group_from_prefix ): - description = self.helptext_from_nested_class_field_name.get(arg.dest_prefix) + description = self.helptext_from_nested_class_field_name.get( + arg.dest_prefix + ) group_from_prefix[arg.dest_prefix] = parser.add_argument_group( format_group_name(arg.dest_prefix), description=description, @@ -330,8 +332,8 @@ class SubparsersSpecification: @staticmethod def from_field( field: _fields.FieldDefinition, - type_from_typevar: Dict[TypeVar, Type], - parent_classes: Set[Type], + type_from_typevar: Dict[TypeVar, Type[Any]], + parent_classes: Set[Type[Any]], prefix: str, ) -> Optional[SubparsersSpecification]: # Union of classes should create subparsers. diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 62856e979..d22b4b45d 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -43,7 +43,7 @@ def is_dataclass(cls: Union[Type, Callable]) -> bool: def resolve_generic_types( cls: TypeOrCallable, -) -> Tuple[TypeOrCallable, Dict[TypeVar, Type]]: +) -> Tuple[TypeOrCallable, Dict[TypeVar, Type[Any]]]: """If the input is a class: no-op. If it's a generic alias: returns the origin class, and a mapping from typevars to concrete types.""" @@ -198,7 +198,7 @@ def unwrap_annotated( def apply_type_from_typevar( - typ: TypeOrCallable, type_from_typevar: Dict[TypeVar, Type] + typ: TypeOrCallable, type_from_typevar: Dict[TypeVar, Type[Any]] ) -> TypeOrCallable: if typ in type_from_typevar: return type_from_typevar[typ] # type: ignore @@ -224,7 +224,7 @@ def apply_type_from_typevar( } if hasattr(types, "UnionType"): # type: ignore # PEP 604. Requires Python 3.10. - shim_table[types.UnionType] = Union # type: ignore + shim_table[types.UnionType[Any]] = Union # type: ignore for new, old in shim_table.items(): if isinstance(typ, new) or get_origin(typ) is new: # type: ignore diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index ec71a8ba8..95e8f09a8 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Callable, Type, TypeVar +from typing import TYPE_CHECKING, Any, Callable, TypeVar from typing_extensions import Annotated @@ -12,7 +12,7 @@ # # An alias could ideally be made, but SpecialForm aliases are not well supported by static analysis tools. -T = TypeVar("T", bound=Type) +T = TypeVar("T") Positional = Annotated[T, None] """A type `T` can be annotated as `Positional[T]` if we want to parse it as a positional @@ -92,11 +92,14 @@ # - Annotated[T, Marker] -class Marker(_singleton.Singleton): +class _Marker(_singleton.Singleton): def __getitem__(self, key): return Annotated.__class_getitem__((key, self)) # type: ignore +Marker = Any + + def configure(*markers: Marker) -> Callable[[CallableType], CallableType]: """Decorator for configuring functions. @@ -125,8 +128,8 @@ def _inner(callable: CallableType) -> CallableType: if not TYPE_CHECKING: - def _make_marker(description: str) -> Marker: - class _InnerMarker(Marker): + def _make_marker(description: str) -> _Marker: + class _InnerMarker(_Marker): def __repr__(self): return description diff --git a/tyro/extras/_serialization.py b/tyro/extras/_serialization.py index ea534062d..c61e674b1 100644 --- a/tyro/extras/_serialization.py +++ b/tyro/extras/_serialization.py @@ -18,9 +18,9 @@ def _get_contained_special_types_from_type( - cls: Type, - _parent_contained_dataclasses: Optional[Set[Type]] = None, -) -> Set[Type]: + cls: Type[Any], + _parent_contained_dataclasses: Optional[Set[Type[Any]]] = None, +) -> Set[Type[Any]]: """Takes a dataclass type, and recursively searches its fields for dataclass or enum types.""" assert _resolver.is_dataclass(cls) @@ -35,7 +35,7 @@ def _get_contained_special_types_from_type( contained_special_types = {cls} - def handle_type(typ: Type) -> Set[Type]: + def handle_type(typ: Type[Any]) -> Set[Type[Any]]: # Handle dataclasses. if _resolver.is_dataclass(typ) and typ not in parent_contained_dataclasses: return _get_contained_special_types_from_type( @@ -69,7 +69,7 @@ def handle_type(typ: Type) -> Set[Type]: return contained_special_types -def _make_loader(cls: Type) -> Type[yaml.Loader]: +def _make_loader(cls: Type[Any]) -> Type[yaml.Loader]: class DataclassLoader(yaml.Loader): pass @@ -91,10 +91,10 @@ class DataclassLoader(yaml.Loader): loader: yaml.Loader node: yaml.Node - def make_dataclass_constructor(typ: Type): + def make_dataclass_constructor(typ: Type[Any]): return lambda loader, node: typ(**loader.construct_mapping(node)) - def make_enum_constructor(typ: Type): + def make_enum_constructor(typ: Type[Any]): return lambda loader, node: typ[loader.construct_python_str(node)] for typ, name in zip(contained_types, contained_type_names): From d8401edc34364205f0395da5a7cdbecff4aabb0d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 28 Nov 2022 12:45:20 -0800 Subject: [PATCH 240/491] Fixes for Python 3.10, mypy --- examples/03_config_systems/01_base_configs.py | 112 ++++++++---------- tests/test_conf.py | 6 +- tyro/_cli.py | 6 +- tyro/_parsers.py | 4 + tyro/_resolver.py | 2 +- tyro/conf/_markers.py | 2 +- tyro/extras/_base_configs.py | 29 +++-- tyro/extras/_choices_type.py | 21 +++- 8 files changed, 101 insertions(+), 81 deletions(-) diff --git a/examples/03_config_systems/01_base_configs.py b/examples/03_config_systems/01_base_configs.py index 8281b18c4..1f34f8ddb 100644 --- a/examples/03_config_systems/01_base_configs.py +++ b/examples/03_config_systems/01_base_configs.py @@ -11,16 +11,17 @@ Usage: `python ./10_base_configs.py --help` -`python ./10_base_configs.py small --help` -`python ./10_base_configs.py small --seed 94720` -`python ./10_base_configs.py big --help` -`python ./10_base_configs.py big --seed 94720` +`python ./10_base_configs.py config:small --help` +`python ./10_base_configs.py config:small --config.seed 94720` +`python ./10_base_configs.py config:big --help` +`python ./10_base_configs.py config:big --config.seed 94720` """ from dataclasses import dataclass from typing import Callable, Literal, Tuple, Union from torch import nn +from typing_extensions import Annotated import tyro @@ -31,18 +32,13 @@ class AdamOptimizer: betas: Tuple[float, float] = (0.9, 0.999) -@dataclass(frozen=True) -class SgdOptimizer: - learning_rate: float = 3e-4 - - @dataclass(frozen=True) class ExperimentConfig: # Dataset to run experiment on. dataset: Literal["mnist", "imagenet-50"] # Optimizer parameters. - optimizer: Union[AdamOptimizer, SgdOptimizer] + optimizer: AdamOptimizer # Model size. num_layers: int @@ -65,61 +61,49 @@ class ExperimentConfig: # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. -descriptions = {} -base_configs = {} - -descriptions["small"] = "Train a smaller model." -base_configs["small"] = ExperimentConfig( - dataset="mnist", - optimizer=SgdOptimizer(), - batch_size=2048, - num_layers=4, - units=64, - train_steps=30_000, - # The tyro.MISSING sentinel allows us to specify that the seed should have no - # default, and needs to be populated from the CLI. - seed=tyro.MISSING, - activation=nn.ReLU, -) - - -descriptions["big"] = "Train a bigger model." -base_configs["big"] = ExperimentConfig( - dataset="imagenet-50", - optimizer=AdamOptimizer(), - batch_size=32, - num_layers=8, - units=256, - train_steps=100_000, - seed=tyro.MISSING, - activation=nn.GELU, -) +SmallConfig = Annotated[ + ExperimentConfig, + tyro.conf.subcommand( + name="small", + default=ExperimentConfig( + dataset="mnist", + optimizer=AdamOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + seed=0, + activation=nn.ReLU, + ), + description="Train a smaller model.", + ), +] +BigConfig = Annotated[ + ExperimentConfig, + tyro.conf.subcommand( + name="big", + default=ExperimentConfig( + dataset="imagenet-50", + optimizer=AdamOptimizer(), + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=0, + activation=nn.GELU, + ), + description="Train a bigger model.", + ), +] + + +def main( + config: Union[SmallConfig, BigConfig], + restore_checkpoint: bool = False, +) -> None: + print(config) if __name__ == "__main__": - config = tyro.cli( - tyro.extras.subcommand_type_from_defaults(base_configs, descriptions), - ) - # ^Note that this is equivalent to: - # - # config = tyro.cli( - # Union[ - # Annotated[ - # ExperimentConfig, - # tyro.conf.subcommand( - # "small", - # default=base_configs["small"], - # description=descriptions["small"], - # ), - # ], - # Annotated[ - # ExperimentConfig, - # tyro.conf.subcommand( - # "big", - # default=base_configs["big"], - # description=descriptions["big"], - # ), - # ], - # ] - # ) + config = tyro.cli(main) print(config) diff --git a/tests/test_conf.py b/tests/test_conf.py index 57597b1e3..532b2d4e1 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -561,7 +561,7 @@ class DefaultInstanceSubparser: # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] bc: tyro.conf.OmitSubcommandPrefixes[ Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] - ] + ] = DefaultInstanceHTTPServer() assert ( tyro.cli( @@ -615,6 +615,10 @@ class DefaultInstanceSubparser: == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) ) + # Despite all defaults being set, a subcommand should be required. + with pytest.raises(SystemExit): + tyro.cli(tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[]) + def test_omit_subcommand_prefix_and_consolidate_subcommand_args_in_function() -> None: @dataclasses.dataclass diff --git a/tyro/_cli.py b/tyro/_cli.py index e6f0a3e61..e1502e82e 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -21,11 +21,13 @@ # Overload notes: -# 1. Type[T] is almost a subtype of Callable[..., T]; the difference is types like -# Union[T1, T2] which fall under the former but not the latter. +# 1. Type[T] is almost a subtype of Callable[..., T]; the difference is (in pyright) +# types like Union[T1, T2] which fall under the former but not the latter. # 2. We really shouldn't need an overload here. But as of 1.1.268, it seems like it's # needed for pyright to understand that Union types are OK to pass in directly. # Hopefully we can just switch to a Union[Type[...], Callable[...]] in the future. +# 3. For equivalent functionality in mypy, we need to wait for typing.TypeForm: +# https://github.com/python/mypy/issues/9773 @overload diff --git a/tyro/_parsers.py b/tyro/_parsers.py index d75bedacb..acce5cb30 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -490,6 +490,10 @@ def from_field( ): required = True + # Required if all args are pushed to the final subcommand. + if _markers.ConsolidateSubcommandArgs in field.markers: + required = True + # Make description. description_parts = [] if field.helptext is not None: diff --git a/tyro/_resolver.py b/tyro/_resolver.py index d22b4b45d..b93c64fed 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -224,7 +224,7 @@ def apply_type_from_typevar( } if hasattr(types, "UnionType"): # type: ignore # PEP 604. Requires Python 3.10. - shim_table[types.UnionType[Any]] = Union # type: ignore + shim_table[types.UnionType] = Union # type: ignore for new, old in shim_table.items(): if isinstance(typ, new) or get_origin(typ) is new: # type: ignore diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index 95e8f09a8..43a2b88e3 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -114,7 +114,7 @@ def configure(*markers: Marker) -> Callable[[CallableType], CallableType]: ```python # Recursively apply FlagConversionOff to all field in `main()`. - @tyro.conf.configure_function(tyro.conf.FlagConversionOff) + @tyro.conf.configure(tyro.conf.FlagConversionOff) def main(field: bool) -> None: ... ``` diff --git a/tyro/extras/_base_configs.py b/tyro/extras/_base_configs.py index 1baf6b29a..e755b1012 100644 --- a/tyro/extras/_base_configs.py +++ b/tyro/extras/_base_configs.py @@ -15,6 +15,25 @@ def subcommand_type_from_defaults( ) -> Type[T]: """Construct a Union type for defining subcommands that choose between defaults. + .. warning:: + Use of this helper is discouraged because it is compatible with `pyright` and + `pylance`, but not with `mypy`. For `pyright` support, you may need to enable + postponed evaluation of annotations (`from __future__ import annotations`). + + At the cost of verbosity, using :func:`tyro.conf.subcommand()` directly is + better supported by external tools. + + Alternatively, we can work around this limitation with an `if TYPE_CHECKING` + guard: + ```python + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + SelectableConfig = Config # For mypy. + else: + SelectableConfig = subcommand_type_from_defaults(...) + ``` + This can most commonly be used to create a "base configuration" pattern: https://brentyi.github.io/tyro/examples/10_base_configs/ @@ -62,16 +81,6 @@ def train( tyro.cli(train) ``` - - Note that Pyright understands the latter case, but mypy does not. If mypy support is - necessary we can work around this with an `if TYPE_CHECKING` guard: - - ```python - if TYPE_CHECKING: - SelectableConfig = Config - else: - SelectableConfig = subcommand_type_from_defaults(base_mapping) - ``` """ return Union.__getitem__( # type: ignore tuple( diff --git a/tyro/extras/_choices_type.py b/tyro/extras/_choices_type.py index 96a06b033..bb5335ff2 100644 --- a/tyro/extras/_choices_type.py +++ b/tyro/extras/_choices_type.py @@ -13,6 +13,23 @@ def literal_type_from_choices(choices: Iterable[T]) -> Type[T]: used in the rare case that choices are generated dynamically. (for example, the keys of a dictionary) - Using the returned type as an annotation currently breaks for mypy, but is - understood by Pyright.""" + .. warning:: + Use of this helper is discouraged because it is compatible with `pyright` and + `pylance`, but not with `mypy`. For `pyright` support, you may need to enable + postponed evaluation of annotations (`from __future__ import annotations`). + + At the cost of verbosity, using `typing.Literal[]` directly is better supported + by external tools. + + Alternatively, we can work around this limitation with an `if TYPE_CHECKING` + guard: + ```python + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + SelectableConfig = str # For mypy. + else: + SelectableConfig = literal_type_from_choices(...) + ``` + """ return Literal.__getitem__(tuple(choices)) # type: ignore From b6c0cd6eb62850fb98121b184972b730a586df89 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 28 Nov 2022 13:08:37 -0800 Subject: [PATCH 241/491] Docs fixes --- .../03_config_systems/01_base_configs.rst | 120 ++++++++---------- .../source/examples/04_additional/06_conf.rst | 30 ----- examples/04_additional/06_conf.py | 30 ----- tyro/extras/_base_configs.py | 27 ++-- tyro/extras/_choices_type.py | 38 +++--- 5 files changed, 87 insertions(+), 158 deletions(-) diff --git a/docs/source/examples/03_config_systems/01_base_configs.rst b/docs/source/examples/03_config_systems/01_base_configs.rst index 0ae974006..968222cba 100644 --- a/docs/source/examples/03_config_systems/01_base_configs.rst +++ b/docs/source/examples/03_config_systems/01_base_configs.rst @@ -24,6 +24,7 @@ Python is convenient for autocompletion and type checking). from typing import Callable, Literal, Tuple, Union from torch import nn + from typing_extensions import Annotated import tyro @@ -34,18 +35,13 @@ Python is convenient for autocompletion and type checking). betas: Tuple[float, float] = (0.9, 0.999) - @dataclass(frozen=True) - class SgdOptimizer: - learning_rate: float = 3e-4 - - @dataclass(frozen=True) class ExperimentConfig: # Dataset to run experiment on. dataset: Literal["mnist", "imagenet-50"] # Optimizer parameters. - optimizer: Union[AdamOptimizer, SgdOptimizer] + optimizer: AdamOptimizer # Model size. num_layers: int @@ -68,63 +64,51 @@ Python is convenient for autocompletion and type checking). # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. - descriptions = {} - base_configs = {} - - descriptions["small"] = "Train a smaller model." - base_configs["small"] = ExperimentConfig( - dataset="mnist", - optimizer=SgdOptimizer(), - batch_size=2048, - num_layers=4, - units=64, - train_steps=30_000, - # The tyro.MISSING sentinel allows us to specify that the seed should have no - # default, and needs to be populated from the CLI. - seed=tyro.MISSING, - activation=nn.ReLU, - ) - - - descriptions["big"] = "Train a bigger model." - base_configs["big"] = ExperimentConfig( - dataset="imagenet-50", - optimizer=AdamOptimizer(), - batch_size=32, - num_layers=8, - units=256, - train_steps=100_000, - seed=tyro.MISSING, - activation=nn.GELU, - ) + SmallConfig = Annotated[ + ExperimentConfig, + tyro.conf.subcommand( + name="small", + default=ExperimentConfig( + dataset="mnist", + optimizer=AdamOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + seed=0, + activation=nn.ReLU, + ), + description="Train a smaller model.", + ), + ] + BigConfig = Annotated[ + ExperimentConfig, + tyro.conf.subcommand( + name="big", + default=ExperimentConfig( + dataset="imagenet-50", + optimizer=AdamOptimizer(), + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=0, + activation=nn.GELU, + ), + description="Train a bigger model.", + ), + ] + + + def main( + config: Union[SmallConfig, BigConfig], + restore_checkpoint: bool = False, + ) -> None: + print(config) if __name__ == "__main__": - config = tyro.cli( - tyro.extras.subcommand_type_from_defaults(base_configs, descriptions), - ) - # ^Note that this is equivalent to: - # - # config = tyro.cli( - # Union[ - # Annotated[ - # ExperimentConfig, - # tyro.conf.subcommand( - # "small", - # default=base_configs["small"], - # description=descriptions["small"], - # ), - # ], - # Annotated[ - # ExperimentConfig, - # tyro.conf.subcommand( - # "big", - # default=base_configs["big"], - # description=descriptions["big"], - # ), - # ], - # ] - # ) + config = tyro.cli(main) print(config) ------------ @@ -139,30 +123,30 @@ Python is convenient for autocompletion and type checking). .. raw:: html - python 03_config_systems/01_base_configs.py small --help + python 03_config_systems/01_base_configs.py config:small --help -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py small --help +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py config:small --help ------------ .. raw:: html - python 03_config_systems/01_base_configs.py small --seed 94720 + python 03_config_systems/01_base_configs.py config:small --config.seed 94720 -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py small --seed 94720 +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py config:small --config.seed 94720 ------------ .. raw:: html - python 03_config_systems/01_base_configs.py big --help + python 03_config_systems/01_base_configs.py config:big --help -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py big --help +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py config:big --help ------------ .. raw:: html - python 03_config_systems/01_base_configs.py big --seed 94720 + python 03_config_systems/01_base_configs.py config:big --config.seed 94720 -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py big --seed 94720 +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py config:big --config.seed 94720 diff --git a/docs/source/examples/04_additional/06_conf.rst b/docs/source/examples/04_additional/06_conf.rst index 83c36839c..309b74e86 100644 --- a/docs/source/examples/04_additional/06_conf.rst +++ b/docs/source/examples/04_additional/06_conf.rst @@ -24,21 +24,6 @@ Features here are supported, but generally unnecessary and should be used sparin import tyro - @dataclasses.dataclass(frozen=True) - class CheckoutArgs: - """Checkout a branch.""" - - branch: str - - - @dataclasses.dataclass(frozen=True) - class CommitArgs: - """Commit changes.""" - - message: str - all: bool = False - - @dataclasses.dataclass class Args: # A numeric field parsed as a positional argument. @@ -60,21 +45,6 @@ Features here are supported, but generally unnecessary and should be used sparin ), ] = "Hello" - # A union over nested structures, but without subcommand generation. When a default - # is provided, the type is simply fixed to that default. - union_without_subcommand: tyro.conf.AvoidSubcommands[ - Union[CheckoutArgs, CommitArgs] - ] = CheckoutArgs("main") - - # `tyro.conf.subcommand()` can be used to configure subcommands in a Union. Here, - # we make the subcommand names more succinct. - renamed_subcommand: Union[ - Annotated[ - CheckoutArgs, tyro.conf.subcommand(name="checkout", prefix_name=False) - ], - Annotated[CommitArgs, tyro.conf.subcommand(name="commit", prefix_name=False)], - ] = CheckoutArgs("main") - if __name__ == "__main__": print(tyro.cli(Args)) diff --git a/examples/04_additional/06_conf.py b/examples/04_additional/06_conf.py index 5d00945e6..51cd7f83f 100644 --- a/examples/04_additional/06_conf.py +++ b/examples/04_additional/06_conf.py @@ -18,21 +18,6 @@ import tyro -@dataclasses.dataclass(frozen=True) -class CheckoutArgs: - """Checkout a branch.""" - - branch: str - - -@dataclasses.dataclass(frozen=True) -class CommitArgs: - """Commit changes.""" - - message: str - all: bool = False - - @dataclasses.dataclass class Args: # A numeric field parsed as a positional argument. @@ -54,21 +39,6 @@ class Args: ), ] = "Hello" - # A union over nested structures, but without subcommand generation. When a default - # is provided, the type is simply fixed to that default. - union_without_subcommand: tyro.conf.AvoidSubcommands[ - Union[CheckoutArgs, CommitArgs] - ] = CheckoutArgs("main") - - # `tyro.conf.subcommand()` can be used to configure subcommands in a Union. Here, - # we make the subcommand names more succinct. - renamed_subcommand: Union[ - Annotated[ - CheckoutArgs, tyro.conf.subcommand(name="checkout", prefix_name=False) - ], - Annotated[CommitArgs, tyro.conf.subcommand(name="commit", prefix_name=False)], - ] = CheckoutArgs("main") - if __name__ == "__main__": print(tyro.cli(Args)) diff --git a/tyro/extras/_base_configs.py b/tyro/extras/_base_configs.py index e755b1012..cb6932e09 100644 --- a/tyro/extras/_base_configs.py +++ b/tyro/extras/_base_configs.py @@ -16,23 +16,26 @@ def subcommand_type_from_defaults( """Construct a Union type for defining subcommands that choose between defaults. .. warning:: - Use of this helper is discouraged because it is compatible with `pyright` and - `pylance`, but not with `mypy`. For `pyright` support, you may need to enable - postponed evaluation of annotations (`from __future__ import annotations`). + + Use of this helper is discouraged because it is compatible with ``pyright`` and + ``pylance``, but not with ``mypy``. For ``pyright`` support, you may need to enable + postponed evaluation of annotations (``from __future__ import annotations``). At the cost of verbosity, using :func:`tyro.conf.subcommand()` directly is better supported by external tools. - Alternatively, we can work around this limitation with an `if TYPE_CHECKING` + Alternatively, we can work around this limitation with an ``if TYPE_CHECKING`` guard: - ```python - from typing import TYPE_CHECKING - - if TYPE_CHECKING: - SelectableConfig = Config # For mypy. - else: - SelectableConfig = subcommand_type_from_defaults(...) - ``` + + .. code-block:: python + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + SelectableConfig = Config # For mypy. + else: + SelectableConfig = subcommand_type_from_defaults(...) + This can most commonly be used to create a "base configuration" pattern: https://brentyi.github.io/tyro/examples/10_base_configs/ diff --git a/tyro/extras/_choices_type.py b/tyro/extras/_choices_type.py index bb5335ff2..c787f85e3 100644 --- a/tyro/extras/_choices_type.py +++ b/tyro/extras/_choices_type.py @@ -7,29 +7,31 @@ def literal_type_from_choices(choices: Iterable[T]) -> Type[T]: - """Generate a typing.Literal[] type that constrains values to a set of choices. - - Using Literal[...] directly should generally be preferred, but this helper can be - used in the rare case that choices are generated dynamically. (for example, the keys - of a dictionary) + """Generate a `typing.Literal[]` type that constrains values to a set of choices. .. warning:: - Use of this helper is discouraged because it is compatible with `pyright` and - `pylance`, but not with `mypy`. For `pyright` support, you may need to enable - postponed evaluation of annotations (`from __future__ import annotations`). - At the cost of verbosity, using `typing.Literal[]` directly is better supported + Use of this helper is discouraged because it is compatible with ``pyright`` and + ``pylance``, but not with ``mypy``. For ``pyright`` support, you may need to enable + postponed evaluation of annotations (``from __future__ import annotations``). + + At the cost of verbosity, using ``typing.Literal[]`` directly is better supported by external tools. - Alternatively, we can work around this limitation with an `if TYPE_CHECKING` + Alternatively, we can work around this limitation with an ``if TYPE_CHECKING`` guard: - ```python - from typing import TYPE_CHECKING - - if TYPE_CHECKING: - SelectableConfig = str # For mypy. - else: - SelectableConfig = literal_type_from_choices(...) - ``` + + .. code-block:: python + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + Color = str # For mypy. + else: + Color = literal_type_from_choices(["red", "green", "blue"]) + + Using `Literal[...]` directly should generally be preferred, but this helper can be + used in the rare case that choices are generated dynamically. (for example, the keys + of a dictionary) """ return Literal.__getitem__(tuple(choices)) # type: ignore From 7c3384e806a1250bc8f5be889bba52acffb9d570 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 28 Nov 2022 23:16:33 -0800 Subject: [PATCH 242/491] Fix + test for missing optional packages --- tests/test_missing_optional_packages.py | 25 +++++++++++++++++++++++++ tyro/_fields.py | 5 +++-- 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 tests/test_missing_optional_packages.py diff --git a/tests/test_missing_optional_packages.py b/tests/test_missing_optional_packages.py new file mode 100644 index 000000000..a093baba0 --- /dev/null +++ b/tests/test_missing_optional_packages.py @@ -0,0 +1,25 @@ +import builtins + +import pytest + + +# https://stackoverflow.com/questions/60227582/making-a-python-test-think-an-installed-package-is-not-available +@pytest.fixture +def hide_optional_packages(monkeypatch): + import_orig = builtins.__import__ + + def mocked_import(name, *args, **kwargs): + if name in ("attr", "pydantic", "flax", "omegaconf"): + raise ImportError() + return import_orig(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", mocked_import) + + +def test_missing_optional_packages(hide_optional_packages): + def main(x: int) -> int: + return x + + import tyro + + assert tyro.cli(main, args=["--x", "5"]) == 5 diff --git a/tyro/_fields.py b/tyro/_fields.py index 23623cacf..d573f08c7 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -365,10 +365,11 @@ def _field_list_from_dataclass( # Support attrs and pydantic if they're installed. + try: import pydantic except ImportError: - pass + pydantic = None # type: ignore def _is_pydantic(cls: Type[Any]) -> bool: @@ -382,7 +383,7 @@ def _field_list_from_pydantic( # Handle pydantic models. field_list = [] - for pd_field in cls.__fields__.values(): + for pd_field in cls.__fields__.values(): # type: ignore field_list.append( FieldDefinition.make( name=pd_field.name, From 3425f0a09a864bb1c964f1361e67b59450e0fe3a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 7 Dec 2022 17:09:08 -0800 Subject: [PATCH 243/491] Add default helptext for positional args --- tyro/_arguments.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tyro/_arguments.py b/tyro/_arguments.py index bc15229af..145f9c111 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -312,6 +312,9 @@ def _rule_generate_helptext( primary_help = arg.field.helptext + if primary_help is None and _markers.Positional in arg.field.markers: + primary_help = _strings.make_field_name([arg.name_prefix, arg.field.name]) + if primary_help is not None and primary_help != "": # Note that the percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string From e604f711cfb7af9c11b356dd40624bf36380aed5 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 8 Dec 2022 12:45:55 -0800 Subject: [PATCH 244/491] Support + tests for functools.partial (cc #23) --- examples/04_additional/06_conf.py | 1 - tests/test_partial.py | 44 +++++++++++++++++++++++++++++++ tyro/_docstrings.py | 5 ++++ tyro/_fields.py | 10 +++++-- 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 tests/test_partial.py diff --git a/examples/04_additional/06_conf.py b/examples/04_additional/06_conf.py index 51cd7f83f..479a888a9 100644 --- a/examples/04_additional/06_conf.py +++ b/examples/04_additional/06_conf.py @@ -11,7 +11,6 @@ """ import dataclasses -from typing import Union from typing_extensions import Annotated diff --git a/tests/test_partial.py b/tests/test_partial.py new file mode 100644 index 000000000..99aad8609 --- /dev/null +++ b/tests/test_partial.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import functools + +from helptext_utils import get_helptext + +import tyro + + +def test_partial_func() -> None: + def main(a: int, b: str) -> str: + return b * a + + assert tyro.cli(functools.partial(main, a=3), args=["--b", "hi"]) == "hihihi" + + +def test_partial_class() -> None: + class Main: + def __init__(self, a: int, b: str) -> None: + self.inner = b * a + + assert tyro.cli(functools.partial(Main, a=3), args=["--b", "hi"]).inner == "hihihi" + + +def test_partial_helptext_func() -> None: + def main(a: int, b: str) -> str: + """Hello!""" + return b * a + + helptext = get_helptext(functools.partial(main, b=3)) + assert "partial" not in helptext + assert "Hello!" in helptext + + +def test_partial_helptext_class() -> None: + class Main: + """Hello!""" + + def __init__(self, a: int, b: str) -> None: + self.inner = b * a + + helptext = get_helptext(functools.partial(Main, b=3)) + assert "partial" not in helptext + assert "Hello!" in helptext diff --git a/tyro/_docstrings.py b/tyro/_docstrings.py index fe96d8dda..a01007aff 100644 --- a/tyro/_docstrings.py +++ b/tyro/_docstrings.py @@ -292,6 +292,11 @@ def get_callable_description(f: Callable) -> str: if f in _callable_description_blocklist: return "" + # Return original docstring when used with functools.partial, not + # functools.partial's docstinrg. + if isinstance(f, functools.partial): + f = f.func + # Note inspect.getdoc() causes some corner cases with TypedDicts. docstring = f.__doc__ if ( diff --git a/tyro/_fields.py b/tyro/_fields.py index d573f08c7..b793f1206 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -1,12 +1,12 @@ """Abstractions for pulling out 'field' definitions, which specify inputs, types, and defaults, from general callables.""" - from __future__ import annotations import collections import collections.abc import dataclasses import enum +import functools import inspect import itertools import typing @@ -624,6 +624,12 @@ def _field_list_from_params( # This will throw a type error for torch.device, typing.Dict, etc. try: + # Special-case for functools.partial(). + if isinstance(f, functools.partial): + f = f.func + if isinstance(f, type): + f = f.__init__ # type: ignore + hints = get_type_hints(f, include_extras=True) except TypeError: return UnsupportedNestedTypeMessage(f"Could not get hints for {f}!") @@ -653,7 +659,7 @@ def _field_list_from_params( field_list.append( FieldDefinition.make( name=param.name, - # Note that param.annotation does not resolve forward references. + # Note that param.annotation doesn't resolve forward references. typ=hints[param.name], default=default, helptext=helptext, From f79a2f8b891b23a702aa78237e713cff89755489 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 8 Dec 2022 13:45:46 -0800 Subject: [PATCH 245/491] Test `functools.wraps`, fix arg helptext for `functools.partial` --- tests/test_functools.py | 81 +++++++++++++++++++++++++++++++++++++++++ tyro/_fields.py | 13 ++++--- 2 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 tests/test_functools.py diff --git a/tests/test_functools.py b/tests/test_functools.py new file mode 100644 index 000000000..863fdc378 --- /dev/null +++ b/tests/test_functools.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import functools + +import pytest +from helptext_utils import get_helptext + +import tyro + + +def test_partial_func() -> None: + def main(a: int, b: str) -> str: + return b * a + + assert tyro.cli(functools.partial(main, a=3), args=["--b", "hi"]) == "hihihi" + + +def test_partial_class() -> None: + class Main: + def __init__(self, a: int, b: str) -> None: + self.inner = b * a + + assert tyro.cli(functools.partial(Main, a=3), args=["--b", "hi"]).inner == "hihihi" + + +def test_partial_helptext_func() -> None: + def main(a: int, b: str) -> str: + """Hello!""" + return b * a + + helptext = get_helptext(functools.partial(main, b="hello world")) + assert "partial" not in helptext + assert "Hello!" in helptext + assert "hello world" in helptext + + +def test_partial_helptext_class() -> None: + class Main: + """Hello!""" + + def __init__(self, a: int, b: str) -> None: + self.inner = b * a + + helptext = get_helptext(functools.partial(Main, b=3)) + assert "partial" not in helptext + assert "Hello!" in helptext + + +def test_wraps_func() -> None: + def main(a: int, b: str) -> str: + return b * a + + @functools.wraps(main) + def wrapper(*args, **kwargs) -> int: + return kwargs["a"] + + assert tyro.cli(wrapper, args=["--a", "3", "--b", "hi"]) == 3 + with pytest.raises(SystemExit): + tyro.cli(wrapper, args=["--a", "3"]) + + +def test_wraps_partial_func_helptext() -> None: + def main(a: int, b: str) -> str: + """Hello! + + Args: + a: Argument. + b: Argument. + """ + return b * a + + @functools.wraps(main) + def wrapper(*args, **kwargs) -> int: + return kwargs["a"] + + assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == 3 + + helptext = get_helptext(functools.partial(wrapper, b=3)) + assert "wraps" not in helptext + assert "Hello!" in helptext + assert "Argument." in helptext diff --git a/tyro/_fields.py b/tyro/_fields.py index b793f1206..63878d725 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -614,6 +614,13 @@ def _field_list_from_params( cls: Optional[Type[Any]], params: List[inspect.Parameter], ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: + # Special-case for functools.partial(). + if isinstance(f, functools.partial): + f = f.func + if isinstance(f, type): + cls = f + f = f.__init__ # type: ignore + # Get type annotations, docstrings. docstring = inspect.getdoc(f) docstring_from_arg_name = {} @@ -624,12 +631,6 @@ def _field_list_from_params( # This will throw a type error for torch.device, typing.Dict, etc. try: - # Special-case for functools.partial(). - if isinstance(f, functools.partial): - f = f.func - if isinstance(f, type): - f = f.__init__ # type: ignore - hints = get_type_hints(f, include_extras=True) except TypeError: return UnsupportedNestedTypeMessage(f"Could not get hints for {f}!") From 7d329cf152cd7d8fb31880b557c6216006a4fa91 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 8 Dec 2022 18:19:47 -0800 Subject: [PATCH 246/491] Support multiple functools.wraps / functools.partial calls --- tests/test_functools.py | 52 +++++++++++++++++++++++++++++++++++++++++ tyro/_fields.py | 21 ++++++++++++----- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/tests/test_functools.py b/tests/test_functools.py index 863fdc378..684534096 100644 --- a/tests/test_functools.py +++ b/tests/test_functools.py @@ -1,5 +1,6 @@ from __future__ import annotations +import dataclasses import functools import pytest @@ -79,3 +80,54 @@ def wrapper(*args, **kwargs) -> int: assert "wraps" not in helptext assert "Hello!" in helptext assert "Argument." in helptext + + +def test_wraps_partial_class_helptext() -> None: + class Main: + """Hello!""" + + def __init__(self, a: int, b: str) -> None: + self.inner = b * a + + @functools.wraps(Main) + def wrapper(*args, **kwargs) -> int: + return kwargs["a"] + + assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == 3 + + helptext = get_helptext(functools.partial(wrapper, b=3)) + assert "wraps" not in helptext + assert "Hello!" in helptext + + +@dataclasses.dataclass +class WrappedDataclass: + """Hello!""" + + a: int + b: str + """Second field.""" + + +def test_wraps_partial_dataclass() -> None: + @functools.wraps(WrappedDataclass) + def wrapper(*args, **kwargs) -> str: + return kwargs["a"] * kwargs["b"] + + assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == "hihihi" + assert ( + tyro.cli( + functools.partial(functools.partial(wrapper, a=3), b="hello"), + args=["--b", "hi"], + ) + == "hihihi" + ) + assert ( + tyro.cli(functools.partial(functools.partial(wrapper, a=3), b="hello"), args=[]) + == "hellohellohello" + ) + + helptext = get_helptext(functools.partial(wrapper, b=3)) + assert "wraps" not in helptext + assert "Hello!" in helptext + assert "Second field." in helptext diff --git a/tyro/_fields.py b/tyro/_fields.py index 63878d725..21cded939 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -614,12 +614,21 @@ def _field_list_from_params( cls: Optional[Type[Any]], params: List[inspect.Parameter], ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - # Special-case for functools.partial(). - if isinstance(f, functools.partial): - f = f.func - if isinstance(f, type): - cls = f - f = f.__init__ # type: ignore + # Unwrap functools.wraps and functools.partial. + done = False + while not done: + done = True + if hasattr(f, "__wrapped__"): + f = f.__wrapped__ + done = False + if isinstance(f, functools.partial): + f = f.func + done = False + + # Sometime functools.* is applied to a class. + if isinstance(f, type): + cls = f + f = f.__init__ # type: ignore # Get type annotations, docstrings. docstring = inspect.getdoc(f) From e88e690e14ffe87e13419fa2b6427bc77c4ae336 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 9 Dec 2022 01:59:04 -0800 Subject: [PATCH 247/491] Minor fix for 3.10-style annotations --- tests/test_new_style_annotations_only_py310.py | 5 +++++ tyro/_resolver.py | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_new_style_annotations_only_py310.py b/tests/test_new_style_annotations_only_py310.py index 8b99a75e4..94d0fc6f2 100644 --- a/tests/test_new_style_annotations_only_py310.py +++ b/tests/test_new_style_annotations_only_py310.py @@ -5,6 +5,11 @@ import tyro +def test_union_direct(): + assert tyro.cli(int | str, args=["5"]) == 5 + assert tyro.cli(int | str, args=["five"]) == "five" + + def test_union_basic(): def main(x: int | str) -> int | str: return x diff --git a/tyro/_resolver.py b/tyro/_resolver.py index b93c64fed..b288e779f 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -50,9 +50,15 @@ def resolve_generic_types( origin_cls = get_origin(cls) type_from_typevar = {} - if origin_cls is not None and hasattr(origin_cls, "__parameters__"): + if ( + # Apply some heuristics for generic types. Should revisit this. + origin_cls is not None + and hasattr(origin_cls, "__parameters__") + and hasattr(origin_cls.__parameters__, "__len__") + ): typevars = origin_cls.__parameters__ typevar_values = get_args(cls) + print(typevars, typevar_values, origin_cls) assert len(typevars) == len(typevar_values) cls = origin_cls type_from_typevar.update(dict(zip(typevars, typevar_values))) From 32022add15f9136bd306475d88d889136f233684 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 12 Dec 2022 12:52:20 -0800 Subject: [PATCH 248/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 84f261241..4e8d95b34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.35" +version = "0.3.36" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] From 10b2e3804b716d2e3264bdf7db93acaf0051a37a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 22 Dec 2022 17:02:47 -0800 Subject: [PATCH 249/491] Docs updates --- README.md | 130 ++++++++---------- .../examples/01_basics/03_containers.rst | 9 +- .../source/examples/01_basics/06_literals.rst | 7 +- .../source/examples/04_additional/06_conf.rst | 1 - docs/source/index.md | 8 +- docs/source/your_first_cli.md | 58 +++++--- examples/01_basics/03_containers.py | 9 +- examples/01_basics/06_literals.py | 7 +- tyro/_instantiators.py | 3 +- 9 files changed, 115 insertions(+), 117 deletions(-) diff --git a/README.md b/README.md index b6eec5f7e..5962a2d5b 100644 --- a/README.md +++ b/README.md @@ -37,65 +37,78 @@
-tyro is a library for building CLI interfaces and -configuration objects with type-annotated Python. +tyro is a tool for building command-line +interfaces and configuration objects in Python. -Our core interface consists of one function, `tyro.cli()`, that generates -argument parsers from Python callables and types. +Our core interface, `tyro.cli()`, generates command-line interfaces from +type-annotated callables. -### A minimal example +--- -As a replacement for `argparse`: +### Brief walkthrough - - - - - - - - - -
with argparsewith tyro
+To summarize how `tyro.cli()` can be used, let's consider a script based on +`argparse`. We define two inputs and print the sum: ```python """Sum two numbers from argparse.""" - import argparse + parser = argparse.ArgumentParser() -parser.add_argument( - "--a", - type=int, - required=True, -) -parser.add_argument( - "--b", - type=int, - default=3, -) +parser.add_argument("--a", type=int, required=True) +parser.add_argument("--b", type=int, default=3) args = parser.parse_args() -print(args.a + args.b) +total = args.a + args.b + +print(total) ``` - +This pattern is dramatically cleaner than manually parsing `sys.argv`, but has +several issues: it lacks type checking and IDE support (consider: jumping to +definitions, finding references, docstrings, refactoring and renaming tools), +requires a significant amount of parsing-specific boilerplate, and becomes +difficult to manage for larger projects. + +The basic goal of `tyro.cli()` is to provide a wrapper for `argparse` that +solves these issues. + +**(1) Command-line interfaces from functions.** + +We can write the same script as above using `tyro.cli()`: ```python -"""Sum two numbers by calling a -function with tyro.""" +"""Sum two numbers by calling a function with tyro.""" +import tyro + +def add(a: int, b: int = 3) -> int: + return a + b + +# Populate the inputs of add(), call it, then return the output. +total = tyro.cli(add) + +print(total) +``` + +Or, more succinctly: +```python +"""Sum two numbers by calling a function with tyro.""" import tyro -def main(a: int, b: int = 3) -> None: +def add(a: int, b: int = 3) -> None: print(a + b) -tyro.cli(main) +tyro.cli(add) # Returns `None`. ``` ---- +**(2) Command-line interfaces from config objects.** -```python -"""Sum two numbers by instantiating -a dataclass with tyro.""" +A class in Python can be treated as a function that returns an instance. This +makes it easy to populate explicit configuration structures: +```python +"""Sum two numbers by instantiating a dataclass with tyro.""" from dataclasses import dataclass import tyro @@ -109,47 +122,16 @@ args = tyro.cli(Args) print(args.a + args.b) ``` -
- -For more examples, see our [documentation](https://brentyi.github.io/tyro). - -### Why `tyro`? - -1. **Strong typing.** - - Unlike tools dependent on dictionaries, YAML, or dynamic namespaces, - arguments populated by `tyro` benefit from IDE and language server-supported - operations — think tab completion, rename, jump-to-def, docstrings on hover — - as well as static checking tools like `pyright` and `mypy`. - -2. **Minimal overhead.** - - Standard Python type annotations, docstrings, and default values are parsed - to automatically generate command-line interfaces with informative helptext. - - `tyro` works seamlessly with tools you already use: examples are included for - [`dataclasses`](https://docs.python.org/3/library/dataclasses.html), - [`attrs`](https://www.attrs.org/), - [`pydantic`](https://pydantic-docs.helpmanual.io/), - [`flax.linen`](https://flax.readthedocs.io/en/latest/api_reference/flax.linen.html), - and more. - - Hate `tyro`? Just remove one line of code, and you're left with beautiful, - type-annotated, and documented vanilla Python that can be used with a range - of other configuration libraries. - -3. **Modularity.** - - `tyro` supports hierarchical configuration structures, which make it easy to - distribute definitions, defaults, and documentation of configurable fields - across modules or source files. +Unlike directly using `argparse`, both the function-based and dataclass-based +approaches are compatible with static analysis; tab completion and type checking +will work out-of-the-box. -4. **Tab completion.** +**(3) Additional features.** - By extending [shtab](https://github.com/iterative/shtab), `tyro` - automatically generates tab completion scripts for bash, zsh, and tcsh. +These examples only scratch the surface of what's possible. `tyro` aims to +support all reasonable type annotations, which can help us define things like +hierachical structures, enums, unions, variable-length inputs, and subcommands. +See [documentation](https://brentyi.github.io/tyro) for examples. ### In the wild diff --git a/docs/source/examples/01_basics/03_containers.rst b/docs/source/examples/01_basics/03_containers.rst index 8db582107..7baeaa942 100644 --- a/docs/source/examples/01_basics/03_containers.rst +++ b/docs/source/examples/01_basics/03_containers.rst @@ -6,8 +6,8 @@ Containers Arguments of both fixed and variable lengths can be annotated with standard Python -container types; such as ``typing.List[T]`` or ``typing.Tuple[T1, T2]``. In Python 3.9, -``list[T]`` and ``tuple[T]`` are also directly supported. +container types: ``typing.List[T]``\ , ``typing.Tuple[T1, T2]``\ , etc. In Python >=3.9, +``list[T]`` and ``tuple[T]`` are also supported. @@ -23,11 +23,6 @@ container types; such as ``typing.List[T]`` or ``typing.Tuple[T1, T2]``. In Pyth import tyro - class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - @dataclasses.dataclass(frozen=True) class TrainConfig: # Example of a variable-length tuple. `typing.List`, `typing.Sequence`, diff --git a/docs/source/examples/01_basics/06_literals.rst b/docs/source/examples/01_basics/06_literals.rst index 22699fc1e..eaa8beedf 100644 --- a/docs/source/examples/01_basics/06_literals.rst +++ b/docs/source/examples/01_basics/06_literals.rst @@ -29,8 +29,11 @@ Choices @dataclasses.dataclass(frozen=True) class Args: # We can use Literal[] to restrict the set of allowable inputs, for example, over - # enums. - restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED + # a set of strings. + strings: Literal["red", "green"] = "red" + + # Enums also work. + enums: Literal[Color.RED, Color.GREEN] = Color.RED # Or mix them with other types! mixed: Literal[Color.RED, Color.GREEN, "blue"] = "blue" diff --git a/docs/source/examples/04_additional/06_conf.rst b/docs/source/examples/04_additional/06_conf.rst index 309b74e86..192c03991 100644 --- a/docs/source/examples/04_additional/06_conf.rst +++ b/docs/source/examples/04_additional/06_conf.rst @@ -17,7 +17,6 @@ Features here are supported, but generally unnecessary and should be used sparin import dataclasses - from typing import Union from typing_extensions import Annotated diff --git a/docs/source/index.md b/docs/source/index.md index e3261aee0..6898be2cc 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -2,11 +2,11 @@ |build| |nbsp| |mypy| |nbsp| |lint| |nbsp| |coverage| |nbsp| |versions| -:code:`tyro` is a library for building CLI interfaces and configuration objects -with type-annotated Python. +:code:`tyro` is a tool for building scalable command-line interfaces and +configuration objects in Python. -Our core interface consists of one function, :func:`tyro.cli()`, that generates -argument parsers from Python callables and types. +Our core interface, :func:`tyro.cli()`, generates command-line interfaces from +type-annotated callables. To get started, we recommend browsing the examples to the left. diff --git a/docs/source/your_first_cli.md b/docs/source/your_first_cli.md index dc26f658b..241f79810 100644 --- a/docs/source/your_first_cli.md +++ b/docs/source/your_first_cli.md @@ -5,7 +5,6 @@ command-line interface: ```python """Sum two numbers from argparse.""" - import argparse parser = argparse.ArgumentParser() @@ -13,37 +12,56 @@ parser.add_argument("--a", type=int, required=True) parser.add_argument("--b", type=int, default=3) args = parser.parse_args() -print(args.a + args.b) +total = args.a + args.b + +print(total) ``` -This is dramatically cleaner than manually parsing `sys.argv`, but has several -issues: it lacks type checking and IDE support (consider: jumping to +This pattern is dramatically cleaner than manually parsing `sys.argv`, but has +several issues: it lacks type checking and IDE support (consider: jumping to definitions, finding references, docstrings, refactoring and renaming tools), -requires a significant amount of boilerplate, and generally becomes difficult to -manage as interfaces grow. +requires a significant amount of parsing-specific boilerplate, and becomes +difficult to manage for larger projects. + +The basic goal of :func:`tyro.cli()` is to provide a wrapper for `argparse` that +solves these issues. -The basic feature of :func:`tyro.cli()` is to provide a wrapper for `argparse` -that solves these issues. +**(1) Command-line interfaces from functions.** -We can specify the same logic with a function signature: +We can write the same script as above using `tyro.cli()`: ```python """Sum two numbers by calling a function with tyro.""" +import tyro + +def add(a: int, b: int = 3) -> int: + return a + b +# Populate the inputs of add(), call it, then return the output. +total = tyro.cli(add) + +print(total) +``` + +Or, more succinctly: + +```python +"""Sum two numbers by calling a function with tyro.""" import tyro -def main(a: int, b: int = 3) -> None: +def add(a: int, b: int = 3) -> None: print(a + b) -tyro.cli(main) +tyro.cli(add) # Returns `None`. ``` -Particularly when interfaces grow in complexity or require hierarchical -structures, dataclasses can also be helpful: +**(2) Command-line interfaces from config objects.** + +A class in Python can be treated as a function that returns an instance. This +makes it easy to populate explicit configuration structures: ```python """Sum two numbers by instantiating a dataclass with tyro.""" - from dataclasses import dataclass import tyro @@ -57,7 +75,11 @@ args = tyro.cli(Args) print(args.a + args.b) ``` -And that's it! By incorporating more advanced type annotations from the standard -library, we can specify a broad range of more advanced behaviors: -variable-length inputs, unions over types, subcommands, and more. Our examples -walk through a selection of these features. +Unlike directly using `argparse`, both the function-based and dataclass-based +approaches are compatible with static analysis; tab completion and type checking +will work out-of-the-box. + +And that's it! By incorporating more standard type annotations, we can specify a +broad range of more advanced behaviors: nested structures, variable-length +inputs, unions over types, subcommands, and more. Our examples walk through a +selection of these features. diff --git a/examples/01_basics/03_containers.py b/examples/01_basics/03_containers.py index ba85c5f34..547164f6e 100644 --- a/examples/01_basics/03_containers.py +++ b/examples/01_basics/03_containers.py @@ -1,8 +1,8 @@ """Containers Arguments of both fixed and variable lengths can be annotated with standard Python -container types; such as `typing.List[T]` or `typing.Tuple[T1, T2]`. In Python 3.9, -`list[T]` and `tuple[T]` are also directly supported. +container types: `typing.List[T]`, `typing.Tuple[T1, T2]`, etc. In Python >=3.9, +`list[T]` and `tuple[T]` are also supported. Usage: `python ./03_containers.py --help` @@ -18,11 +18,6 @@ import tyro -class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - @dataclasses.dataclass(frozen=True) class TrainConfig: # Example of a variable-length tuple. `typing.List`, `typing.Sequence`, diff --git a/examples/01_basics/06_literals.py b/examples/01_basics/06_literals.py index 64a2f1958..22e38e987 100644 --- a/examples/01_basics/06_literals.py +++ b/examples/01_basics/06_literals.py @@ -22,8 +22,11 @@ class Color(enum.Enum): @dataclasses.dataclass(frozen=True) class Args: # We can use Literal[] to restrict the set of allowable inputs, for example, over - # enums. - restricted_enum: Literal[Color.RED, Color.GREEN] = Color.RED + # a set of strings. + strings: Literal["red", "green"] = "red" + + # Enums also work. + enums: Literal[Color.RED, Color.GREEN] = Color.RED # Or mix them with other types! mixed: Literal[Color.RED, Color.GREEN, "blue"] = "blue" diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 5e86837a5..8fcfaa204 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -152,8 +152,7 @@ def instantiator(strings: List[str]) -> None: if i == 0 and param.annotation not in (str, inspect.Parameter.empty): raise UnsupportedTypeAnnotationError( f"Expected {typ} to be an `(arg: str) -> T` type converter, but" - f" got {signature}. You may have a nested type in a container," - " which is unsupported." + f" got {signature}." ) if param.kind is inspect.Parameter.VAR_POSITIONAL: has_var_positional = True From 721b21713935f4cdb4b4c0a795b80c44dda2f6f2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 22 Dec 2022 17:02:55 -0800 Subject: [PATCH 250/491] Add functools.partial test for dataclasses --- examples/01_basics/03_containers.py | 1 - tests/test_partial.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/examples/01_basics/03_containers.py b/examples/01_basics/03_containers.py index 547164f6e..9c3a881de 100644 --- a/examples/01_basics/03_containers.py +++ b/examples/01_basics/03_containers.py @@ -11,7 +11,6 @@ """ import dataclasses -import enum import pathlib from typing import Tuple diff --git a/tests/test_partial.py b/tests/test_partial.py index 99aad8609..e1d63aed8 100644 --- a/tests/test_partial.py +++ b/tests/test_partial.py @@ -1,5 +1,6 @@ from __future__ import annotations +import dataclasses import functools from helptext_utils import get_helptext @@ -42,3 +43,19 @@ def __init__(self, a: int, b: str) -> None: helptext = get_helptext(functools.partial(Main, b=3)) assert "partial" not in helptext assert "Hello!" in helptext + + +def test_partial_helptext_dataclass() -> None: + @dataclasses.dataclass + class Config: + a: int + b: int + """Hello!""" + c: int + + ConfigWithDefaults = functools.partial(functools.partial(Config, a=3), c=5) + assert tyro.cli(ConfigWithDefaults, args=["--b", "4"]) == Config(3, 4, 5) + + helptext = get_helptext(ConfigWithDefaults) + assert "partial" not in helptext + assert "Hello!" in helptext From 0c82e42afc5bdceee34e0fe27b57c3ee1eecb355 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 25 Dec 2022 18:32:18 -0800 Subject: [PATCH 251/491] Add TypeForm placeholder, remove flax bug workaround --- .../examples/01_basics/03_containers.rst | 1 - .../source/examples/04_additional/07_flax.rst | 3 +- examples/04_additional/07_flax.py | 3 +- poetry.lock | 252 ++++++++++++------ tests/test_flax_ignore_py310.py | 1 - tyro/_arguments.py | 16 +- tyro/_cli.py | 29 +- tyro/_fields.py | 46 ++-- tyro/_instantiators.py | 34 +-- tyro/_parsers.py | 3 +- tyro/_resolver.py | 29 +- tyro/_typing.py | 16 ++ tyro/extras/_base_configs.py | 19 +- tyro/extras/_choices_type.py | 20 +- 14 files changed, 287 insertions(+), 185 deletions(-) create mode 100644 tyro/_typing.py diff --git a/docs/source/examples/01_basics/03_containers.rst b/docs/source/examples/01_basics/03_containers.rst index 7baeaa942..f30129ba6 100644 --- a/docs/source/examples/01_basics/03_containers.rst +++ b/docs/source/examples/01_basics/03_containers.rst @@ -16,7 +16,6 @@ container types: ``typing.List[T]``\ , ``typing.Tuple[T1, T2]``\ , etc. In Pytho import dataclasses - import enum import pathlib from typing import Tuple diff --git a/docs/source/examples/04_additional/07_flax.rst b/docs/source/examples/04_additional/07_flax.rst index 2d47b727d..d9116016b 100644 --- a/docs/source/examples/04_additional/07_flax.rst +++ b/docs/source/examples/04_additional/07_flax.rst @@ -15,13 +15,12 @@ directly from ``tyro.cli``. from flax import linen as nn - from flax.linen import Module # https://github.com/google/flax/issues/2636 from jax import numpy as jnp import tyro - class Classifier(Module): + class Classifier(nn.Module): layers: int """Layers in our network.""" units: int = 32 diff --git a/examples/04_additional/07_flax.py b/examples/04_additional/07_flax.py index 42364965d..6707964ee 100644 --- a/examples/04_additional/07_flax.py +++ b/examples/04_additional/07_flax.py @@ -9,13 +9,12 @@ """ from flax import linen as nn -from flax.linen import Module # https://github.com/google/flax/issues/2636 from jax import numpy as jnp import tyro -class Classifier(Module): +class Classifier(nn.Module): layers: int """Layers in our network.""" units: int = 32 diff --git a/poetry.lock b/poetry.lock index 2563d063e..eb85007b9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -36,6 +36,14 @@ category = "main" optional = false python-versions = ">=3.6.0" +[[package]] +name = "cached-property" +version = "1.5.2" +description = "A decorator for caching properties in classes." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "chex" version = "0.1.5" @@ -95,7 +103,7 @@ python-versions = ">=3.6" [[package]] name = "dm-tree" -version = "0.1.7" +version = "0.1.8" description = "Tree is a library for working with nested data structures." category = "dev" optional = false @@ -109,9 +117,34 @@ category = "main" optional = false python-versions = ">=3.6,<4.0" +[[package]] +name = "etils" +version = "0.9.0" +description = "Collection of common python utils" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +all = ["etils[array-types]", "etils[eapp]", "etils[ecolab]", "etils[edc]", "etils[enp]", "etils[epath]", "etils[epy]", "etils[etqdm]", "etils[etree-dm]", "etils[etree-jax]", "etils[etree-tf]", "etils[etree]"] +array-types = ["etils[enp]"] +dev = ["chex", "pylint (>=2.6.0)", "pytest", "pytest-subtests", "pytest-xdist", "yapf"] +eapp = ["absl-py", "simple_parsing"] +ecolab = ["etils[enp]", "etils[epy]", "jupyter", "mediapy", "numpy"] +edc = ["etils[epy]", "typing_extensions"] +enp = ["etils[epy]", "numpy"] +epath = ["etils[epy]", "importlib_resources", "typing_extensions", "zipp"] +epy = ["typing_extensions"] +etqdm = ["absl-py", "etils[epy]", "tqdm"] +etree = ["etils[array_types]", "etils[enp]", "etils[epy]", "etils[etqdm]"] +etree-dm = ["dm-tree", "etils[etree]"] +etree-jax = ["etils[etree]", "jax[cpu]"] +etree-tf = ["etils[etree]", "tf-nightly"] +lazy-imports = ["etils[ecolab]"] + [[package]] name = "exceptiongroup" -version = "1.0.4" +version = "1.1.0" description = "Backport of PEP 654 (exception groups)" category = "dev" optional = false @@ -122,7 +155,7 @@ test = ["pytest (>=6)"] [[package]] name = "flax" -version = "0.6.2" +version = "0.6.3" description = "Flax: A neural network library for JAX designed for flexibility" category = "dev" optional = false @@ -134,6 +167,7 @@ matplotlib = "*" msgpack = "*" numpy = ">=1.12" optax = "*" +orbax = "*" PyYAML = ">=5.4.1" rich = ">=11.1" tensorstore = "*" @@ -174,7 +208,7 @@ python-versions = ">=3.6" [[package]] name = "importlib-metadata" -version = "5.0.0" +version = "5.2.0" description = "Read metadata from Python packages" category = "dev" optional = false @@ -185,10 +219,25 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] +[[package]] +name = "importlib-resources" +version = "5.10.1" +description = "Read resources from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + [[package]] name = "iniconfig" version = "1.1.1" @@ -368,7 +417,7 @@ wheel = "*" [[package]] name = "omegaconf" -version = "2.2.3" +version = "2.3.0" description = "A flexible configuration library" category = "dev" optional = false @@ -395,7 +444,7 @@ tests = ["pytest", "pytest-cov", "pytest-pep8"] [[package]] name = "optax" -version = "0.1.3" +version = "0.1.4" description = "A gradient processing and optimisation library in JAX." category = "dev" optional = false @@ -403,22 +452,43 @@ python-versions = ">=3.7" [package.dependencies] absl-py = ">=0.7.1" -chex = ">=0.0.4" +chex = ">=0.1.5" jax = ">=0.1.55" jaxlib = ">=0.1.37" numpy = ">=1.18.0" typing-extensions = ">=3.10.0" [[package]] -name = "packaging" -version = "21.3" -description = "Core utilities for Python packages" +name = "orbax" +version = "0.0.23" +description = "Orbax" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" +absl-py = "*" +cached_property = "*" +etils = "*" +flax = "*" +importlib_resources = "*" +jax = "*" +jaxlib = "*" +numpy = "*" +pytest = "*" +pyyaml = "*" +tensorstore = ">=0.1.20" + +[package.extras] +dev = ["pytest-xdist"] + +[[package]] +name = "packaging" +version = "22.0" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" [[package]] name = "Pillow" @@ -486,7 +556,7 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pyright" -version = "1.1.280" +version = "1.1.285" description = "Command line wrapper for pyright" category = "dev" optional = false @@ -584,7 +654,7 @@ numpy = ">=1.16.5" [[package]] name = "setuptools" -version = "65.6.0" +version = "65.6.3" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "dev" optional = false @@ -657,17 +727,17 @@ python-versions = ">=3.5" [[package]] name = "torch" -version = "1.13.0" +version = "1.13.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" category = "dev" optional = false python-versions = ">=3.7.0" [package.dependencies] -nvidia-cublas-cu11 = "11.10.3.66" -nvidia-cuda-nvrtc-cu11 = "11.7.99" -nvidia-cuda-runtime-cu11 = "11.7.99" -nvidia-cudnn-cu11 = "8.5.0.96" +nvidia-cublas-cu11 = {version = "11.10.3.66", markers = "platform_system == \"Linux\""} +nvidia-cuda-nvrtc-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\""} +nvidia-cuda-runtime-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\""} +nvidia-cudnn-cu11 = {version = "8.5.0.96", markers = "platform_system == \"Linux\""} typing-extensions = "*" [package.extras] @@ -702,7 +772,7 @@ test = ["pytest (>=3.0.0)"] [[package]] name = "zipp" -version = "3.10.0" +version = "3.11.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false @@ -733,6 +803,10 @@ attrs = [ {file = "backports.cached-property-1.0.2.tar.gz", hash = "sha256:9306f9eed6ec55fd156ace6bc1094e2c86fae5fb2bf07b6a9c00745c656e75dd"}, {file = "backports.cached_property-1.0.2-py3-none-any.whl", hash = "sha256:baeb28e1cd619a3c9ab8941431fe34e8490861fb998c6c4590693d50171db0cc"}, ] +cached-property = [ + {file = "cached-property-1.5.2.tar.gz", hash = "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130"}, + {file = "cached_property-1.5.2-py2.py3-none-any.whl", hash = "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0"}, +] chex = [ {file = "chex-0.1.5-py3-none-any.whl", hash = "sha256:b3321184850d5fc29b2eca63087cdbdd83a1b3e4f33c1314ff8b3b8bd67abbca"}, {file = "chex-0.1.5.tar.gz", hash = "sha256:686858320f8f220c82a6c7eeb54dcdcaa4f3d7f66690dacd13a24baa1ee8299e"}, @@ -802,40 +876,46 @@ cycler = [ {file = "cycler-0.11.0.tar.gz", hash = "sha256:9c87405839a19696e837b3b818fed3f5f69f16f1eec1a1ad77e043dcea9c772f"}, ] dm-tree = [ - {file = "dm-tree-0.1.7.tar.gz", hash = "sha256:30fec8aca5b92823c0e796a2f33b875b4dccd470b57e91e6c542405c5f77fd2a"}, - {file = "dm_tree-0.1.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3fae437135b6cbbdd51e96488a35e78c3617defa0b65265e7e8752d506f933fd"}, - {file = "dm_tree-0.1.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d377bd621b485db42c4aeea0eabbd8f6274b89a9c338c2c1bf69a40c3b86a1fd"}, - {file = "dm_tree-0.1.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1410fa2f2cc8dc7c01386f4e93ddeeb56765574ffafb632a9b6bd96496195b10"}, - {file = "dm_tree-0.1.7-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57edb6fbd88fcdd9908547cbf21045a9d663c0d9e5983dca7e6f9cf8b6584bb5"}, - {file = "dm_tree-0.1.7-cp310-cp310-win_amd64.whl", hash = "sha256:9edc1783a08d87c4e130781f55cbd904d6a564f7cce7dfb63f9ef3bee8e38209"}, - {file = "dm_tree-0.1.7-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:98fce150ceebb0a818f0eace1616004031cfa5e3375f50599ad790ff52414ba9"}, - {file = "dm_tree-0.1.7-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b4364fc9a5721a2b840ac8ea75b8f58b430bec9fdc8b99304d2aecb3cfe46b1b"}, - {file = "dm_tree-0.1.7-cp36-cp36m-win_amd64.whl", hash = "sha256:a085f500b295a6bf439c538e9058c7798ecb8c7d0dc916291f3d8d79d6124d17"}, - {file = "dm_tree-0.1.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f3e2bd9b9c05d1a0039f7c128d8b055c8a05708ef569cdbbeec0a2946e425bd4"}, - {file = "dm_tree-0.1.7-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:91c6240e47c9d80dbd7de5a29a2ca663143717a72c613130ba8ac4354fa741a9"}, - {file = "dm_tree-0.1.7-cp37-cp37m-win_amd64.whl", hash = "sha256:0f01743cc2247170e64798c6b4b31853717054bf9ceec47a1b1b8c2a4baf5792"}, - {file = "dm_tree-0.1.7-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:4992ac5c42af1d73042cd2d3af4e7892d3750e6c1bb8e5a4f81534aa6515f350"}, - {file = "dm_tree-0.1.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:20f24cad4decbf4c1f176a959d16e877c73df33b07d7d1f078a5b8abe72f79f8"}, - {file = "dm_tree-0.1.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3166304411d14c50a5da1c583e24d6069b44de0c9e06479cb36cdf048a466945"}, - {file = "dm_tree-0.1.7-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3b00885c21267934a3d3c68660811d3f891c9539fd53712f5b2423c6d74bf1e6"}, - {file = "dm_tree-0.1.7-cp38-cp38-win_amd64.whl", hash = "sha256:7f1f3dca9d669f3c09654ff6d69cfafd86a7f967c3095405b2692ee8d8ef3cfd"}, - {file = "dm_tree-0.1.7-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:51b9bdf1109b47cc22884b1919e6fe38edf28b5aa02e7c661bb760a0e7cf0157"}, - {file = "dm_tree-0.1.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2a843608e078d1622ebb5e50962a8c718d3fa1ab9461b95a12395a803545b2f5"}, - {file = "dm_tree-0.1.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7fa0740b7fbae2c3a43a3114a514891b5d6c383050828f36aa1816cf40f73a6a"}, - {file = "dm_tree-0.1.7-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1379a02df36e2bbff9819ceafa55ccd436b15af398803f781f372f8ead7ed871"}, - {file = "dm_tree-0.1.7-cp39-cp39-win_amd64.whl", hash = "sha256:3ca0a58e219b7b0bc201fea4679971188d0a9028a2543c16803a84e8f8c7eb2c"}, + {file = "dm-tree-0.1.8.tar.gz", hash = "sha256:0fcaabbb14e7980377439e7140bd05552739ca5e515ecb3119f234acee4b9430"}, + {file = "dm_tree-0.1.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:35cc164a79336bfcfafb47e5f297898359123bbd3330c1967f0c4994f9cf9f60"}, + {file = "dm_tree-0.1.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39070ba268c0491af9fe7a58644d99e8b4f2cde6e5884ba3380bddc84ed43d5f"}, + {file = "dm_tree-0.1.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2869228d9c619074de501a3c10dc7f07c75422f8fab36ecdcb859b6f1b1ec3ef"}, + {file = "dm_tree-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4d714371bb08839e4e5e29024fc95832d9affe129825ef38836b143028bd144"}, + {file = "dm_tree-0.1.8-cp310-cp310-win_amd64.whl", hash = "sha256:d40fa4106ca6edc66760246a08f500ec0c85ef55c762fb4a363f6ee739ba02ee"}, + {file = "dm_tree-0.1.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad16ceba90a56ec47cf45b21856d14962ac314787975ef786efb5e6e9ca75ec7"}, + {file = "dm_tree-0.1.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:803bfc53b4659f447ac694dbd04235f94a73ef7c1fd1e0df7c84ac41e0bc963b"}, + {file = "dm_tree-0.1.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:378cc8ad93c5fe3590f405a309980721f021c790ca1bdf9b15bb1d59daec57f5"}, + {file = "dm_tree-0.1.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83b7764de0d855338abefc6e3ee9fe40d301668310aa3baea3f778ff051f4393"}, + {file = "dm_tree-0.1.8-cp311-cp311-win_amd64.whl", hash = "sha256:a5d819c38c03f0bb5b3b3703c60e4b170355a0fc6b5819325bf3d4ceb3ae7e80"}, + {file = "dm_tree-0.1.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8c60a7eadab64c2278861f56bca320b2720f163dca9d7558103c3b77f2416571"}, + {file = "dm_tree-0.1.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f7915660f59c09068e428613c480150180df1060561fd0d1470684ae7007bd1"}, + {file = "dm_tree-0.1.8-cp37-cp37m-win_amd64.whl", hash = "sha256:b9f89a454e98806b44fe9d40ec9eee61f848388f7e79ac2371a55679bd5a3ac6"}, + {file = "dm_tree-0.1.8-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0e9620ccf06393eb6b613b5e366469304622d4ea96ae6540b28a33840e6c89cf"}, + {file = "dm_tree-0.1.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b095ba4f8ca1ba19350fd53cf1f8f3eb0bd406aa28af64a6dfc86707b32a810a"}, + {file = "dm_tree-0.1.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b9bd9b9ccb59409d33d51d84b7668010c04c2af7d4a371632874c1ca356cff3d"}, + {file = "dm_tree-0.1.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:694c3654cfd2a81552c08ec66bb5c4a3d48fa292b9a181880fb081c36c5b9134"}, + {file = "dm_tree-0.1.8-cp38-cp38-win_amd64.whl", hash = "sha256:bb2d109f42190225112da899b9f3d46d0d5f26aef501c61e43529fe9322530b5"}, + {file = "dm_tree-0.1.8-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d16e1f2a073604cfcc09f7131ae8d534674f43c3aef4c25742eae295bc60d04f"}, + {file = "dm_tree-0.1.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:250b692fb75f45f02e2f58fbef9ab338904ef334b90557565621fa251df267cf"}, + {file = "dm_tree-0.1.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81fce77f22a302d7a5968aebdf4efafef4def7ce96528719a354e6990dcd49c7"}, + {file = "dm_tree-0.1.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:181c35521d480d0365f39300542cb6cd7fd2b77351bb43d7acfda15aef63b317"}, + {file = "dm_tree-0.1.8-cp39-cp39-win_amd64.whl", hash = "sha256:8ed3564abed97c806db122c2d3e1a2b64c74a63debe9903aad795167cc301368"}, ] docstring-parser = [ {file = "docstring_parser-0.14.1-py3-none-any.whl", hash = "sha256:14ac6ec1f1ba6905c4d8cb90fd0bc55394f5678183752c90e44812bf28d7a515"}, {file = "docstring_parser-0.14.1.tar.gz", hash = "sha256:2c77522e31b7c88b1ab457a1f3c9ae38947ad719732260ba77ee8a3deb58622a"}, ] +etils = [ + {file = "etils-0.9.0-py3-none-any.whl", hash = "sha256:635d6f7d1c519eb194304228543a4c5c7df0e6b58243302473e34c18cf720588"}, + {file = "etils-0.9.0.tar.gz", hash = "sha256:489103e9e499a566765c60458ee15d185cf0065f2060a4d16a68f8f46962ed0d"}, +] exceptiongroup = [ - {file = "exceptiongroup-1.0.4-py3-none-any.whl", hash = "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828"}, - {file = "exceptiongroup-1.0.4.tar.gz", hash = "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"}, + {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, + {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, ] flax = [ - {file = "flax-0.6.2-py3-none-any.whl", hash = "sha256:9f933c87fb5762fbbf0920531e770bc385f24ef6eeb2f473641591fdbde9de89"}, - {file = "flax-0.6.2.tar.gz", hash = "sha256:a6247b412f14466fefcc70d043bd0facf72552d322acdda8a8700285308d390f"}, + {file = "flax-0.6.3-py3-none-any.whl", hash = "sha256:0cc0830f76a45c54ebe993aaa751319ea609ca1fb036d418d22259a0074a8759"}, + {file = "flax-0.6.3.tar.gz", hash = "sha256:064f33f24f49ecef01c171cc770d22493fb8f9a36ed29db5e75f82d2052682a9"}, ] fonttools = [ {file = "fonttools-4.38.0-py3-none-any.whl", hash = "sha256:820466f43c8be8c3009aef8b87e785014133508f0de64ec469e4efb643ae54fb"}, @@ -861,8 +941,12 @@ frozendict = [ {file = "frozendict-2.3.4.tar.gz", hash = "sha256:15b4b18346259392b0d27598f240e9390fafbff882137a9c48a1e0104fb17f78"}, ] importlib-metadata = [ - {file = "importlib_metadata-5.0.0-py3-none-any.whl", hash = "sha256:ddb0e35065e8938f867ed4928d0ae5bf2a53b7773871bfe6bcc7e4fcdc7dea43"}, - {file = "importlib_metadata-5.0.0.tar.gz", hash = "sha256:da31db32b304314d044d3c12c79bd59e307889b287ad12ff387b3500835fc2ab"}, + {file = "importlib_metadata-5.2.0-py3-none-any.whl", hash = "sha256:0eafa39ba42bf225fc00e67f701d71f85aead9f878569caf13c3724f704b970f"}, + {file = "importlib_metadata-5.2.0.tar.gz", hash = "sha256:404d48d62bba0b7a77ff9d405efd91501bef2e67ff4ace0bed40a0cf28c3c7cd"}, +] +importlib-resources = [ + {file = "importlib_resources-5.10.1-py3-none-any.whl", hash = "sha256:c09b067d82e72c66f4f8eb12332f5efbebc9b007c0b6c40818108c9870adc363"}, + {file = "importlib_resources-5.10.1.tar.gz", hash = "sha256:32bb095bda29741f6ef0e5278c42df98d135391bee5f932841efc0041f748dc3"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -1136,20 +1220,24 @@ nvidia-cudnn-cu11 = [ {file = "nvidia_cudnn_cu11-8.5.0.96-py3-none-manylinux1_x86_64.whl", hash = "sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108"}, ] omegaconf = [ - {file = "omegaconf-2.2.3-py3-none-any.whl", hash = "sha256:d6f2cbf79a992899eb76c6cb1aedfcf0fe7456a8654382edd5ee0c1b199c0657"}, - {file = "omegaconf-2.2.3.tar.gz", hash = "sha256:59ff9fba864ffbb5fb710b64e8a9ba37c68fa339a2e2bb4f1b648d6901552523"}, + {file = "omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b"}, + {file = "omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7"}, ] opt-einsum = [ {file = "opt_einsum-3.3.0-py3-none-any.whl", hash = "sha256:2455e59e3947d3c275477df7f5205b30635e266fe6dc300e3d9f9646bfcea147"}, {file = "opt_einsum-3.3.0.tar.gz", hash = "sha256:59f6475f77bbc37dcf7cd748519c0ec60722e91e63ca114e68821c0c54a46549"}, ] optax = [ - {file = "optax-0.1.3-py3-none-any.whl", hash = "sha256:33ac3b36bc8f6e087e112fd3a14ab054b99d1c26867aae552db80e234916e522"}, - {file = "optax-0.1.3.tar.gz", hash = "sha256:159e954405c3ba2072c2add7cec5532be7399bcafab3039acbf608b11844a879"}, + {file = "optax-0.1.4-py3-none-any.whl", hash = "sha256:12fcf33bd682f9a162a3deb097f864130c3224d76771af2ba09410de80399a9b"}, + {file = "optax-0.1.4.tar.gz", hash = "sha256:fb7a0550d57a6636164a3de25986a8a19be8ff6431fcdf1225b4e05175810f22"}, +] +orbax = [ + {file = "orbax-0.0.23-py3-none-any.whl", hash = "sha256:870d31d7e6b2aebb502bd6f90670f3f0e53ca4e4563773d6631629920c884744"}, + {file = "orbax-0.0.23.tar.gz", hash = "sha256:2d009e579b38a5a94f04bafc01f6d71b0220fc6664c3622bd951f2874e6c0cf1"}, ] packaging = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, + {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, + {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, ] Pillow = [ {file = "Pillow-9.3.0-1-cp37-cp37m-win32.whl", hash = "sha256:e6ea6b856a74d560d9326c0f5895ef8050126acfdc7ca08ad703eb0081e82b74"}, @@ -1265,8 +1353,8 @@ pyparsing = [ {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] pyright = [ - {file = "pyright-1.1.280-py3-none-any.whl", hash = "sha256:25917a14d873252c5c2e6fdbec322888c0480f6db95068ff6459befa9af3c92a"}, - {file = "pyright-1.1.280.tar.gz", hash = "sha256:4bcb167251419b3b736137b0535cb6bbfb5bef16eb74057eaf3ccaccb01d74c1"}, + {file = "pyright-1.1.285-py3-none-any.whl", hash = "sha256:8a6b60b3ff0d000c549621c367cdf0013abdaf24d09e6f0b4b95031b357cc4b1"}, + {file = "pyright-1.1.285.tar.gz", hash = "sha256:ecd28e8556352e2c7eb5f412c6841ec768d25e8a6136326d4a6a67d94370eba1"}, ] pytest = [ {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, @@ -1348,8 +1436,8 @@ scipy = [ {file = "scipy-1.6.1.tar.gz", hash = "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11"}, ] setuptools = [ - {file = "setuptools-65.6.0-py3-none-any.whl", hash = "sha256:6211d2f5eddad8757bd0484923ca7c0a6302ebc4ab32ea5e94357176e0ca0840"}, - {file = "setuptools-65.6.0.tar.gz", hash = "sha256:d1eebf881c6114e51df1664bc2c9133d022f78d12d5f4f665b9191f084e2862d"}, + {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, + {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, ] setuptools-scm = [ {file = "setuptools_scm-6.4.2-py3-none-any.whl", hash = "sha256:acea13255093849de7ccb11af9e1fb8bde7067783450cee9ef7a93139bddf6d4"}, @@ -1390,27 +1478,27 @@ toolz = [ {file = "toolz-0.12.0.tar.gz", hash = "sha256:88c570861c440ee3f2f6037c4654613228ff40c93a6c25e0eba70d17282c6194"}, ] torch = [ - {file = "torch-1.13.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:f68edfea71ade3862039ba66bcedf954190a2db03b0c41a9b79afd72210abd97"}, - {file = "torch-1.13.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:d2d2753519415d154de4d3e64d2eaaeefdba6b6fd7d69d5ffaef595988117700"}, - {file = "torch-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:6c227c16626e4ce766cca5351cc62a2358a11e8e466410a298487b9dff159eb1"}, - {file = "torch-1.13.0-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:49a949b8136b32b2ec0724cbf4c6678b54e974b7d68f19f1231eea21cde5c23b"}, - {file = "torch-1.13.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:0fdd38c96230947b1ed870fed4a560252f8d23c3a2bf4dab9d2d42b18f2e67c8"}, - {file = "torch-1.13.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:43db0723fc66ad6486f86dc4890c497937f7cd27429f28f73fb7e4d74b7482e2"}, - {file = "torch-1.13.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e643ac8d086706e82f77b5d4dfcf145a9dd37b69e03e64177fc23821754d2ed7"}, - {file = "torch-1.13.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:bb33a911460475d1594a8c8cb73f58c08293211760796d99cae8c2509b86d7f1"}, - {file = "torch-1.13.0-cp37-cp37m-win_amd64.whl", hash = "sha256:220325d0f4e69ee9edf00c04208244ef7cf22ebce083815ce272c7491f0603f5"}, - {file = "torch-1.13.0-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:cd1e67db6575e1b173a626077a54e4911133178557aac50683db03a34e2b636a"}, - {file = "torch-1.13.0-cp37-none-macosx_11_0_arm64.whl", hash = "sha256:9197ec216833b836b67e4d68e513d31fb38d9789d7cd998a08fba5b499c38454"}, - {file = "torch-1.13.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:fa768432ce4b8ffa29184c79a3376ab3de4a57b302cdf3c026a6be4c5a8ab75b"}, - {file = "torch-1.13.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:635dbb99d981a6483ca533b3dc7be18ef08dd9e1e96fb0bb0e6a99d79e85a130"}, - {file = "torch-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:857c7d5b1624c5fd979f66d2b074765733dba3f5e1cc97b7d6909155a2aae3ce"}, - {file = "torch-1.13.0-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:ef934a21da6f6a516d0a9c712a80d09c56128abdc6af8dc151bee5199b4c3b4e"}, - {file = "torch-1.13.0-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:f01a9ae0d4b69d2fc4145e8beab45b7877342dddbd4838a7d3c11ca7f6680745"}, - {file = "torch-1.13.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:9ac382cedaf2f70afea41380ad8e7c06acef6b5b7e2aef3971cdad666ca6e185"}, - {file = "torch-1.13.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:e20df14d874b024851c58e8bb3846249cb120e677f7463f60c986e3661f88680"}, - {file = "torch-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:4a378f5091307381abfb30eb821174e12986f39b1cf7c4522bf99155256819eb"}, - {file = "torch-1.13.0-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:922a4910613b310fbeb87707f00cb76fec328eb60cc1349ed2173e7c9b6edcd8"}, - {file = "torch-1.13.0-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:47fe6228386bff6d74319a2ffe9d4ed943e6e85473d78e80502518c607d644d2"}, + {file = "torch-1.13.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:fd12043868a34a8da7d490bf6db66991108b00ffbeecb034228bfcbbd4197143"}, + {file = "torch-1.13.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:d9fe785d375f2e26a5d5eba5de91f89e6a3be5d11efb497e76705fdf93fa3c2e"}, + {file = "torch-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:98124598cdff4c287dbf50f53fb455f0c1e3a88022b39648102957f3445e9b76"}, + {file = "torch-1.13.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:393a6273c832e047581063fb74335ff50b4c566217019cc6ace318cd79eb0566"}, + {file = "torch-1.13.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:0122806b111b949d21fa1a5f9764d1fd2fcc4a47cb7f8ff914204fd4fc752ed5"}, + {file = "torch-1.13.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:22128502fd8f5b25ac1cd849ecb64a418382ae81dd4ce2b5cebaa09ab15b0d9b"}, + {file = "torch-1.13.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:76024be052b659ac1304ab8475ab03ea0a12124c3e7626282c9c86798ac7bc11"}, + {file = "torch-1.13.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:ea8dda84d796094eb8709df0fcd6b56dc20b58fdd6bc4e8d7109930dafc8e419"}, + {file = "torch-1.13.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2ee7b81e9c457252bddd7d3da66fb1f619a5d12c24d7074de91c4ddafb832c93"}, + {file = "torch-1.13.1-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:0d9b8061048cfb78e675b9d2ea8503bfe30db43d583599ae8626b1263a0c1380"}, + {file = "torch-1.13.1-cp37-none-macosx_11_0_arm64.whl", hash = "sha256:f402ca80b66e9fbd661ed4287d7553f7f3899d9ab54bf5c67faada1555abde28"}, + {file = "torch-1.13.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:727dbf00e2cf858052364c0e2a496684b9cb5aa01dc8a8bc8bbb7c54502bdcdd"}, + {file = "torch-1.13.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:df8434b0695e9ceb8cc70650afc1310d8ba949e6db2a0525ddd9c3b2b181e5fe"}, + {file = "torch-1.13.1-cp38-cp38-win_amd64.whl", hash = "sha256:5e1e722a41f52a3f26f0c4fcec227e02c6c42f7c094f32e49d4beef7d1e213ea"}, + {file = "torch-1.13.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:33e67eea526e0bbb9151263e65417a9ef2d8fa53cbe628e87310060c9dcfa312"}, + {file = "torch-1.13.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:eeeb204d30fd40af6a2d80879b46a7efbe3cf43cdbeb8838dd4f3d126cc90b2b"}, + {file = "torch-1.13.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:50ff5e76d70074f6653d191fe4f6a42fdbe0cf942fbe2a3af0b75eaa414ac038"}, + {file = "torch-1.13.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:2c3581a3fd81eb1f0f22997cddffea569fea53bafa372b2c0471db373b26aafc"}, + {file = "torch-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:0aa46f0ac95050c604bcf9ef71da9f1172e5037fdf2ebe051962d47b123848e7"}, + {file = "torch-1.13.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:6930791efa8757cb6974af73d4996b6b50c592882a324b8fb0589c6a9ba2ddaf"}, + {file = "torch-1.13.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:e0df902a7c7dd6c795698532ee5970ce898672625635d885eade9976e5a04949"}, ] typed-ast = [ {file = "typed_ast-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4"}, @@ -1447,6 +1535,6 @@ wheel = [ {file = "wheel-0.38.4.tar.gz", hash = "sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac"}, ] zipp = [ - {file = "zipp-3.10.0-py3-none-any.whl", hash = "sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1"}, - {file = "zipp-3.10.0.tar.gz", hash = "sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8"}, + {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, + {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, ] diff --git a/tests/test_flax_ignore_py310.py b/tests/test_flax_ignore_py310.py index 7f057d3a0..9642692e2 100644 --- a/tests/test_flax_ignore_py310.py +++ b/tests/test_flax_ignore_py310.py @@ -2,7 +2,6 @@ import jax import pytest from flax import linen as nn -from flax.linen import Module # https://github.com/google/flax/issues/2636 from jax import numpy as jnp import tyro diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 145f9c111..eeff27a74 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -8,23 +8,13 @@ import functools import itertools import shlex -from typing import ( - Any, - Dict, - Mapping, - Optional, - Sequence, - Set, - Tuple, - Type, - TypeVar, - Union, -) +from typing import Any, Dict, Mapping, Optional, Sequence, Set, Tuple, TypeVar, Union import rich.markup import shtab from . import _fields, _instantiators, _resolver, _strings +from ._typing import TypeForm from .conf import _markers try: @@ -43,7 +33,7 @@ class ArgumentDefinition: name_prefix: str # User-facing prefix. subcommand_prefix: str # Prefix for nesting. field: _fields.FieldDefinition - type_from_typevar: Dict[TypeVar, Type[Any]] + type_from_typevar: Dict[TypeVar, TypeForm[Any]] def add_argument( self, parser: Union[argparse.ArgumentParser, argparse._ArgumentGroup] diff --git a/tyro/_cli.py b/tyro/_cli.py index e1502e82e..346f9cdfe 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -3,7 +3,7 @@ import dataclasses import sys import warnings -from typing import Callable, Optional, Sequence, Type, TypeVar, Union, cast, overload +from typing import Callable, Optional, Sequence, TypeVar, Union, cast, overload import shtab @@ -16,23 +16,20 @@ _strings, conf, ) +from ._typing import TypeForm OutT = TypeVar("OutT") -# Overload notes: -# 1. Type[T] is almost a subtype of Callable[..., T]; the difference is (in pyright) -# types like Union[T1, T2] which fall under the former but not the latter. -# 2. We really shouldn't need an overload here. But as of 1.1.268, it seems like it's -# needed for pyright to understand that Union types are OK to pass in directly. -# Hopefully we can just switch to a Union[Type[...], Callable[...]] in the future. -# 3. For equivalent functionality in mypy, we need to wait for typing.TypeForm: -# https://github.com/python/mypy/issues/9773 +# Note that the overload here is necessary for pyright and pylance due to special-casing +# related to using typing.Type[] as a temporary replacement for typing.TypeForm[]. +# +# https://github.com/microsoft/pyright/issues/4298 @overload def cli( - f: Type[OutT], + f: TypeForm[OutT], *, prog: Optional[str] = None, description: Optional[str] = None, @@ -58,7 +55,7 @@ def cli( def cli( - f: Union[Type[OutT], Callable[..., OutT]], + f: Union[TypeForm[OutT], Callable[..., OutT]], *, prog: Optional[str] = None, description: Optional[str] = None, @@ -138,7 +135,7 @@ def cli( @overload def get_parser( - f: Type[OutT], + f: TypeForm[OutT], *, prog: Optional[str] = None, description: Optional[str] = None, @@ -159,7 +156,7 @@ def get_parser( def get_parser( - f: Union[Type[OutT], Callable[..., OutT]], + f: Union[TypeForm[OutT], Callable[..., OutT]], *, # Note that we have no `args` argument, since this is only used when # parser.parse_args() is called. @@ -186,7 +183,7 @@ def get_parser( def _cli_impl( - f: Union[Type[OutT], Callable[..., OutT]], + f: Union[TypeForm[OutT], Callable[..., OutT]], *, prog: Optional[str] = None, description: Optional[str], @@ -225,7 +222,7 @@ def _cli_impl( # We wrap our type with a dummy dataclass if it can't be treated as a nested type. # For example: passing in f=int will result in a dataclass with a single field # typed as int. - if not _fields.is_nested_type(cast(Type, f), default_instance_internal): + if not _fields.is_nested_type(cast(type, f), default_instance_internal): dummy_field = cast( dataclasses.Field, dataclasses.field( @@ -234,7 +231,7 @@ def _cli_impl( ) f = dataclasses.make_dataclass( cls_name="", - fields=[(_strings.dummy_field_name, cast(Type, f), dummy_field)], + fields=[(_strings.dummy_field_name, cast(type, f), dummy_field)], ) dummy_wrapped = True else: diff --git a/tyro/_fields.py b/tyro/_fields.py index 21cded939..c54e2305e 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -20,7 +20,6 @@ List, Optional, Tuple, - Type, Union, cast, ) @@ -31,13 +30,14 @@ from . import conf # Avoid circular import. from . import _docstrings, _instantiators, _resolver, _singleton, _strings +from ._typing import TypeForm from .conf import _confstruct, _markers @dataclasses.dataclass(frozen=True) class FieldDefinition: name: str - typ: Type[Any] + typ: TypeForm[Any] default: Any helptext: Optional[str] markers: FrozenSet[_markers._Marker] @@ -59,7 +59,7 @@ def __post_init__(self): @staticmethod def make( name: str, - typ: Type[Any], + typ: TypeForm[Any], default: Any, helptext: Optional[str], call_argname_override: Optional[Any] = None, @@ -160,7 +160,7 @@ class UnsupportedNestedTypeMessage: message: str -def is_nested_type(typ: Type[Any], default_instance: _DefaultInstance) -> bool: +def is_nested_type(typ: TypeForm[Any], default_instance: _DefaultInstance) -> bool: """Determine whether a type should be treated as a 'nested type', where a single type can be broken down into multiple fields (eg for nested dataclasses or classes).""" @@ -171,7 +171,7 @@ def is_nested_type(typ: Type[Any], default_instance: _DefaultInstance) -> bool: def field_list_from_callable( - f: Union[Callable, Type[Any]], + f: Union[Callable, TypeForm[Any]], default_instance: _DefaultInstance, ) -> List[FieldDefinition]: """Generate a list of generic 'field' objects corresponding to the inputs of some @@ -208,7 +208,7 @@ def field_list_from_callable( def _try_field_list_from_callable( - f: Union[Callable, Type[Any]], + f: Union[Callable, TypeForm[Any]], default_instance: _DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: f, found_subcommand_configs = _resolver.unwrap_annotated( @@ -220,12 +220,12 @@ def _try_field_list_from_callable( # Unwrap generics. f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) - f_origin = _resolver.unwrap_origin_strip_extras(cast(Type, f)) + f_origin = _resolver.unwrap_origin_strip_extras(cast(TypeForm, f)) # If `f` is a type: # 1. Set cls to the type. # 2. Consider `f` to be `cls.__init__`. - cls: Optional[Type[Any]] = None + cls: Optional[TypeForm[Any]] = None if isinstance(f, type): cls = f f = cls.__init__ # type: ignore @@ -273,7 +273,7 @@ def _try_field_list_from_callable( def _field_list_from_typeddict( - cls: Type[Any], default_instance: _DefaultInstance + cls: TypeForm[Any], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: field_list = [] valid_default_instance = ( @@ -305,7 +305,7 @@ def _field_list_from_typeddict( def _field_list_from_namedtuple( - cls: Type[Any], default_instance: _DefaultInstance + cls: TypeForm[Any], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Handle NamedTuples. # @@ -336,7 +336,7 @@ def _field_list_from_namedtuple( def _field_list_from_dataclass( - cls: Type[Any], default_instance: _DefaultInstance + cls: TypeForm[Any], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Handle dataclasses. field_list = [] @@ -372,12 +372,12 @@ def _field_list_from_dataclass( pydantic = None # type: ignore -def _is_pydantic(cls: Type[Any]) -> bool: +def _is_pydantic(cls: TypeForm[Any]) -> bool: return pydantic is not None and issubclass(cls, pydantic.BaseModel) def _field_list_from_pydantic( - cls: Type[Any], default_instance: _DefaultInstance + cls: TypeForm[Any], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: assert pydantic is not None @@ -403,12 +403,12 @@ def _field_list_from_pydantic( attr = None # type: ignore -def _is_attrs(cls: Type[Any]) -> bool: +def _is_attrs(cls: TypeForm[Any]) -> bool: return attr is not None and attr.has(cls) def _field_list_from_attrs( - cls: Type[Any], default_instance: _DefaultInstance + cls: TypeForm[Any], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: assert attr is not None @@ -435,7 +435,7 @@ def _field_list_from_attrs( def _field_list_from_tuple( - f: Union[Callable, Type[Any]], default_instance: _DefaultInstance + f: Union[Callable, TypeForm[Any]], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Fixed-length tuples. field_list = [] @@ -495,7 +495,7 @@ def _field_list_from_tuple( def _field_list_from_sequence_checked( - f: Union[Callable, Type[Any]], default_instance: _DefaultInstance + f: Union[Callable, TypeForm[Any]], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: contained_type: Any if len(get_args(f)) == 0: @@ -512,7 +512,7 @@ def _field_list_from_sequence_checked( def _try_field_list_from_sequence_inner( - contained_type: Type[Any], + contained_type: TypeForm[Any], default_instance: _DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # When no default instance is specified: @@ -556,7 +556,7 @@ def _try_field_list_from_sequence_inner( def _field_list_from_dict( - f: Union[Callable, Type[Any]], + f: Union[Callable, TypeForm[Any]], default_instance: _DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: if default_instance in MISSING_SINGLETONS: @@ -579,8 +579,8 @@ def _field_list_from_dict( def _try_field_list_from_general_callable( - f: Union[Callable, Type[Any]], - cls: Optional[Type[Any]], + f: Union[Callable, TypeForm[Any]], + cls: Optional[TypeForm[Any]], default_instance: _DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Handle general callables. @@ -610,8 +610,8 @@ def _try_field_list_from_general_callable( def _field_list_from_params( - f: Union[Callable, Type[Any]], - cls: Optional[Type[Any]], + f: Union[Callable, TypeForm[Any]], + cls: Optional[TypeForm[Any]], params: List[inspect.Parameter], ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Unwrap functools.wraps and functools.partial. diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 8fcfaa204..08720160e 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -44,7 +44,6 @@ List, Optional, Tuple, - Type, TypeVar, Union, cast, @@ -54,6 +53,7 @@ from typing_extensions import Annotated, Final, Literal, get_args, get_origin from . import _strings +from ._typing import TypeForm _StandardInstantiator = Callable[[List[str]], Any] # Special case: the only time that argparse doesn't give us a string is when the @@ -94,7 +94,7 @@ class UnsupportedTypeAnnotationError(Exception): def instantiator_from_type( - typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] + typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] ) -> Tuple[Instantiator, InstantiatorMetadata]: """Recursive helper for parsing type annotations. @@ -192,7 +192,7 @@ def instantiator_base_case(strings: List[str]) -> Any: print(parser.parse_args().flag) ``` """ - assert len(get_args(typ)) == 0, f"Type {typ} cannot be instantiated." + assert len(get_args(typ)) == 0, f"TypeForm {typ} cannot be instantiated." (string,) = strings if typ is bool: return {"True": True, "False": False}[string] # type: ignore @@ -214,8 +214,8 @@ def instantiator_base_case(strings: List[str]) -> Any: @overload def _instantiator_from_type_inner( - typ: Type, - type_from_typevar: Dict[TypeVar, Type[Any]], + typ: TypeForm, + type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal["fixed_length"], ) -> Tuple[Instantiator, InstantiatorMetadata]: ... @@ -223,8 +223,8 @@ def _instantiator_from_type_inner( @overload def _instantiator_from_type_inner( - typ: Type, - type_from_typevar: Dict[TypeVar, Type[Any]], + typ: TypeForm, + type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal[False], ) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: ... @@ -232,16 +232,16 @@ def _instantiator_from_type_inner( @overload def _instantiator_from_type_inner( - typ: Type, - type_from_typevar: Dict[TypeVar, Type[Any]], + typ: TypeForm, + type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal[True], ) -> Tuple[Instantiator, InstantiatorMetadata]: ... def _instantiator_from_type_inner( - typ: Type, - type_from_typevar: Dict[TypeVar, Type[Any]], + typ: TypeForm, + type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal["fixed_length", True, False], ) -> Tuple[Instantiator, InstantiatorMetadata]: """Thin wrapper over instantiator_from_type, with some extra asserts for catching @@ -260,7 +260,7 @@ def _instantiator_from_type_inner( def _instantiator_from_container_type( - typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] + typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] ) -> Optional[Tuple[Instantiator, InstantiatorMetadata]]: """Attempt to create an instantiator from a container type. Returns `None` is no container type is found.""" @@ -296,7 +296,7 @@ def _instantiator_from_container_type( def _instantiator_from_tuple( - typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] + typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] ) -> Tuple[Instantiator, InstantiatorMetadata]: types = get_args(typ) typeset = set(types) # Note that sets are unordered. @@ -380,7 +380,7 @@ def _join_union_metavars(metavars: Iterable[str]) -> str: def _instantiator_from_union( - typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] + typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] ) -> Tuple[Instantiator, InstantiatorMetadata]: options = list(get_args(typ)) if NoneType in options: @@ -456,7 +456,7 @@ def union_instantiator(strings: List[str]) -> Any: def _instantiator_from_dict( - typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] + typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] ) -> Tuple[Instantiator, InstantiatorMetadata]: key_type, val_type = get_args(typ) key_instantiator, key_meta = _instantiator_from_type_inner( @@ -512,7 +512,7 @@ def dict_instantiator(strings: List[str]) -> Any: def _instantiator_from_sequence( - typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] + typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] ) -> Tuple[Instantiator, InstantiatorMetadata]: """Instantiator for variable-length sequences: list, sets, Tuple[T, ...], etc.""" container_type = get_origin(typ) @@ -556,7 +556,7 @@ def sequence_instantiator(strings: List[str]) -> Any: def _instantiator_from_literal( - typ: Type, type_from_typevar: Dict[TypeVar, Type[Any]] + typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] ) -> Tuple[Instantiator, InstantiatorMetadata]: choices = get_args(typ) str_choices = tuple(x.name if isinstance(x, enum.Enum) else str(x) for x in choices) diff --git a/tyro/_parsers.py b/tyro/_parsers.py index acce5cb30..06fd574a5 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -29,6 +29,7 @@ _resolver, _strings, ) +from ._typing import TypeForm from .conf import _confstruct, _markers T = TypeVar("T") @@ -332,7 +333,7 @@ class SubparsersSpecification: @staticmethod def from_field( field: _fields.FieldDefinition, - type_from_typevar: Dict[TypeVar, Type[Any]], + type_from_typevar: Dict[TypeVar, TypeForm[Any]], parent_classes: Set[Type[Any]], prefix: str, ) -> Optional[SubparsersSpecification]: diff --git a/tyro/_resolver.py b/tyro/_resolver.py index b288e779f..6e3b7160a 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -14,7 +14,6 @@ Optional, Set, Tuple, - Type, TypeVar, Union, cast, @@ -22,7 +21,9 @@ from typing_extensions import Annotated, get_args, get_origin, get_type_hints -TypeOrCallable = TypeVar("TypeOrCallable", Type, Callable) +from ._typing import TypeForm + +TypeOrCallable = TypeVar("TypeOrCallable", TypeForm, Callable) def unwrap_origin_strip_extras(typ: TypeOrCallable) -> TypeOrCallable: @@ -36,14 +37,14 @@ def unwrap_origin_strip_extras(typ: TypeOrCallable) -> TypeOrCallable: return origin -def is_dataclass(cls: Union[Type, Callable]) -> bool: +def is_dataclass(cls: Union[TypeForm, Callable]) -> bool: """Same as `dataclasses.is_dataclass`, but also handles generic aliases.""" return dataclasses.is_dataclass(unwrap_origin_strip_extras(cls)) def resolve_generic_types( cls: TypeOrCallable, -) -> Tuple[TypeOrCallable, Dict[TypeVar, Type[Any]]]: +) -> Tuple[TypeOrCallable, Dict[TypeVar, TypeForm[Any]]]: """If the input is a class: no-op. If it's a generic alias: returns the origin class, and a mapping from typevars to concrete types.""" @@ -76,7 +77,7 @@ def resolve_generic_types( return cls, type_from_typevar -def resolved_fields(cls: Type) -> List[dataclasses.Field]: +def resolved_fields(cls: TypeForm) -> List[dataclasses.Field]: """Similar to dataclasses.fields(), but includes dataclasses.InitVar types and resolves forward references.""" @@ -103,7 +104,7 @@ def resolved_fields(cls: Type) -> List[dataclasses.Field]: return fields -def is_namedtuple(cls: Type) -> bool: +def is_namedtuple(cls: TypeForm) -> bool: return ( hasattr(cls, "_fields") # `_field_types` was removed in Python >=3.9. @@ -112,7 +113,9 @@ def is_namedtuple(cls: Type) -> bool: ) -def type_from_typevar_constraints(typ: Union[Type, TypeVar]) -> Union[Type, TypeVar]: +def type_from_typevar_constraints( + typ: Union[TypeForm, TypeVar] +) -> Union[TypeForm, TypeVar]: """Try to concretize a type from a TypeVar's bounds or constraints. Identity if unsuccessful.""" if isinstance(typ, TypeVar): @@ -126,12 +129,12 @@ def type_from_typevar_constraints(typ: Union[Type, TypeVar]) -> Union[Type, Type # Be a little bit permissive with types here, since we often blur the lines between -# Callable[..., T] and Type[T]... this could be cleaned up! -TypeT = TypeVar("TypeT", bound=Callable) +# Callable[..., T] and TypeForm[T]... this could be cleaned up! +TypeT = TypeVar("TypeT", bound=Union[TypeForm, Callable]) def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: - """Type narrowing: if we annotate as Animal but specify a default instance of Cat, we + """TypeForm narrowing: if we annotate as Animal but specify a default instance of Cat, we should parse as Cat. This should generally only be applied to fields used as nested structures, not @@ -165,7 +168,7 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: def narrow_container_types(typ: TypeT, default_instance: Any) -> TypeT: - """Type narrowing for containers. Infers types of container contents.""" + """TypeForm narrowing for containers. Infers types of container contents.""" if typ is list and isinstance(default_instance, list): typ = List.__getitem__(Union.__getitem__(tuple(map(type, default_instance)))) # type: ignore elif typ is set and isinstance(default_instance, set): @@ -179,7 +182,7 @@ def narrow_container_types(typ: TypeT, default_instance: Any) -> TypeT: def unwrap_annotated( - typ: TypeOrCallable, search_type: Optional[Type[MetadataType]] = None + typ: TypeOrCallable, search_type: Optional[TypeForm[MetadataType]] = None ) -> Tuple[TypeOrCallable, Tuple[MetadataType, ...]]: """Helper for parsing typing.Annotated types. @@ -204,7 +207,7 @@ def unwrap_annotated( def apply_type_from_typevar( - typ: TypeOrCallable, type_from_typevar: Dict[TypeVar, Type[Any]] + typ: TypeOrCallable, type_from_typevar: Dict[TypeVar, TypeForm[Any]] ) -> TypeOrCallable: if typ in type_from_typevar: return type_from_typevar[typ] # type: ignore diff --git a/tyro/_typing.py b/tyro/_typing.py new file mode 100644 index 000000000..b925e3c36 --- /dev/null +++ b/tyro/_typing.py @@ -0,0 +1,16 @@ +# tyro's API relies heavily on parsing type annotations, which may be both "regular" +# types like `int`, `str`, and classes, or more general forms like `int | str` or +# `Annotated[str, ...]`. +# +# To correctly annotate variables that can take these more general type forms, we need +# to wait for a typing.TypeForm PEP: +# https://github.com/python/mypy/issues/9773 +# +# This doesn't yet exist, so in the meantime we use Type[T] everywhere we would +# otherwise have TypeForm[T]. This mostly works, and fortunately is supported by pyright +# and pylance (relevant: https://github.com/microsoft/pyright/issues/4298), but should +# be switched for the correct typing.TypeForm annotation once it's available. + +from typing import Type as TypeForm + +__all__ = ["TypeForm"] diff --git a/tyro/extras/_base_configs.py b/tyro/extras/_base_configs.py index cb6932e09..792782091 100644 --- a/tyro/extras/_base_configs.py +++ b/tyro/extras/_base_configs.py @@ -1,7 +1,8 @@ -from typing import Mapping, Type, TypeVar, Union +from typing import Mapping, TypeVar, Union from typing_extensions import Annotated +from .._typing import TypeForm from ..conf import subcommand T = TypeVar("T") @@ -12,17 +13,19 @@ def subcommand_type_from_defaults( descriptions: Mapping[str, str] = {}, *, prefix_names: bool = True, -) -> Type[T]: +) -> TypeForm[T]: """Construct a Union type for defining subcommands that choose between defaults. .. warning:: - Use of this helper is discouraged because it is compatible with ``pyright`` and - ``pylance``, but not with ``mypy``. For ``pyright`` support, you may need to enable - postponed evaluation of annotations (``from __future__ import annotations``). + Use of this helper is discouraged. It will likely be deprecated. + + Using the the returned type is understood as an annotation by ``pyright`` and + ``pylance`` (with ``from __future__ import annotations``), but it relies on + behavior that isn't defined by the Python language specifications. At the cost of verbosity, using :func:`tyro.conf.subcommand()` directly is - better supported by external tools. + better supported by tools like ``mypy``. Alternatively, we can work around this limitation with an ``if TYPE_CHECKING`` guard: @@ -32,8 +35,10 @@ def subcommand_type_from_defaults( from typing import TYPE_CHECKING if TYPE_CHECKING: - SelectableConfig = Config # For mypy. + # Static type seen by mypy, language servers, etc. + SelectableConfig = Config else: + # Runtime type used by tyro. SelectableConfig = subcommand_type_from_defaults(...) diff --git a/tyro/extras/_choices_type.py b/tyro/extras/_choices_type.py index c787f85e3..d10819d62 100644 --- a/tyro/extras/_choices_type.py +++ b/tyro/extras/_choices_type.py @@ -1,22 +1,26 @@ import enum -from typing import Iterable, Type, TypeVar, Union +from typing import Iterable, TypeVar, Union from typing_extensions import Literal +from .._typing import TypeForm + T = TypeVar("T", bound=Union[int, str, bool, enum.Enum]) -def literal_type_from_choices(choices: Iterable[T]) -> Type[T]: +def literal_type_from_choices(choices: Iterable[T]) -> TypeForm[T]: """Generate a `typing.Literal[]` type that constrains values to a set of choices. .. warning:: - Use of this helper is discouraged because it is compatible with ``pyright`` and - ``pylance``, but not with ``mypy``. For ``pyright`` support, you may need to enable - postponed evaluation of annotations (``from __future__ import annotations``). + Use of this helper is discouraged. It will likely be deprecated. + + The the returned type is understood as an annotation by ``pyright`` and + ``pylance`` (with ``from __future__ import annotations``), but it relies on + behavior that isn't defined by the Python language specifications. At the cost of verbosity, using ``typing.Literal[]`` directly is better supported - by external tools. + by tools like ``mypy``. Alternatively, we can work around this limitation with an ``if TYPE_CHECKING`` guard: @@ -26,8 +30,10 @@ def literal_type_from_choices(choices: Iterable[T]) -> Type[T]: from typing import TYPE_CHECKING if TYPE_CHECKING: - Color = str # For mypy. + # Static type seen by mypy, language servers, etc. + Color = str else: + # Runtime type used by tyro. Color = literal_type_from_choices(["red", "green", "blue"]) Using `Literal[...]` directly should generally be preferred, but this helper can be From 7d481abcd8871a48d7d07d931c1f6209ced4b4a1 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 30 Dec 2022 05:01:03 -0800 Subject: [PATCH 252/491] Fix broken f-string (cc #28) --- tyro/_arguments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tyro/_arguments.py b/tyro/_arguments.py index eeff27a74..8653d30f7 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -186,8 +186,8 @@ def _rule_handle_boolean_flags( ) assert False, ( - "Expected a boolean as a default for {arg.field.name}, but got" - " {lowered.default}." + f"Expected a boolean as a default for {arg.field.name}, but got" + f" {lowered.default}." ) From 46cb2ba51f1a013eaf015b95dfff6846c076c7c5 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 4 Jan 2023 14:50:38 -0800 Subject: [PATCH 253/491] Handle unions with mixed nested/non-nested types more gracefully --- tests/test_conf.py | 6 +-- tests/test_mixed_unions.py | 87 ++++++++++++++++++++++++++++++++++++ tyro/_calling.py | 12 ++--- tyro/_fields.py | 50 +++++++++++++++++---- tyro/_parsers.py | 24 ++-------- tyro/_resolver.py | 91 +++++++++++++++++++++++++++++++------- 6 files changed, 217 insertions(+), 53 deletions(-) create mode 100644 tests/test_mixed_unions.py diff --git a/tests/test_conf.py b/tests/test_conf.py index 532b2d4e1..a34508e18 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -372,7 +372,7 @@ class A: def test_fixed() -> None: - """When boolean flags have no default value, they must be explicitly specified.""" + """When an argument is fixed, we shouldn't be able to override it from the CLI.""" @dataclasses.dataclass class A: @@ -393,7 +393,7 @@ class A: def test_fixed_recursive() -> None: - """When boolean flags have no default value, they must be explicitly specified.""" + """When an argument is fixed, we shouldn't be able to override it from the CLI.""" @dataclasses.dataclass class A: @@ -561,7 +561,7 @@ class DefaultInstanceSubparser: # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] bc: tyro.conf.OmitSubcommandPrefixes[ Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] - ] = DefaultInstanceHTTPServer() + ] = dataclasses.field(default_factory=DefaultInstanceHTTPServer) assert ( tyro.cli( diff --git a/tests/test_mixed_unions.py b/tests/test_mixed_unions.py new file mode 100644 index 000000000..47fb35833 --- /dev/null +++ b/tests/test_mixed_unions.py @@ -0,0 +1,87 @@ +"""Tests for unsupported union types. + +Unions like `int | str` or `SomeDataclassA | SomeDataclassB` are generally OK (note that +the latter will produce a pair of subcommands), but when we write things like +`int | SomeDataclassA` handling gets more complicated; see docstring for +`narrow_union_type()` in _resolvers.py. + +Hopefully we can fix/improve this in the future! +""" + + +import dataclasses +from typing import Union + +import pytest + +import tyro + + +def test_subparser_strip_non_nested() -> None: + @dataclasses.dataclass + class DefaultHTTPServer: + y: int + + @dataclasses.dataclass + class DefaultSMTPServer: + z: int + + @dataclasses.dataclass + class DefaultSubparser: + x: int + # Note that we add [int, str] to the annotation here... this should be ignored. + bc: Union[int, str, DefaultHTTPServer, DefaultSMTPServer] = dataclasses.field( + default_factory=lambda: DefaultHTTPServer(5) + ) + + assert ( + tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "5"] + ) + == tyro.cli(DefaultSubparser, args=["--x", "1"]) + == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=5)) + ) + assert tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-smtp-server", "--bc.z", "3"] + ) == DefaultSubparser(x=1, bc=DefaultSMTPServer(z=3)) + assert ( + tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "8"] + ) + == tyro.cli( + DefaultSubparser, + args=[], + default=DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)), + ) + == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)) + ) + + with pytest.raises(SystemExit): + tyro.cli(DefaultSubparser, args=["--x", "1", "b", "--bc.z", "3"]) + with pytest.raises(SystemExit): + tyro.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) + + +def test_subparser_strip_nested() -> None: + @dataclasses.dataclass + class DefaultHTTPServer: + y: int + + @dataclasses.dataclass + class DefaultSMTPServer: + z: int + + @dataclasses.dataclass + class DefaultSubparser: + x: int + # Note that we add [int, str] to the annotation here... this should be ignored. + bc: Union[int, str, DefaultHTTPServer, DefaultSMTPServer] = 5 + + assert ( + tyro.cli(DefaultSubparser, args=["--x", "1", "--bc", "5"]) + == tyro.cli(DefaultSubparser, args=["--x", "1"]) + == DefaultSubparser(x=1, bc=5) + ) + assert tyro.cli( + DefaultSubparser, args=["--x", "1", "--bc", "five"] + ) == DefaultSubparser(x=1, bc="five") diff --git a/tyro/_calling.py b/tyro/_calling.py index 7dd2329f4..b3e579963 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -29,8 +29,10 @@ def call_from_args( Returns the output of `f` and a set of used arguments.""" - f, type_from_typevar = _resolver.resolve_generic_types(f) - f = _resolver.narrow_type(f, default_instance) + # Resolve the type of `f`, generate a field list. + f, type_from_typevar, field_list = _fields.field_list_from_callable( + f=f, default_instance=default_instance + ) args: List[Any] = [] kwargs: Dict[str, Any] = {} @@ -48,14 +50,12 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: _strings.make_field_name([arg.dest_prefix, arg.field.name]) ] = arg - for field in _fields.field_list_from_callable( - f, default_instance=default_instance - ): # type: ignore + for field in field_list: value: Any prefixed_field_name = _strings.make_field_name([field_name_prefix, field.name]) # Resolve field type. - field_type = _resolver.apply_type_from_typevar(field.typ, type_from_typevar) # type: ignore + field_type = field.typ if prefixed_field_name in arg_from_prefixed_field_name: assert prefixed_field_name not in consumed_keywords diff --git a/tyro/_fields.py b/tyro/_fields.py index c54e2305e..6e1739dbe 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -14,19 +14,22 @@ from typing import ( Any, Callable, + Dict, FrozenSet, Hashable, Iterable, List, Optional, Tuple, + TypeVar, Union, cast, ) import docstring_parser import typing_extensions -from typing_extensions import get_args, get_type_hints, is_typeddict +from attr import dataclass +from typing_extensions import Annotated, get_args, get_type_hints, is_typeddict from . import conf # Avoid circular import. from . import _docstrings, _instantiators, _resolver, _singleton, _strings @@ -163,7 +166,10 @@ class UnsupportedNestedTypeMessage: def is_nested_type(typ: TypeForm[Any], default_instance: _DefaultInstance) -> bool: """Determine whether a type should be treated as a 'nested type', where a single type can be broken down into multiple fields (eg for nested dataclasses or - classes).""" + classes). + + TODO: we should come up with a better name than 'nested type', which is a little bit + misleading.""" return not isinstance( _try_field_list_from_callable(typ, default_instance), UnsupportedNestedTypeMessage, @@ -173,18 +179,44 @@ def is_nested_type(typ: TypeForm[Any], default_instance: _DefaultInstance) -> bo def field_list_from_callable( f: Union[Callable, TypeForm[Any]], default_instance: _DefaultInstance, -) -> List[FieldDefinition]: +) -> Tuple[ + Union[Callable, TypeForm[Any]], Dict[TypeVar, TypeForm], List[FieldDefinition] +]: """Generate a list of generic 'field' objects corresponding to the inputs of some - annotated callable.""" - out = _try_field_list_from_callable(f, default_instance) + annotated callable. + + Returns: + The type that `f` is resolved as. + A type_from_typevar dict. + A list of field definitions. + """ + # Resolve generic types. + f, type_from_typevar = _resolver.resolve_generic_types(f) + f = _resolver.narrow_type(f, default_instance) + + # Try to generate field list. + field_list = _try_field_list_from_callable(f, default_instance) - if isinstance(out, UnsupportedNestedTypeMessage): - raise _instantiators.UnsupportedTypeAnnotationError(out.message) + if isinstance(field_list, UnsupportedNestedTypeMessage): + raise _instantiators.UnsupportedTypeAnnotationError(field_list.message) # Recursively apply markers. _, parent_markers = _resolver.unwrap_annotated(f, _markers._Marker) - out = list(map(lambda field: field.add_markers(parent_markers), out)) - return out + field_list = list(map(lambda field: field.add_markers(parent_markers), field_list)) + + # Try to resolve types in our list of fields. + def resolve(field: FieldDefinition) -> FieldDefinition: + typ = field.typ + typ = _resolver.apply_type_from_typevar(typ, type_from_typevar) + typ = _resolver.type_from_typevar_constraints(typ) + typ = _resolver.narrow_container_types(typ, field.default) + typ = _resolver.narrow_union_type(typ, field.default) + field = dataclasses.replace(field, typ=typ) + return field + + field_list = list(map(resolve, field_list)) + + return f, type_from_typevar, field_list # Implementation details below. diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 06fd574a5..315b78856 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -74,9 +74,10 @@ def from_callable_or_type( in _resolver.unwrap_annotated(f, _markers._Marker)[1] ) - # Resolve generic types. - f, type_from_typevar = _resolver.resolve_generic_types(f) - f = _resolver.narrow_type(f, default_instance) + # Resolve the type of `f`, generate a field list. + f, type_from_typevar, field_list = _fields.field_list_from_callable( + f=f, default_instance=default_instance + ) # Cycle detection. # @@ -99,24 +100,7 @@ def from_callable_or_type( subparsers = None subparsers_from_prefix = {} - field_list = _fields.field_list_from_callable( - f=f, default_instance=default_instance - ) for field in field_list: - field = dataclasses.replace( - field, - # Resolve generic types. - typ=_resolver.narrow_container_types( - _resolver.type_from_typevar_constraints( # type: ignore - _resolver.apply_type_from_typevar( - field.typ, - type_from_typevar, - ) - ), - default_instance=field.default, - ), - ) - if isinstance(field.typ, TypeVar): raise _instantiators.UnsupportedTypeAnnotationError( f"Field {field.name} has an unbound TypeVar: {field.typ}." diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 6e3b7160a..cf390ea6e 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -21,6 +21,7 @@ from typing_extensions import Annotated, get_args, get_origin, get_type_hints +from . import _fields from ._typing import TypeForm TypeOrCallable = TypeVar("TypeOrCallable", TypeForm, Callable) @@ -59,7 +60,6 @@ def resolve_generic_types( ): typevars = origin_cls.__parameters__ typevar_values = get_args(cls) - print(typevars, typevar_values, origin_cls) assert len(typevars) == len(typevar_values) cls = origin_cls type_from_typevar.update(dict(zip(typevars, typevar_values))) @@ -113,9 +113,7 @@ def is_namedtuple(cls: TypeForm) -> bool: ) -def type_from_typevar_constraints( - typ: Union[TypeForm, TypeVar] -) -> Union[TypeForm, TypeVar]: +def type_from_typevar_constraints(typ: TypeOrCallable) -> TypeOrCallable: """Try to concretize a type from a TypeVar's bounds or constraints. Identity if unsuccessful.""" if isinstance(typ, TypeVar): @@ -128,14 +126,9 @@ def type_from_typevar_constraints( return typ -# Be a little bit permissive with types here, since we often blur the lines between -# Callable[..., T] and TypeForm[T]... this could be cleaned up! -TypeT = TypeVar("TypeT", bound=Union[TypeForm, Callable]) - - -def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: - """TypeForm narrowing: if we annotate as Animal but specify a default instance of Cat, we - should parse as Cat. +def narrow_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCallable: + """Type narrowing: if we annotate as Animal but specify a default instance of Cat, + we should parse as Cat. This should generally only be applied to fields used as nested structures, not individual arguments/fields. (if a field is annotated as Union[int, str], and a @@ -151,7 +144,7 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: superclass = unwrap_annotated(typ)[0] - # For Python 3.10: don't narrow union types. + # For Python 3.10. if get_origin(superclass) is Union: return typ @@ -160,14 +153,18 @@ def narrow_type(typ: TypeT, default_instance: Any) -> TypeT: return Annotated.__class_getitem__( # type: ignore (potential_subclass,) + get_args(typ)[1:] ) - typ = cast(TypeT, potential_subclass) + typ = cast(TypeOrCallable, potential_subclass) except TypeError: + # TODO: document where this TypeError can be raised, and reduce the amount of + # code in it. pass return typ -def narrow_container_types(typ: TypeT, default_instance: Any) -> TypeT: +def narrow_container_types( + typ: TypeOrCallable, default_instance: Any +) -> TypeOrCallable: """TypeForm narrowing for containers. Infers types of container contents.""" if typ is list and isinstance(default_instance, list): typ = List.__getitem__(Union.__getitem__(tuple(map(type, default_instance)))) # type: ignore @@ -242,3 +239,67 @@ def apply_type_from_typevar( return typ.copy_with(tuple(apply_type_from_typevar(x, type_from_typevar) for x in args)) # type: ignore return typ + + +def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCallable: + """Narrow union types. This is a shim for failing more gracefully when we we're + given an unsupported union. + + When do we want to narrow Union types? + + Unions over nested types: no. + typ = NestedA | NestedB + => NestedA | NestedB can be converted to two subcommands. + + Unions over nested and not nested types: no. + typ = int | str + => int | str can be instantiated as a union. + + Unions over mixed nested / not nested types: if the default is a nested + type, strip out the non-nested ones. If the default is a non-nested + type, strip out the nested ones. + + typ = NestedA | int, default_instance = NestedA() + => NestedA + + typ = NestedA | int, default_instance = 5 + => int + + typ = NestedA | NestedB | int, default_instance = NestedA() + => NestedA + + This is a hack to get around the fact that we don't currently support + mixing nested types (eg `SomeDataclass`) and non-nested ones (eg `int` or + `int | str`) in unions. This should be supported in the future, but will + likely require a big code refactor.""" + if get_origin(typ) is not Union: + return typ + options = get_args(typ) + is_nested = tuple( + map( + lambda option: _fields.is_nested_type( + option, + _fields.MISSING_NONPROP, + ), + options, + ) + ) + if type(None) in options: + none_index = options.index(type(None)) + is_nested_no_none = is_nested[:none_index] + is_nested[none_index + 1 :] + else: + is_nested_no_none = is_nested + + if all(is_nested_no_none) or not any(is_nested_no_none): + # Either all types are nested or none of them are. + return typ + else: + is_default_nested = _fields.is_nested_type(type(default_instance), default_instance) # type: ignore + out = Union.__getitem__( # type: ignore + tuple( + option + for option, nested in zip(get_args(typ), is_nested) + if nested is is_default_nested + ) + ) + return out # type: ignore From 444fbf9629c13b218f62a0c7ab5087e0fd8da50b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 5 Jan 2023 16:40:26 -0800 Subject: [PATCH 254/491] Simplify call mechanics, extra TypedDict test --- tests/test_dict_namedtuple.py | 17 +++++++++++++++++ tyro/_calling.py | 24 +++++++++--------------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index 88c47cf4f..9f3efb1b8 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -72,6 +72,23 @@ class ManyTypesTypedDict(TypedDict): tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) +def test_positional_in_typeddict() -> None: + class ManyTypesTypedDict(TypedDict): + i: tyro.conf.Positional[int] + s: str + + assert tyro.cli( + ManyTypesTypedDict, + args="5 --s 5".split(" "), + ) == dict(i=5, s="5") + + with pytest.raises(SystemExit): + tyro.cli(ManyTypesTypedDict, args="5".split(" ")) + + with pytest.raises(SystemExit): + tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) + + def test_total_false_typeddict() -> None: class ManyTypesTypedDict(TypedDict, total=False): i: int diff --git a/tyro/_calling.py b/tyro/_calling.py index b3e579963..68ebbb7e0 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -34,7 +34,7 @@ def call_from_args( f=f, default_instance=default_instance ) - args: List[Any] = [] + positional_args: List[Any] = [] kwargs: Dict[str, Any] = {} consumed_keywords: Set[str] = set() @@ -162,7 +162,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: if value is not _fields.EXCLUDE_FROM_CALL: if field.is_positional_call(): - args.append(value) + positional_args.append(value) else: kwargs[field.call_argname] = value @@ -176,19 +176,13 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore if unwrapped_f in (tuple, list, set): - if len(args) == 0: - # When tuples are used as nested structures (eg Tuple[SomeDataclass]), we - # use keyword arguments. - return unwrapped_f(kwargs.values()), consumed_keywords # type: ignore - else: - # When tuples are directly parsed (eg Tuple[int, int]), we end up with a - # single set of positional arguments. - assert len(args) == 1 - return unwrapped_f(args[0]), consumed_keywords # type: ignore + assert len(positional_args) == 0 + # When tuples are used as nested structures (eg Tuple[SomeDataclass]), we + # use keyword arguments. + assert len(positional_args) == 0 + return unwrapped_f(kwargs.values()), consumed_keywords # type: ignore elif unwrapped_f is dict: - for arg in args: - assert isinstance(arg, dict) - kwargs.update(arg) + assert len(positional_args) == 0 return kwargs, consumed_keywords # type: ignore else: - return unwrapped_f(*args, **kwargs), consumed_keywords # type: ignore + return unwrapped_f(*positional_args, **kwargs), consumed_keywords # type: ignore From 4016a27db52263e45b84f7099275fc9925d5d156 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 6 Jan 2023 01:04:17 -0800 Subject: [PATCH 255/491] Fix subcommand edge case + add test (cc #29) --- tests/test_nested.py | 51 ++++++++++++++++++++++++++++++++++++++++++++ tyro/_fields.py | 3 +-- tyro/_parsers.py | 6 +++--- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/tests/test_nested.py b/tests/test_nested.py index 69cefff09..53ed493f3 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -628,6 +628,57 @@ class MultipleSubparsers: ) == MultipleSubparsers(Subcommand2(Subcommand1(3)), Subcommand2(Subcommand1(7))) +def test_nested_subparsers_multiple_in_container() -> None: + @dataclasses.dataclass(frozen=True) + class Subcommand1: + x: int = 0 + + @dataclasses.dataclass(frozen=True) + class Subcommand3: + z: int = 2 + + @dataclasses.dataclass(frozen=True) + class Subcommand2: + y: Union[Subcommand1, Subcommand3] + + @dataclasses.dataclass(frozen=True) + class MultipleSubparsers: + a: Union[Subcommand1, Subcommand2] + b: Union[Subcommand1, Subcommand2] + + @dataclasses.dataclass(frozen=True) + class Root: + inner: MultipleSubparsers + + with pytest.raises(SystemExit): + tyro.cli(Root, args=[]) + assert tyro.cli( + Root, args="inner.a:subcommand1 inner.b:subcommand1".split(" ") + ) == Root(MultipleSubparsers(Subcommand1(), Subcommand1())) + assert tyro.cli( + Root, + args="inner.a:subcommand1 inner.b:subcommand2 inner.b.y:subcommand1".split(" "), + ) == Root(MultipleSubparsers(Subcommand1(), Subcommand2(Subcommand1()))) + assert tyro.cli( + Root, + args=( + "inner.a:subcommand2 inner.a.y:subcommand1 inner.b:subcommand2" + " inner.b.y:subcommand1".split(" ") + ), + ) == Root( + MultipleSubparsers(Subcommand2(Subcommand1()), Subcommand2(Subcommand1())) + ) + assert tyro.cli( + Root, + args=( + "inner.a:subcommand2 inner.a.y:subcommand1 --inner.a.y.x 3" + " inner.b:subcommand2 inner.b.y:subcommand1 --inner.b.y.x 7".split(" ") + ), + ) == Root( + MultipleSubparsers(Subcommand2(Subcommand1(3)), Subcommand2(Subcommand1(7))) + ) + + def test_tuple_nesting() -> None: @dataclasses.dataclass(frozen=True) class Color: diff --git a/tyro/_fields.py b/tyro/_fields.py index 6e1739dbe..2018a38ab 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -28,8 +28,7 @@ import docstring_parser import typing_extensions -from attr import dataclass -from typing_extensions import Annotated, get_args, get_type_hints, is_typeddict +from typing_extensions import get_args, get_type_hints, is_typeddict from . import conf # Avoid circular import. from . import _docstrings, _instantiators, _resolver, _singleton, _strings diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 315b78856..f017e9945 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -160,9 +160,9 @@ def from_callable_or_type( # Include nested subparsers. if nested_parser.subparsers is not None: - subparsers_from_prefix[ - nested_parser.subparsers.prefix - ] = nested_parser.subparsers + subparsers_from_prefix.update( + nested_parser.subparsers_from_prefix + ) subparsers = add_subparsers_to_leaves( subparsers, nested_parser.subparsers ) From 37cf5b8522298c162cbc16b5032e5c19afaf2899 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 6 Jan 2023 01:09:33 -0800 Subject: [PATCH 256/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4e8d95b34..746d9212c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.36" +version = "0.3.37" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] From ff146ed62c6e7c77f9454de1fe33191b1179f762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Sok=C3=B3lski?= Date: Mon, 16 Jan 2023 18:07:00 +0000 Subject: [PATCH 257/491] Move mypy to dev dependencies (#31) It does not appear to be used for anything other than tests --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 746d9212c..cacd6d9f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,6 @@ colorama = {version = "^0.4.0", platform = "win32"} frozendict = "^2.3.4" rich = ">=11.1.0" shtab = "^1.5.6" -mypy = "^0.991" [tool.poetry.group.dev.dependencies] pytest = "^7.1.2" @@ -28,6 +27,7 @@ omegaconf = "^2.2.2" attrs = "^21.4.0" torch = "^1.10.0" pyright = "^1.1.264" +mypy = "^0.991" numpy = ">=1.20.0" flax = "^0.6.0" pydantic = "^1.10.2" From 4ee8be028cfd9bc881496bef133ea76c9b34c06f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 25 Jan 2023 14:42:09 -0800 Subject: [PATCH 258/491] Minor pydantic helptext improvements + tests --- pyproject.toml | 7 +++++- tests/test_conf.py | 6 ++++-- tests/test_pydantic.py | 49 ++++++++++++++++++++++++++++++++++++++++++ tyro/_cli.py | 6 ++++-- tyro/_docstrings.py | 8 +++++++ tyro/_fields.py | 14 ++++++++---- tyro/_resolver.py | 3 ++- 7 files changed, 83 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cacd6d9f5..4eef48541 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.37" +version = "0.3.38" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] @@ -76,3 +76,8 @@ exclude_lines = [ # or anything that's deprecated "deprecated", ] + +[tool.ruff] +ignore = [ + "E501", # Ignore line length errors. +] diff --git a/tests/test_conf.py b/tests/test_conf.py index a34508e18..285187770 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Any, Callable, Generic, TypeVar, Union +from typing import Any, Generic, TypeVar, Union import pytest from helptext_utils import get_helptext @@ -496,7 +496,9 @@ def test_suppress_auto_fixed() -> None: @dataclasses.dataclass class Struct: a: int = 5 - b: Callable = lambda x: 5 + + def b(x): + return 5 def main(x: tyro.conf.SuppressFixed[Any] = Struct()): pass diff --git a/tests/test_pydantic.py b/tests/test_pydantic.py index da9621e7a..a296ceb55 100644 --- a/tests/test_pydantic.py +++ b/tests/test_pydantic.py @@ -48,3 +48,52 @@ class Helptext(BaseModel): assert "Documentation 1" in helptext assert "Documentation 2" in helptext assert "Documentation 3" in helptext + + +def test_pydantic_suppress_base_model_helptext() -> None: + class Helptext(BaseModel): + x: int = Field(description="Documentation 1") + + y: int = Field(description="Documentation 2") + + z: int = Field(description="Documentation 3") + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(Helptext, args=["--help"]) + helptext = f.getvalue() + + assert "Create a new model by parsing and validating" not in helptext + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + assert "Documentation 3" in helptext + + +class HelptextWithFieldDocstring(BaseModel): + """This docstring should be printed as a description.""" + + x: int + """Documentation 1""" + + y: int = Field(description="Documentation 2") + + z: int = Field(description="Documentation 3") + + +def test_pydantic_field_helptext_from_docstring() -> None: + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(HelptextWithFieldDocstring, args=["--help"]) + helptext = f.getvalue() + assert ( + tyro._strings.strip_ansi_sequences( + cast(str, HelptextWithFieldDocstring.__doc__) + ) + in helptext + ) + + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + assert "Documentation 3" in helptext diff --git a/tyro/_cli.py b/tyro/_cli.py index 346f9cdfe..d76c1a9d3 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -204,8 +204,10 @@ def _cli_impl( if deprecated_kwargs.get("avoid_subparsers", False): f = conf.AvoidSubcommands[f] # type: ignore warnings.warn( - "`avoid_subparsers=` is deprecated! use `tyro.conf.AvoidSubparsers[]`" - " instead.", + ( + "`avoid_subparsers=` is deprecated! use `tyro.conf.AvoidSubparsers[]`" + " instead." + ), stacklevel=2, ) diff --git a/tyro/_docstrings.py b/tyro/_docstrings.py index a01007aff..32bafc6eb 100644 --- a/tyro/_docstrings.py +++ b/tyro/_docstrings.py @@ -297,6 +297,11 @@ def get_callable_description(f: Callable) -> str: if isinstance(f, functools.partial): f = f.func + try: + import pydantic + except ImportError: + pydantic = None # type: ignore + # Note inspect.getdoc() causes some corner cases with TypedDicts. docstring = f.__doc__ if ( @@ -304,7 +309,10 @@ def get_callable_description(f: Callable) -> str: and isinstance(f, type) # Ignore TypedDict's __init__ docstring, because it will just be `dict` and not is_typeddict(f) + # Ignore NamedTuple __init__ docstring. and not _resolver.is_namedtuple(f) + # Ignore pydantic base model constructor docstring. + and not (pydantic is not None and f.__init__ is pydantic.BaseModel.__init__) # type: ignore ): docstring = f.__init__.__doc__ # type: ignore if docstring is None: diff --git a/tyro/_fields.py b/tyro/_fields.py index 2018a38ab..0fe1517c9 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -415,6 +415,10 @@ def _field_list_from_pydantic( # Handle pydantic models. field_list = [] for pd_field in cls.__fields__.values(): # type: ignore + helptext = pd_field.field_info.description + if helptext is None: + helptext = _docstrings.get_field_docstring(cls, pd_field.name) + field_list.append( FieldDefinition.make( name=pd_field.name, @@ -422,7 +426,7 @@ def _field_list_from_pydantic( default=MISSING_NONPROP if pd_field.required else pd_field.get_default(), - helptext=pd_field.field_info.description, + helptext=helptext, ) ) return field_list @@ -747,9 +751,11 @@ def _get_dataclass_field_default( return getattr(parent_default_instance, field.name) else: warnings.warn( - f"Could not find field {field.name} in default instance" - f" {parent_default_instance}, which has" - f" type {type(parent_default_instance)},", + ( + f"Could not find field {field.name} in default instance" + f" {parent_default_instance}, which has" + f" type {type(parent_default_instance)}," + ), stacklevel=2, ) diff --git a/tyro/_resolver.py b/tyro/_resolver.py index cf390ea6e..506d36d49 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -28,7 +28,8 @@ def unwrap_origin_strip_extras(typ: TypeOrCallable) -> TypeOrCallable: - """Returns the origin, ignoring typing.Annotated, of typ if it exists. Otherwise, returns typ.""" + """Returns the origin, ignoring typing.Annotated, of typ if it exists. Otherwise, + returns typ.""" # TODO: Annotated[] handling should be revisited... typ, _ = unwrap_annotated(typ) origin = get_origin(typ) From 4ea2bddc0bcf16a8533a6f98b607f93fe6ca89f2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 12 Feb 2023 01:19:01 -0800 Subject: [PATCH 259/491] Add special casing for flax, formatting --- tests/test_flax_ignore_py310.py | 5 +++++ tests/test_helptext.py | 4 +--- tests/test_nested_in_containers.py | 20 ++++++++++++++++---- tyro/_argparse_formatter.py | 14 ++++++++------ tyro/_cli.py | 6 +++++- tyro/_fields.py | 28 ++++++++++++++++++++++------ tyro/_instantiators.py | 8 +++++--- tyro/_parsers.py | 22 +++++++++++++--------- 8 files changed, 75 insertions(+), 32 deletions(-) diff --git a/tests/test_flax_ignore_py310.py b/tests/test_flax_ignore_py310.py index 9642692e2..4448c556b 100644 --- a/tests/test_flax_ignore_py310.py +++ b/tests/test_flax_ignore_py310.py @@ -2,6 +2,7 @@ import jax import pytest from flax import linen as nn +from helptext_utils import get_helptext from jax import numpy as jnp import tyro @@ -46,6 +47,10 @@ def test_ok(): params = network.init(jax.random.PRNGKey(0), x) assert network.apply(params, x).shape == (10, 3) + helptext = get_helptext(Classifier) + assert "parent" not in helptext + assert "name" not in helptext + def test_missing(): with pytest.raises(SystemExit): diff --git a/tests/test_helptext.py b/tests/test_helptext.py index af8dbaaf7..b8aa3e476 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -510,9 +510,7 @@ def main( def test_metavar_5() -> None: def main( - x: List[ - Union[Tuple[int, int], Tuple[str, str]], - ] = [(1, 1), (2, 2)] + x: List[Union[Tuple[int, int], Tuple[str, str]],] = [(1, 1), (2, 2)] ) -> None: pass diff --git a/tests/test_nested_in_containers.py b/tests/test_nested_in_containers.py index 91e095aac..a3eab0e1e 100644 --- a/tests/test_nested_in_containers.py +++ b/tests/test_nested_in_containers.py @@ -321,7 +321,10 @@ def main( ) -> Any: return x - assert tyro.cli(main, args="--x.float.g 0.1".split(" "),)[ + assert tyro.cli( + main, + args="--x.float.g 0.1".split(" "), + )[ "float" ] == GenericColor(0.5, 0.1, 0.3) assert tyro.cli( @@ -349,10 +352,16 @@ def main( ) -> Any: return x - assert tyro.cli(main, args="--x.hello.float.g 0.1".split(" "),)["hello"][ + assert tyro.cli( + main, + args="--x.hello.float.g 0.1".split(" "), + )["hello"][ "float" ] == GenericColor(0.5, 0.1, 0.3) - assert tyro.cli(main, args="--x.hello.int.g 0".split(" "),) == { + assert tyro.cli( + main, + args="--x.hello.int.g 0".split(" "), + ) == { "hello": {"float": GenericColor(0.5, 0.2, 0.3), "int": GenericColor(25, 0, 3)} } @@ -368,6 +377,9 @@ def main( ) -> Any: return x - assert tyro.cli(main, args="--x.hello.a.g 1".split(" "),)["hello"][ + assert tyro.cli( + main, + args="--x.hello.a.g 1".split(" "), + )["hello"][ "a" ] == Color(5, 1, 3) diff --git a/tyro/_argparse_formatter.py b/tyro/_argparse_formatter.py index d5ed0ea3e..1e7f35aa2 100644 --- a/tyro/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -154,9 +154,9 @@ def _format_args(self, action, default_metavar): else str_from_rich( Text.from_ansi( out, - style=THEME.metavar_fixed - if out == "{fixed}" - else THEME.metavar, + style=( + THEME.metavar_fixed if out == "{fixed}" else THEME.metavar + ), ), soft_wrap=True, ) @@ -424,9 +424,11 @@ def _tyro_format_nonroot(self): ) .rstrip() .split("\n"), - item_parts + [description_part] - if description_part is not None - else item_parts, + ( + item_parts + [description_part] + if description_part is not None + else item_parts + ), ) ) ) diff --git a/tyro/_cli.py b/tyro/_cli.py index d76c1a9d3..ebdfdd27a 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -301,7 +301,11 @@ def fix_arg(arg: str) -> str: if print_completion: _arguments.USE_RICH = True - assert completion_shell in ("bash", "zsh", "tcsh",), ( + assert completion_shell in ( + "bash", + "zsh", + "tcsh", + ), ( "Shell should be one `bash`, `zsh`, or `tcsh`, but got" f" {completion_shell}" ) diff --git a/tyro/_fields.py b/tyro/_fields.py index 0fe1517c9..baf55e348 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -369,9 +369,23 @@ def _field_list_from_namedtuple( def _field_list_from_dataclass( cls: TypeForm[Any], default_instance: _DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: + # Check if dataclass is a flax module. + is_flax_module = False + try: + import flax + + if issubclass(cls, flax.linen.Module): + is_flax_module = True + except ImportError: + pass + # Handle dataclasses. field_list = [] for dc_field in filter(lambda field: field.init, _resolver.resolved_fields(cls)): + # For flax modules, we ignore the built-in "name" and "parent" fields. + if is_flax_module and dc_field.name in ("name", "parent"): + continue + default = _get_dataclass_field_default(dc_field, default_instance) # Try to get helptext from field metadata. This is also intended to be @@ -423,9 +437,9 @@ def _field_list_from_pydantic( FieldDefinition.make( name=pd_field.name, typ=pd_field.outer_type_, - default=MISSING_NONPROP - if pd_field.required - else pd_field.get_default(), + default=( + MISSING_NONPROP if pd_field.required else pd_field.get_default() + ), helptext=helptext, ) ) @@ -708,9 +722,11 @@ def _field_list_from_params( typ=hints[param.name], default=default, helptext=helptext, - markers=(_markers.Positional, _markers._PositionalCall) - if param.kind is inspect.Parameter.POSITIONAL_ONLY - else (), + markers=( + (_markers.Positional, _markers._PositionalCall) + if param.kind is inspect.Parameter.POSITIONAL_ONLY + else () + ), ) ) diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 08720160e..08aee6f38 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -205,9 +205,11 @@ def instantiator_base_case(strings: List[str]) -> Any: return instantiator_base_case, InstantiatorMetadata( nargs=1, - metavar=typ.__name__.upper() - if auto_choices is None - else "{" + ",".join(map(str, auto_choices)) + "}", + metavar=( + typ.__name__.upper() + if auto_choices is None + else "{" + ",".join(map(str, auto_choices)) + "}" + ), choices=auto_choices, ) diff --git a/tyro/_parsers.py b/tyro/_parsers.py index f017e9945..05351dec9 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -142,11 +142,13 @@ def from_callable_or_type( ), ) nested_parser = ParserSpecification.from_callable_or_type( - # Recursively apply marker types. - field.typ - if len(field.markers) == 0 - else Annotated.__class_getitem__( # type: ignore - (field.typ,) + tuple(field.markers) + ( + # Recursively apply marker types. + field.typ + if len(field.markers) == 0 + else Annotated.__class_getitem__( # type: ignore + (field.typ,) + tuple(field.markers) + ) ), description=None, parent_classes=parent_classes, @@ -439,10 +441,12 @@ def from_field( subcommand_config, default=field.default ) subparser = ParserSpecification.from_callable_or_type( - # Recursively apply markers. - Annotated.__class_getitem__((option,) + tuple(field.markers)) # type: ignore - if len(field.markers) > 0 - else option, + ( + # Recursively apply markers. + Annotated.__class_getitem__((option,) + tuple(field.markers)) # type: ignore + if len(field.markers) > 0 + else option + ), description=subcommand_config.description, parent_classes=parent_classes, default_instance=subcommand_config.default, From 9269b6f6aba972ba6088b1fb7c2abec8b268c03d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 12 Feb 2023 01:27:59 -0800 Subject: [PATCH 260/491] Deprecate --tyro-print-completion in favor of --tyro-write-completion --- docs/source/tab_completion.md | 12 ++--- tests/helptext_utils.py | 4 ++ ...print_completion.py => test_completion.py} | 0 tyro/_cli.py | 54 +++++++++++++------ 4 files changed, 47 insertions(+), 23 deletions(-) rename tests/{test_print_completion.py => test_completion.py} (100%) diff --git a/docs/source/tab_completion.md b/docs/source/tab_completion.md index 14153cf3c..7984d9a06 100644 --- a/docs/source/tab_completion.md +++ b/docs/source/tab_completion.md @@ -4,10 +4,10 @@ Interfaces built with :func:`tyro.cli()` can be tab completed in interactive shells without any source code modification. Completion scripts can be generated by passing the -`--tyro-print-completion {bash/zsh/tcsh}` flag to a tyro CLI. This generates -a completion script via [shtab](https://docs.iterative.ai/shtab/) and prints it -to stdout. To set up tab completion, the printed script simply needs to be -written somewhere where your shell will find it. +`--tyro-write-completion {bash/zsh/tcsh} PATH` flag to a tyro CLI. This +generates a completion script via [shtab](https://docs.iterative.ai/shtab/) and +prints it to stdout. To set up tab completion, the printed script simply needs +to be written somewhere where your shell will find it. ## Zsh @@ -23,7 +23,7 @@ mkdir -p ~/.zfunc # (2) Write completion script. The name here (_01_functions_py) doesn't matter, # as long as it's prefixed with an underscore. -python 01_functions.py --tyro-print-completion zsh > ~/.zfunc/_01_functions_py +python 01_functions.py --tyro-write-completion zsh ~/.zfunc/_01_functions_py ``` And if it's not in your `.zshrc` already: @@ -66,7 +66,7 @@ mkdir -p $completion_dir # (2) Write completion scripts. Note that the name of the completion script must # match the name of the file. -python 01_functions.py --tyro-print-completion bash > ${completion_dir}/01_functions.py +python 01_functions.py --tyro-write-completion bash ${completion_dir}/01_functions.py ``` In contrast to zsh, tab completion in bash requires that scripts are either set diff --git a/tests/helptext_utils.py b/tests/helptext_utils.py index 44b94e147..fa8e8e845 100644 --- a/tests/helptext_utils.py +++ b/tests/helptext_utils.py @@ -34,6 +34,10 @@ def get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: tyro.cli(f, args=["--tyro-print-completion", "bash"]) with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): tyro.cli(f, args=["--tyro-print-completion", "zsh"]) + with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): + tyro.cli(f, args=["--tyro-write-completion", "bash", os.devnull]) + with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): + tyro.cli(f, args=["--tyro-write-completion", "zsh", os.devnull]) # Check helptext with vs without formatting. This can help catch text wrapping bugs # caused by ANSI sequences. diff --git a/tests/test_print_completion.py b/tests/test_completion.py similarity index 100% rename from tests/test_print_completion.py rename to tests/test_completion.py diff --git a/tyro/_cli.py b/tyro/_cli.py index ebdfdd27a..6534fe05b 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -1,6 +1,7 @@ """Core public API.""" import argparse import dataclasses +import pathlib import sys import warnings from typing import Callable, Optional, Sequence, TypeVar, Union, cast, overload @@ -97,9 +98,9 @@ def cli( - Optional unions over nested structures (optional subparsers). - Generics (including nested generics). - Completion script generation for interactive shells is also provided. To print a - script that can be used for tab completion, pass in `--tyro-print-completion - {bash/zsh/tcsh}`. + Completion script generation for interactive shells is also provided. To write a + script that can be used for tab completion, pass in: + `--tyro-write-completion {bash/zsh/tcsh} {path to script to write}`. Args: f: Function or type. @@ -167,7 +168,7 @@ def get_parser( """Get the `argparse.ArgumentParser` object generated under-the-hood by `tyro.cli()`. Useful for tools like `sphinx-argparse`, `argcomplete`, etc. - For tab completion, we recommend using `tyro.cli()`'s built-in `--tyro-print-completion` + For tab completion, we recommend using `tyro.cli()`'s built-in `--tyro-write-completion` flag.""" return cast( argparse.ArgumentParser, @@ -254,20 +255,28 @@ def fix_arg(arg: str) -> str: args = list(map(fix_arg, args)) - # If we pass in the --tyro-print-completion flag: turn formatting tags, and get - # the shell we want to generate a completion script for (bash/zsh/tcsh). + # If we pass in the --tyro-print-completion or --tyro-write-completion flags: turn + # formatting tags, and get the shell we want to generate a completion script for + # (bash/zsh/tcsh). # - # Note that shtab also offers an add_argument_to() functions that fulfills a similar - # goal, but manual parsing of argv is convenient for turning off formatting. + # shtab also offers an add_argument_to() functions that fulfills a similar goal, but + # manual parsing of argv is convenient for turning off formatting. + # + # Note: --tyro-print-completion is deprecated! --tyro-write-completion is less prone + # to errors from accidental logging, print statements, etc. print_completion = len(args) >= 2 and args[0] == "--tyro-print-completion" + write_completion = len(args) >= 3 and args[0] == "--tyro-write-completion" # Note: setting USE_RICH must happen before the parser specification is generated. # TODO: revisit this. Ideally we should be able to eliminate the global state # changes. completion_shell = None - if print_completion: + completion_target_path = None + if print_completion or write_completion: completion_shell = args[1] - if print_completion or return_parser: + if write_completion: + completion_target_path = pathlib.Path(args[2]) + if print_completion or write_completion or return_parser: _arguments.USE_RICH = False else: _arguments.USE_RICH = True @@ -299,7 +308,7 @@ def fix_arg(arg: str) -> str: _arguments.USE_RICH = True return parser - if print_completion: + if print_completion or write_completion: _arguments.USE_RICH = True assert completion_shell in ( "bash", @@ -309,13 +318,24 @@ def fix_arg(arg: str) -> str: "Shell should be one `bash`, `zsh`, or `tcsh`, but got" f" {completion_shell}" ) - print( - shtab.complete( - parser=parser, - shell=completion_shell, - root_prefix=f"tyro_{parser.prog}", + + if write_completion: + assert completion_target_path is not None + completion_target_path.write_text( + shtab.complete( + parser=parser, + shell=completion_shell, + root_prefix=f"tyro_{parser.prog}", + ) + ) + else: + print( + shtab.complete( + parser=parser, + shell=completion_shell, + root_prefix=f"tyro_{parser.prog}", + ) ) - ) raise SystemExit() value_from_prefixed_field_name = vars(parser.parse_args(args=args)) From 1917cc494250aee2be806fe4d2d662352c8da2ea Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 15 Feb 2023 12:21:40 -0800 Subject: [PATCH 261/491] Add support for os.PathLike (#37) * Support os.PathLike * Fix missing import, add os.PathLike helptext test * Un-rename file to avoid merge conflict --- tests/test_dcargs.py | 8 ++++++++ tests/test_helptext.py | 9 +++++++++ tyro/_instantiators.py | 8 ++++++++ 3 files changed, 25 insertions(+) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 1e20975ed..c7f8e5369 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -2,6 +2,7 @@ import copy import dataclasses import enum +import os import pathlib from typing import ( Any, @@ -548,3 +549,10 @@ def main() -> argparse.ArgumentParser: return parser assert isinstance(tyro.cli(main, args=[]), argparse.ArgumentParser) + + +def test_pathlike(): + def main(x: os.PathLike) -> os.PathLike: + return x + + assert tyro.cli(main, args=["--x", "/dev/null"]) == pathlib.Path("/dev/null") diff --git a/tests/test_helptext.py b/tests/test_helptext.py index b8aa3e476..c229a3d35 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -1,5 +1,6 @@ import dataclasses import enum +import os import pathlib from collections.abc import Callable from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar, Union, cast @@ -562,3 +563,11 @@ def main2(x: Callable = nn.ReLU): assert "--x {fixed}" in helptext assert "(fixed to:" in helptext assert "torch" in helptext + + +def test_pathlike() -> None: + def main(x: os.PathLike) -> None: + pass + + helptext = get_helptext(main) + assert "--x PATH " in helptext diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 08aee6f38..43b4ef310 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -34,6 +34,8 @@ import dataclasses import enum import inspect +import os +import pathlib from collections import deque from typing import ( Any, @@ -123,6 +125,12 @@ def instantiator(strings: List[str]) -> None: choices=("None",), ) + # Instantiate os.PathLike annotations using pathlib.Path. + # Ideally this should be implemented in a more general way, eg using + # https://github.com/brentyi/tyro/pull/30 + if typ is os.PathLike: + typ = pathlib.Path + # Address container types. If a matching container is found, this will recursively # call instantiator_from_type(). container_out = _instantiator_from_container_type(typ, type_from_typevar) From 1ba5d5cfa24be4b2335fb7d084accdc95d539e28 Mon Sep 17 00:00:00 2001 From: Jesse Farebrother Date: Wed, 15 Feb 2023 17:33:24 -0500 Subject: [PATCH 262/491] Add `return_unknown_args` to `tyro.cli` (#35) --- pyproject.toml | 5 ++ tests/test_dcargs.py | 84 ++++++++++++++++++++++++++++ tyro/_cli.py | 130 ++++++++++++++++++++++++++++++++++--------- 3 files changed, 194 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4eef48541..f8ffbd68f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,3 +81,8 @@ exclude_lines = [ ignore = [ "E501", # Ignore line length errors. ] + +[tool.pytest.ini_options] +pythonpath = [ + "." +] diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index c7f8e5369..79f88e5b2 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -551,6 +551,90 @@ def main() -> argparse.ArgumentParser: assert isinstance(tyro.cli(main, args=[]), argparse.ArgumentParser) +def test_return_unknown_args() -> None: + @dataclasses.dataclass + class A: + x: int = 0 + + a, unknown_args = tyro.cli( + A, args=["positional", "--x", "5", "--y", "7"], return_unknown_args=True + ) + assert a == A(x=5) + assert unknown_args == ["positional", "--y", "7"] + + +def test_unknown_args_with_arg_fixing() -> None: + @dataclasses.dataclass + class A: + x: int = 0 + + a, unknown_args = tyro.cli( + A, + args=["--x", "5", "--a_b", "--a-c"], + return_unknown_args=True, + ) + assert a == A(x=5) + # Should return the unfixed arguments + assert unknown_args == ["--a_b", "--a-c"] + + +def test_allow_ambiguous_args_when_not_returning_unknown_args() -> None: + @dataclasses.dataclass + class A: + a_b: List[int] = dataclasses.field(default_factory=list) + + a = tyro.cli( + A, + args=["--a_b", "5", "--a-b", "7"], + ) + assert a == A(a_b=[7]) + + +def test_disallow_ambiguous_args_when_returning_unknown_args() -> None: + @dataclasses.dataclass + class A: + x: int = 0 + + # If there's an argument that's ambiguous then we should raise an error when we're + # returning unknown args. + with pytest.raises(RuntimeError, match="Ambiguous .* --a_b and --a-b"): + tyro.cli( + A, + args=["--x", "5", "--a_b", "--a-b"], + return_unknown_args=True, + ) + + +def test_unknown_args_with_consistent_duplicates() -> None: + @dataclasses.dataclass + class A: + a_b: List[int] = dataclasses.field(default_factory=list) + c_d: List[int] = dataclasses.field(default_factory=list) + + # Tests logic for consistent duplicate arguments when performing argument fixing. + # i.e., we can fix arguments if the separator is consistent (all _'s or all -'s). + a, unknown_args = tyro.cli( + A, + args=[ + "--a-b", + "5", + "--a-b", + "7", + "--c_d", + "5", + "--c_d", + "7", + "--e-f", + "--e-f", + "--g_h", + "--g_h", + ], + return_unknown_args=True, + ) + assert a == A(a_b=[7], c_d=[7]) + assert unknown_args == ["--e-f", "--e-f", "--g_h", "--g_h"] + + def test_pathlike(): def main(x: os.PathLike) -> os.PathLike: return x diff --git a/tyro/_cli.py b/tyro/_cli.py index 6534fe05b..dd7187e9b 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -4,9 +4,21 @@ import pathlib import sys import warnings -from typing import Callable, Optional, Sequence, TypeVar, Union, cast, overload +from typing import ( + Callable, + Dict, + List, + Optional, + Sequence, + Tuple, + TypeVar, + Union, + cast, + overload, +) import shtab +from typing_extensions import Literal from . import ( _argparse_formatter, @@ -36,10 +48,24 @@ def cli( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default: Optional[OutT] = None, + return_unknown_args: Literal[False] = False, ) -> OutT: ... +@overload +def cli( + f: TypeForm[OutT], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + default: Optional[OutT] = None, + return_unknown_args: Literal[True], +) -> Tuple[OutT, List[str]]: + ... + + @overload def cli( f: Callable[..., OutT], @@ -51,10 +77,27 @@ def cli( # supported for general callables. These can, however, be specified in the signature # of the callable itself. default: None = None, + return_unknown_args: Literal[False] = False, ) -> OutT: ... +@overload +def cli( + f: Callable[..., OutT], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + # Note that passing a default makes sense for things like dataclasses, but are not + # supported for general callables. These can, however, be specified in the signature + # of the callable itself. + default: None = None, + return_unknown_args: Literal[True], +) -> Tuple[OutT, List[str]]: + ... + + def cli( f: Union[TypeForm[OutT], Callable[..., OutT]], *, @@ -62,8 +105,9 @@ def cli( description: Optional[str] = None, args: Optional[Sequence[str]] = None, default: Optional[OutT] = None, + return_unknown_args: bool = False, **deprecated_kwargs, -) -> OutT: +) -> Union[OutT, Tuple[OutT, List[str]]]: """Call or instantiate `f`, with inputs populated from an automatically generated CLI interface. @@ -115,23 +159,29 @@ def cli( type like a dataclass or dictionary, but not if `f` is a general callable like a function or standard class. Helpful for merging CLI arguments with values loaded from elsewhere. (for example, a config object loaded from a yaml file) + return_unknown_args: If True, return a tuple of the output of `f` and a list of + unknown arguments. Mirrors the unknown arguments returned from + `argparse.ArgumentParser.parse_known_args()`. Returns: The output of `f(...)` or an instance `f`. If `f` is a class, the two are - equivalent. + equivalent. If `return_unknown_args` is True, returns a tuple of the output of + `f(...)` and a list of unknown arguments. """ - return cast( - OutT, - _cli_impl( - f, - prog=prog, - description=description, - args=args, - default=default, - return_parser=False, - **deprecated_kwargs, - ), + output = _cli_impl( + f, + prog=prog, + description=description, + args=args, + default=default, + return_parser=False, + return_unknown_args=return_unknown_args, + **deprecated_kwargs, ) + if return_unknown_args: + return cast(Tuple[OutT, List[str]], output) + else: + return cast(OutT, output) @overload @@ -179,6 +229,7 @@ def get_parser( args=None, default=default, return_parser=True, + return_unknown_args=False, ), ) @@ -191,8 +242,9 @@ def _cli_impl( args: Optional[Sequence[str]], default: Optional[OutT], return_parser: bool, + return_unknown_args: bool, **deprecated_kwargs, -) -> Union[OutT, argparse.ArgumentParser]: +) -> Union[OutT, argparse.ArgumentParser, Tuple[OutT, List[str]],]: """Helper for stitching the `tyro` pipeline together. Converts `f` into a @@ -242,18 +294,32 @@ def _cli_impl( # Read and fix arguments. If the user passes in --field_name instead of # --field-name, correct for them. - args = sys.argv[1:] if args is None else args - - def fix_arg(arg: str) -> str: + args = list(sys.argv[1:]) if args is None else list(args) + + # Fix arguments. This will modify all option-style arguments replacing + # underscores with dashes. This is to support the common convention of using + # underscores in variable names, but dashes in command line arguments. + # If two options are ambiguous, e.g., --a_b and --a-b, raise a runtime error. + modified_args: Dict[str, str] = {} + for index, arg in enumerate(args): if not arg.startswith("--"): - return arg + continue + if "=" in arg: arg, _, val = arg.partition("=") - return arg.replace("_", "-") + "=" + val + fixed = arg.replace("_", "-") + "=" + val else: - return arg.replace("_", "-") - - args = list(map(fix_arg, args)) + fixed = arg.replace("_", "-") + if ( + return_unknown_args + and fixed in modified_args + and modified_args[fixed] != arg + ): + raise RuntimeError( + f"Ambiguous arguments: " + modified_args[fixed] + " and " + arg + ) + modified_args[fixed] = arg + args[index] = fixed # If we pass in the --tyro-print-completion or --tyro-write-completion flags: turn # formatting tags, and get the shell we want to generate a completion script for @@ -338,7 +404,12 @@ def fix_arg(arg: str) -> str: ) raise SystemExit() - value_from_prefixed_field_name = vars(parser.parse_args(args=args)) + if return_unknown_args: + namespace, unknown_args = parser.parse_known_args(args=args) + else: + unknown_args = None + namespace = parser.parse_args(args=args) + value_from_prefixed_field_name = vars(namespace) if dummy_wrapped: value_from_prefixed_field_name = { @@ -369,4 +440,13 @@ def fix_arg(arg: str) -> str: if dummy_wrapped: out = getattr(out, _strings.dummy_field_name) - return out + + if return_unknown_args: + assert unknown_args is not None, "Should have parsed with `parse_known_args()`" + # If we're parsed unknown args, we should return the original args, not + # the fixed ones. + unknown_args = [modified_args.get(arg, arg) for arg in unknown_args] + return out, unknown_args + else: + assert unknown_args is None, "Should have parsed with `parse_args()`" + return out From 59f64bcd6747aa42510349502347fab0d1a2545e Mon Sep 17 00:00:00 2001 From: Jesse Farebrother Date: Fri, 17 Feb 2023 17:04:45 -0500 Subject: [PATCH 263/491] Add support for PathLike classes & objects with __new__ but no __init__ (#38) * Add support for PathLike classes & objects with __new__ but no __init__ * Update _instantiators.py * Formatting --------- Co-authored-by: Brent Yi --- tests/test_dcargs.py | 28 +++++++++++++++ tests/test_helptext.py | 2 +- tyro/_docstrings.py | 2 +- tyro/_fields.py | 25 ++++++++++--- tyro/_instantiators.py | 80 +++++++++++++++++++++++++----------------- 5 files changed, 98 insertions(+), 39 deletions(-) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 79f88e5b2..f11760ef3 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -551,6 +551,34 @@ def main() -> argparse.ArgumentParser: assert isinstance(tyro.cli(main, args=[]), argparse.ArgumentParser) +def test_pathlike_custom_class() -> None: + class CustomPath(pathlib.PurePosixPath): + def __new__(cls, *args: Union[str, os.PathLike]): + return super().__new__(cls, *args) + + def main(a: CustomPath) -> CustomPath: + return a + + assert tyro.cli(main, args=["--a", "/dev/null"]) == CustomPath("/dev/null") + + +def test_class_with_new_and_no_init() -> None: + class A(object): + def __new__(cls, x: int = 5): + return cls._custom_initializer(x) + + @classmethod + def _custom_initializer(cls, x: int = 5): + self = object.__new__(cls) + self.x = x # type: ignore + return self + + def __eq__(self, other) -> bool: + return self.x == other.x # type: ignore + + assert tyro.cli(A, args=["--x", "5"]) == A(x=5) + + def test_return_unknown_args() -> None: @dataclasses.dataclass class A: diff --git a/tests/test_helptext.py b/tests/test_helptext.py index c229a3d35..749541d4c 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -511,7 +511,7 @@ def main( def test_metavar_5() -> None: def main( - x: List[Union[Tuple[int, int], Tuple[str, str]],] = [(1, 1), (2, 2)] + x: List[Union[Tuple[int, int], Tuple[str, str]]] = [(1, 1), (2, 2)] ) -> None: pass diff --git a/tyro/_docstrings.py b/tyro/_docstrings.py index 32bafc6eb..4c7139d4c 100644 --- a/tyro/_docstrings.py +++ b/tyro/_docstrings.py @@ -306,7 +306,7 @@ def get_callable_description(f: Callable) -> str: docstring = f.__doc__ if ( docstring is None - and isinstance(f, type) + and inspect.isclass(f) # Ignore TypedDict's __init__ docstring, because it will just be `dict` and not is_typeddict(f) # Ignore NamedTuple __init__ docstring. diff --git a/tyro/_fields.py b/tyro/_fields.py index baf55e348..a7ea7ec81 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -9,6 +9,7 @@ import functools import inspect import itertools +import os import typing import warnings from typing import ( @@ -249,7 +250,7 @@ def _try_field_list_from_callable( default_instance = found_subcommand_configs[0].default # Unwrap generics. - f, type_from_typevar = _resolver.resolve_generic_types(f) + f, _ = _resolver.resolve_generic_types(f) f = _resolver.narrow_type(f, default_instance) f_origin = _resolver.unwrap_origin_strip_extras(cast(TypeForm, f)) @@ -257,9 +258,17 @@ def _try_field_list_from_callable( # 1. Set cls to the type. # 2. Consider `f` to be `cls.__init__`. cls: Optional[TypeForm[Any]] = None - if isinstance(f, type): + if inspect.isclass(f): cls = f - f = cls.__init__ # type: ignore + if hasattr(cls, "__init__") and cls.__init__ is not object.__init__: + f = cls.__init__ # type: ignore + elif hasattr(cls, "__new__") and cls.__new__ is not object.__new__: + f = cls.__new__ + else: + return UnsupportedNestedTypeMessage( + f"Cannot instantiate class {cls} with no unique __init__ or __new__" + " method." + ) f_origin = cls # type: ignore # Try field generation from class inputs. @@ -299,6 +308,14 @@ def _try_field_list_from_callable( cls is not None and cls in _known_parsable_types ) or _resolver.unwrap_origin_strip_extras(f) in _known_parsable_types: return UnsupportedNestedTypeMessage(f"{f} should be parsed directly!") + elif ( + cls is not None + and issubclass(cls, os.PathLike) + and _instantiators.is_type_string_converter(cls) + ): + return UnsupportedNestedTypeMessage( + f"PathLike {cls} should be parsed directly!" + ) else: return _try_field_list_from_general_callable(f, cls, default_instance) @@ -675,7 +692,7 @@ def _field_list_from_params( done = False # Sometime functools.* is applied to a class. - if isinstance(f, type): + if inspect.isclass(f): cls = f f = f.__init__ # type: ignore diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 43b4ef310..8ad012671 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -52,7 +52,14 @@ overload, ) -from typing_extensions import Annotated, Final, Literal, get_args, get_origin +from typing_extensions import ( + Annotated, + Final, + Literal, + get_args, + get_origin, + get_type_hints, +) from . import _strings from ._typing import TypeForm @@ -95,6 +102,39 @@ class UnsupportedTypeAnnotationError(Exception): ) +def is_type_string_converter(typ: Union[Callable, TypeForm[Any]]) -> bool: + """Check if type is a string converter, i.e., (arg: Union[str, Any]) -> T.""" + param_count = 0 + has_var_positional = False + try: + signature = inspect.signature(typ) + except ValueError: + # pybind objects might not have a parsable signature. We try to be tolerant in this case. + # One day this should be fixed with `__text_signature__`. + return True + + type_annotations = get_type_hints(typ) + # Some checks we can do if the signature is available! + for i, param in enumerate(signature.parameters.values()): + annotation = type_annotations.get(param.name, param.annotation) + if i == 0 and not ( + (get_origin(annotation) is Union and str in get_args(annotation)) + or annotation in (str, inspect.Parameter.empty) + ): + return False + if param.kind is inspect.Parameter.VAR_POSITIONAL: + has_var_positional = True + elif param.default is inspect.Parameter.empty and param.kind is not ( + inspect.Parameter.VAR_KEYWORD + ): + param_count += 1 + + # Raise an error if parameters look wrong. + if not (param_count == 1 or (param_count == 0 and has_var_positional)): + return False + return True + + def instantiator_from_type( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] ) -> Tuple[Instantiator, InstantiatorMetadata]: @@ -145,43 +185,17 @@ def instantiator(strings: List[str]) -> None: f"Expected {typ} to be an `(arg: str) -> T` type converter, but is not" " callable." ) - else: - param_count = 0 - has_var_positional = False - try: - signature = inspect.signature(typ) - except ValueError: - # No signature, this is often the case with pybind, etc. - signature = None - - if signature is not None: - # Some checks we can do if the signature is available! - for i, param in enumerate(signature.parameters.values()): - if i == 0 and param.annotation not in (str, inspect.Parameter.empty): - raise UnsupportedTypeAnnotationError( - f"Expected {typ} to be an `(arg: str) -> T` type converter, but" - f" got {signature}." - ) - if param.kind is inspect.Parameter.VAR_POSITIONAL: - has_var_positional = True - elif param.default is inspect.Parameter.empty and param.kind is not ( - inspect.Parameter.VAR_KEYWORD - ): - param_count += 1 - - # Raise an error if parameters look wrong. - if not (param_count == 1 or (param_count == 0 and has_var_positional)): - raise UnsupportedTypeAnnotationError( - f"Expected {typ} to be an `(arg: str) -> T` type converter, but got" - f" {signature}. You may have a nested type in a container, which is" - " unsupported." - ) + elif not is_type_string_converter(typ): + raise UnsupportedTypeAnnotationError( + f"Expected {typ} to be an `(arg: str) -> T` type converter, but is not" + " a valid type converter." + ) # Special case `choices` for some types, as implemented in `instance_from_string()`. auto_choices: Optional[Tuple[str, ...]] = None if typ is bool: auto_choices = ("True", "False") - elif isinstance(typ, type) and issubclass(typ, enum.Enum): + elif inspect.isclass(typ) and issubclass(typ, enum.Enum): auto_choices = tuple(x.name for x in typ) def instantiator_base_case(strings: List[str]) -> Any: From b3c9871ee4134ec916fbb87e9d67a0c86589979b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 20 Feb 2023 14:56:44 -0800 Subject: [PATCH 264/491] Support variadic arguments (#39) --- tests/test_dcargs.py | 9 +++++++++ tyro/_calling.py | 19 ++++++++++++++----- tyro/_fields.py | 31 +++++++++++++++++++++++++------ tyro/conf/_markers.py | 4 ++++ 4 files changed, 52 insertions(+), 11 deletions(-) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index f11760ef3..c62a60998 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -668,3 +668,12 @@ def main(x: os.PathLike) -> os.PathLike: return x assert tyro.cli(main, args=["--x", "/dev/null"]) == pathlib.Path("/dev/null") + + +def test_variadics() -> None: + def main(*args: int, **kwargs: float) -> Tuple[Tuple[int, ...], Dict[str, float]]: + return args, kwargs + + assert tyro.cli( + main, args="--args 1 2 3 --kwargs learning_rate 1e-4 beta1 0.99".split(" ") + ) == ((1, 2, 3), {"learning_rate": 1e-4, "beta1": 0.99}) diff --git a/tyro/_calling.py b/tyro/_calling.py index 68ebbb7e0..ec6b38ac1 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -8,6 +8,7 @@ from typing_extensions import get_args from . import _arguments, _fields, _parsers, _resolver, _strings +from .conf import _markers class InstantiationError(Exception): @@ -160,11 +161,19 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: ) consumed_keywords |= consumed_keywords_child - if value is not _fields.EXCLUDE_FROM_CALL: - if field.is_positional_call(): - positional_args.append(value) - else: - kwargs[field.call_argname] = value + if value is _fields.EXCLUDE_FROM_CALL: + continue + + if _markers._UnpackArgsCall in field.markers: + assert isinstance(value, tuple) + positional_args.extend(value) + elif _markers._UnpackKwargsCall in field.markers: + assert isinstance(value, dict) + kwargs.update(value) + elif field.is_positional_call(): + positional_args.append(value) + else: + kwargs[field.call_argname] = value # Note: we unwrap types both before and after narrowing. This is because narrowing # sometimes produces types like `Tuple[T1, T2, ...]`, where we actually want just diff --git a/tyro/_fields.py b/tyro/_fields.py index a7ea7ec81..c13eec4c7 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -732,18 +732,37 @@ def _field_list_from_params( raise _instantiators.UnsupportedTypeAnnotationError(out.message) return out + # Set markers for positional + variadic arguments. + markers = () + typ = hints[param.name] + if param.kind is inspect.Parameter.POSITIONAL_ONLY: + markers = (_markers.Positional, _markers._PositionalCall) + elif param.kind is inspect.Parameter.VAR_POSITIONAL: + # Handle *args signatures. + # + # This will create a `--args T [T ...]` CLI argument. + markers = (_markers._UnpackArgsCall,) + typ = Tuple[typ, ...] + elif param.kind is inspect.Parameter.VAR_KEYWORD: + # Handle *kwargs signatures. + # + # This will create a `--kwargs STR T [STR T ...]` CLI argument. + # + # Note that it would be straightforward to make both this and *args truly + # positional, omitting the --args/--kwargs prefix, but we are choosing not + # to because it would make *args and **kwargs difficult to use in + # conjunction. + markers = (_markers._UnpackKwargsCall,) + typ = Dict[str, typ] + field_list.append( FieldDefinition.make( name=param.name, # Note that param.annotation doesn't resolve forward references. - typ=hints[param.name], + typ=typ, default=default, helptext=helptext, - markers=( - (_markers.Positional, _markers._PositionalCall) - if param.kind is inspect.Parameter.POSITIONAL_ONLY - else () - ), + markers=markers, ) ) diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index 43a2b88e3..df0a4d57a 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -22,6 +22,10 @@ # the callable. _PositionalCall = Annotated[T, None] +# Private markers for when arguments should be passed in via *args or **kwargs. +_UnpackArgsCall = Annotated[T, None] +_UnpackKwargsCall = Annotated[T, None] + # TODO: the verb tenses here are inconsistent, naming could be revisited. # Perhaps Suppress should be Suppressed? But SuppressedFixed would be weird. From 7fdc4323aa97a72212ab13c0695916c76014502e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 20 Feb 2023 14:59:28 -0800 Subject: [PATCH 265/491] Smarter matching for default subcommands (#33) * Start subcommand matching refactor * Improve test coverage * Format * Patch for 3.7 --- tests/test_conf.py | 11 ++-- tyro/_fields.py | 28 +++++----- tyro/_parsers.py | 57 ++++--------------- tyro/_subcommand_matching.py | 104 +++++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 67 deletions(-) create mode 100644 tyro/_subcommand_matching.py diff --git a/tests/test_conf.py b/tests/test_conf.py index 285187770..6d79e1710 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -321,15 +321,14 @@ class B: a: A = A(5) default_one = B(3) - default_two = B(9) + default_three = B(9) @dataclasses.dataclass class Nested: subcommand: Union[ - # Annotated[A, tyro.conf.subcommand("zero")], Annotated[B, tyro.conf.subcommand("one", default=default_one)], - Annotated[B, tyro.conf.subcommand("two", default=default_two)], - Annotated[B, tyro.conf.subcommand("three")], + Annotated[B, tyro.conf.subcommand("two")], + Annotated[B, tyro.conf.subcommand("three", default=default_three)], ] # Match by hash. @@ -342,13 +341,13 @@ def main_one(x: Nested = Nested(default_one)) -> None: def main_two(x: Nested = Nested(B(9))) -> None: pass - assert "default: x.subcommand:two" in get_helptext(main_two) + assert "default: x.subcommand:three" in get_helptext(main_two) # Match by type. def main_three(x: Nested = Nested(B(15))) -> None: pass - assert "default: x.subcommand:three" in get_helptext(main_three) + assert "default: x.subcommand:one" in get_helptext(main_three) def test_flag() -> None: diff --git a/tyro/_fields.py b/tyro/_fields.py index c13eec4c7..f74f55f52 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -163,7 +163,7 @@ class UnsupportedNestedTypeMessage: message: str -def is_nested_type(typ: TypeForm[Any], default_instance: _DefaultInstance) -> bool: +def is_nested_type(typ: TypeForm[Any], default_instance: DefaultInstance) -> bool: """Determine whether a type should be treated as a 'nested type', where a single type can be broken down into multiple fields (eg for nested dataclasses or classes). @@ -178,7 +178,7 @@ def is_nested_type(typ: TypeForm[Any], default_instance: _DefaultInstance) -> bo def field_list_from_callable( f: Union[Callable, TypeForm[Any]], - default_instance: _DefaultInstance, + default_instance: DefaultInstance, ) -> Tuple[ Union[Callable, TypeForm[Any]], Dict[TypeVar, TypeForm], List[FieldDefinition] ]: @@ -222,7 +222,7 @@ def resolve(field: FieldDefinition) -> FieldDefinition: # Implementation details below. -_DefaultInstance = Union[ +DefaultInstance = Union[ Any, PropagatingMissingType, NonpropagatingMissingType, ExcludeFromCallType ] @@ -241,7 +241,7 @@ def resolve(field: FieldDefinition) -> FieldDefinition: def _try_field_list_from_callable( f: Union[Callable, TypeForm[Any]], - default_instance: _DefaultInstance, + default_instance: DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: f, found_subcommand_configs = _resolver.unwrap_annotated( f, conf._confstruct._SubcommandConfiguration @@ -321,7 +321,7 @@ def _try_field_list_from_callable( def _field_list_from_typeddict( - cls: TypeForm[Any], default_instance: _DefaultInstance + cls: TypeForm[Any], default_instance: DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: field_list = [] valid_default_instance = ( @@ -353,7 +353,7 @@ def _field_list_from_typeddict( def _field_list_from_namedtuple( - cls: TypeForm[Any], default_instance: _DefaultInstance + cls: TypeForm[Any], default_instance: DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Handle NamedTuples. # @@ -384,7 +384,7 @@ def _field_list_from_namedtuple( def _field_list_from_dataclass( - cls: TypeForm[Any], default_instance: _DefaultInstance + cls: TypeForm[Any], default_instance: DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Check if dataclass is a flax module. is_flax_module = False @@ -439,7 +439,7 @@ def _is_pydantic(cls: TypeForm[Any]) -> bool: def _field_list_from_pydantic( - cls: TypeForm[Any], default_instance: _DefaultInstance + cls: TypeForm[Any], default_instance: DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: assert pydantic is not None @@ -474,7 +474,7 @@ def _is_attrs(cls: TypeForm[Any]) -> bool: def _field_list_from_attrs( - cls: TypeForm[Any], default_instance: _DefaultInstance + cls: TypeForm[Any], default_instance: DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: assert attr is not None @@ -501,7 +501,7 @@ def _field_list_from_attrs( def _field_list_from_tuple( - f: Union[Callable, TypeForm[Any]], default_instance: _DefaultInstance + f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Fixed-length tuples. field_list = [] @@ -561,7 +561,7 @@ def _field_list_from_tuple( def _field_list_from_sequence_checked( - f: Union[Callable, TypeForm[Any]], default_instance: _DefaultInstance + f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: contained_type: Any if len(get_args(f)) == 0: @@ -579,7 +579,7 @@ def _field_list_from_sequence_checked( def _try_field_list_from_sequence_inner( contained_type: TypeForm[Any], - default_instance: _DefaultInstance, + default_instance: DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # When no default instance is specified: # If we have List[int] => this can be parsed as a single field. @@ -623,7 +623,7 @@ def _try_field_list_from_sequence_inner( def _field_list_from_dict( f: Union[Callable, TypeForm[Any]], - default_instance: _DefaultInstance, + default_instance: DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: if default_instance in MISSING_SINGLETONS: return UnsupportedNestedTypeMessage( @@ -647,7 +647,7 @@ def _field_list_from_dict( def _try_field_list_from_general_callable( f: Union[Callable, TypeForm[Any]], cls: Optional[TypeForm[Any]], - default_instance: _DefaultInstance, + default_instance: DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Handle general callables. if default_instance not in MISSING_SINGLETONS: diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 05351dec9..42bdb448a 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -28,6 +28,7 @@ _instantiators, _resolver, _strings, + _subcommand_matching, ) from ._typing import TypeForm from .conf import _confstruct, _markers @@ -346,8 +347,7 @@ def from_field( subcommand_config_from_name: Dict[ str, _confstruct._SubcommandConfiguration ] = {} - subcommand_name_from_default_hash: Dict[int, str] = {} - subcommand_name_from_type: Dict[Type, str] = {} # Used for default matching. + subcommand_type_from_name: Dict[str, type] = {} for option in options_no_none: subcommand_name = _strings.subparser_name_from_type(prefix, option) option, found_subcommand_configs = _resolver.unwrap_annotated( @@ -363,52 +363,15 @@ def from_field( subcommand_config_from_name[subcommand_name] = found_subcommand_configs[ 0 ] + subcommand_type_from_name[subcommand_name] = option - if ( - found_subcommand_configs[0].default - not in _fields.MISSING_SINGLETONS - ): - default_hash = object.__hash__(found_subcommand_configs[0].default) - subcommand_name_from_default_hash[default_hash] = subcommand_name - - # Use subcommand types for default matching if no default is explicitly - # annotated. - if default_hash is None: - subcommand_name_from_type[option] = subcommand_name - - # If there are any required arguments in the default subparser, we should mark - # the subparser group as a whole as required. - default_name = None - if ( - field.default is not None - and field.default not in _fields.MISSING_SINGLETONS - ): - # It's really hard to concretize a generic type at runtime, so we just... - # don't. :-) - if hasattr(type(field.default), "__parameters__"): - raise _instantiators.UnsupportedTypeAnnotationError( - "Default values for generic subparsers are not supported." - ) - - # Get default subcommand name: by default hash. - default_hash = object.__hash__(field.default) - default_name = subcommand_name_from_default_hash.get(default_hash, None) - - # Get default subcommand name: by default value. - if default_name is None: - for ( - subcommand_name, - subcommand_config, - ) in subcommand_config_from_name.items(): - equal = field.default == subcommand_config.default - if isinstance(equal, bool) and equal: - default_name = subcommand_name - break - - # Get default subcommand name: by default type. - if default_name is None: - default_name = subcommand_name_from_type.get(type(field.default), None) - + # If a field default is provided, try to find a matching subcommand name. + if field.default is None or field.default in _fields.MISSING_SINGLETONS: + default_name = None + else: + default_name = _subcommand_matching.match_subcommand( + field.default, subcommand_config_from_name, subcommand_type_from_name + ) if default_name is None: raise AssertionError( f"`{prefix}` was provided a default value of type" diff --git a/tyro/_subcommand_matching.py b/tyro/_subcommand_matching.py new file mode 100644 index 000000000..5026a1c5b --- /dev/null +++ b/tyro/_subcommand_matching.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import dataclasses +from typing import Any, Callable, Dict, Optional, Tuple, Union + +from typing_extensions import get_args, get_origin + +from . import _fields, _instantiators, _resolver, _typing +from .conf import _confstruct + + +def match_subcommand( + default: Any, + subcommand_config_from_name: Dict[str, _confstruct._SubcommandConfiguration], + subcommand_type_from_name: Dict[str, type], +) -> Optional[str]: + # It's really hard to concretize a generic type at runtime, so we just... + # don't. :-) + if hasattr(type(default), "__parameters__"): + raise _instantiators.UnsupportedTypeAnnotationError( + "Default values for generic subparsers are not supported." + ) + + # Get default subcommand name: by default hash. + default_hash = object.__hash__(default) + for subcommand_name, conf in subcommand_config_from_name.items(): + if conf.default in _fields.MISSING_SINGLETONS: + continue + if default_hash == object.__hash__(conf.default): + return subcommand_name + + # Get default subcommand name: by default value. + for subcommand_name, conf in subcommand_config_from_name.items(): + if conf.default in _fields.MISSING_SINGLETONS: + continue + equal = default == conf.default + if isinstance(equal, bool) and equal: + return subcommand_name + + # Get default subcommand name: by concrete type tree. + typetree = _TypeTree.make(type(default), default) + for subcommand_name, conf in subcommand_config_from_name.items(): + if conf.default in _fields.MISSING_SINGLETONS: + continue + if typetree == _TypeTree.make(type(conf.default), conf.default): + return subcommand_name + + # Get default subcommand name: by annotated type tree. + for subcommand_name, subcommand_type in subcommand_type_from_name.items(): + if typetree.is_subtype_of( + _TypeTree.make(subcommand_type, _fields.MISSING_NONPROP) + ): + return subcommand_name + + # Failed! + return None + + +@dataclasses.dataclass(frozen=True) +class _TypeTree: + typ: Any + children: Dict[str, _TypeTree] + + @staticmethod + def make( + typ: Union[Callable, _typing.TypeForm], + default_instance: _fields.DefaultInstance, + ) -> _TypeTree: + """From an object instance, return a data structure representing the types in the object.""" + try: + typ, _type_from_typevar, field_list = _fields.field_list_from_callable( + typ, default_instance=default_instance + ) + except _instantiators.UnsupportedTypeAnnotationError: + return _TypeTree(typ, {}) + + return _TypeTree( + typ, + { + field.name: _TypeTree.make(field.typ, field.default) + for field in field_list + }, + ) + + def is_subtype_of(self, supertype: _TypeTree) -> bool: + # Generalize to unions. + def _get_type_options(typ: _typing.TypeForm) -> Tuple[_typing.TypeForm, ...]: + return get_args(typ) if get_origin(typ) is Union else (typ,) + + self_types = _get_type_options(self.typ) + super_types = _get_type_options(supertype.typ) + + # Check against supertypes. + for self_type in self_types: + self_type = _resolver.unwrap_annotated(self_type)[0] + ok = False + for super_type in super_types: + super_type = _resolver.unwrap_annotated(super_type)[0] + if issubclass(self_type, super_type): + ok = True + if not ok: + return False + + return True From c81478f0c9c9fcbceff2e1571bdf668faf6f8113 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 20 Feb 2023 14:59:51 -0800 Subject: [PATCH 266/491] Drop Python version for codecov --- .github/workflows/coverage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index be421f157..e6eda0614 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python 3.10 + - name: Set up Python 3.8 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.8" - name: Install dependencies run: | curl -sSL https://install.python-poetry.org | python3 - From 847bdeffad94daa1abc3c8e6816535cce3b2eb78 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 20 Feb 2023 15:01:26 -0800 Subject: [PATCH 267/491] Version bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f8ffbd68f..603854b7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.3.38" +version = "0.4.0" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] From 47a211aaa0649f58892934522ac2e0649453bb58 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 20 Feb 2023 15:05:51 -0800 Subject: [PATCH 268/491] Fix mypy --- tyro/_fields.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tyro/_fields.py b/tyro/_fields.py index f74f55f52..f327c4386 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -733,8 +733,8 @@ def _field_list_from_params( return out # Set markers for positional + variadic arguments. - markers = () - typ = hints[param.name] + markers: Tuple[Any, ...] = () + typ: Any = hints[param.name] if param.kind is inspect.Parameter.POSITIONAL_ONLY: markers = (_markers.Positional, _markers._PositionalCall) elif param.kind is inspect.Parameter.VAR_POSITIONAL: @@ -742,7 +742,7 @@ def _field_list_from_params( # # This will create a `--args T [T ...]` CLI argument. markers = (_markers._UnpackArgsCall,) - typ = Tuple[typ, ...] + typ = Tuple.__getitem__((typ, ...)) # type: ignore elif param.kind is inspect.Parameter.VAR_KEYWORD: # Handle *kwargs signatures. # @@ -753,7 +753,7 @@ def _field_list_from_params( # to because it would make *args and **kwargs difficult to use in # conjunction. markers = (_markers._UnpackKwargsCall,) - typ = Dict[str, typ] + typ = Dict.__getitem__((str, typ)) # type: ignore field_list.append( FieldDefinition.make( From 5105811160079e34a5552fa653c897b751809be3 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 10 Mar 2023 13:08:11 -0800 Subject: [PATCH 269/491] Add test for configure() + subcommand(), closes #40 --- tests/test_conf.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_conf.py b/tests/test_conf.py index 6d79e1710..2b75bc905 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -622,6 +622,7 @@ class DefaultInstanceSubparser: def test_omit_subcommand_prefix_and_consolidate_subcommand_args_in_function() -> None: + @tyro.conf.configure(tyro.conf.subcommand(name="http-server")) @dataclasses.dataclass class DefaultInstanceHTTPServer: y: int = 0 @@ -647,7 +648,7 @@ def func(parent: DefaultInstanceSubparser) -> DefaultInstanceSubparser: assert tyro.cli( func, args=[ - "parent.bc:default-instance-http-server", + "parent.bc:http-server", "--parent.x", "1", # --y and --no-flag are in a subcommand with prefix omission. @@ -659,7 +660,7 @@ def func(parent: DefaultInstanceSubparser) -> DefaultInstanceSubparser: assert tyro.cli( func, args=[ - "parent.bc:default-instance-http-server", + "parent.bc:http-server", "--parent.x", "1", # --y is in a subcommand with prefix omission. From 7d137572777141227d4660400007dd5c8326ce39 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 10 Mar 2023 23:37:56 -0800 Subject: [PATCH 270/491] Be more tolerant of static type errors as per #20 --- tests/test_dcargs.py | 1 - tests/test_nested.py | 10 +++++++--- tyro/_parsers.py | 9 +++++++-- tyro/_resolver.py | 36 ++++++++++++++++++++++++++++++++++-- 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index c62a60998..3f6cc9d12 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -458,7 +458,6 @@ def add(a: SomeTypeAlias, b: SomeTypeAlias) -> SomeTypeAlias: assert tyro.cli(add, args=["--a", "5", "--b", "7"]) == 12 -@pytest.mark.filterwarnings("ignore::Warning") def test_any() -> None: def main(x: Any = 5) -> Any: return x diff --git a/tests/test_nested.py b/tests/test_nested.py index 53ed493f3..3b97e164a 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -335,9 +335,13 @@ class DefaultSubparser: default_factory=lambda: 5 # type: ignore ) - with pytest.raises(AssertionError): - tyro.cli( - DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "5"] + # Tolerate bad static types: https://github.com/brentyi/tyro/issues/20 + # Should give us a bunch of warnings! + with pytest.warns(UserWarning): + assert tyro.cli( + DefaultSubparser, args=["--x", "1", "--bc", "3"] + ) == DefaultSubparser( + 1, 3 # type: ignore ) diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 42bdb448a..c91e742dc 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -4,6 +4,7 @@ import argparse import dataclasses +import warnings from typing import ( Any, Callable, @@ -373,12 +374,16 @@ def from_field( field.default, subcommand_config_from_name, subcommand_type_from_name ) if default_name is None: - raise AssertionError( + # This should really be an error, but we can raise a warning to make + # hacking at subcommands easier: + # https://github.com/brentyi/tyro/issues/20 + warnings.warn( f"`{prefix}` was provided a default value of type" f" {type(field.default)} but no matching subcommand was found. A" " type may be missing in the Union type declaration for" - f" `{prefix}`, which is currently set to {field.typ}." + f" `{prefix}`, which currently expects {options_no_none}." ) + return None # Add subcommands for each option. parser_from_name: Dict[str, ParserSpecification] = {} diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 506d36d49..278573a40 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -1,9 +1,11 @@ """Utilities for resolving types and forward references.""" + import collections.abc import copy import dataclasses import sys import types +import warnings from typing import ( Any, Callable, @@ -243,8 +245,21 @@ def apply_type_from_typevar( def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCallable: - """Narrow union types. This is a shim for failing more gracefully when we we're - given an unsupported union. + """Narrow union types. + + This is a shim for failing more gracefully when we we're given one of two errors: + (A) A Union type that doesn't match the default value. + (B) An unsupported Union type, which mixes "nested" types (like dataclasses) with + non-"nested" types (like strings). + + -- + For (A): + + We raise a warning, then take the type of the default value. + Loosely motivated by: https://github.com/brentyi/tyro/issues/20 + + -- + For (B): When do we want to narrow Union types? @@ -275,7 +290,24 @@ def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCalla likely require a big code refactor.""" if get_origin(typ) is not Union: return typ + options = get_args(typ) + options_unwrapped = [unwrap_origin_strip_extras(o) for o in options] + + # (A) + try: + if default_instance not in _fields.MISSING_SINGLETONS and not any( + isinstance(default_instance, o) for o in options_unwrapped + ): + warnings.warn( + f"{type(default_instance)} does not match any type in Union:" + f" {options_unwrapped}" + ) + return type(default_instance) + except TypeError: + pass + + # (B) is_nested = tuple( map( lambda option: _fields.is_nested_type( From d994bb004f801b0c55466b22c2a95deb3198d927 Mon Sep 17 00:00:00 2001 From: Gaoyang Zhang Date: Sat, 11 Mar 2023 15:44:37 +0800 Subject: [PATCH 271/491] Fix `%` character as default str argument in dataclass (#41) Previously, if a `str` type argument's default value contains `%`, the `--help` action fails. Reproducible example: ```bash $ cat >percent-char.py < --- tests/test_helptext.py | 3 +++ tyro/_argparse_formatter.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 749541d4c..506e287bd 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -188,6 +188,7 @@ class Color(enum.Enum): class HelptextWithVariousDefaults: x: pathlib.Path = pathlib.Path("/some/path/to/a/file") y: Color = Color.RED + z: str = "%" helptext = get_helptext(HelptextWithVariousDefaults) assert "show this help message and exit" in helptext @@ -195,6 +196,8 @@ class HelptextWithVariousDefaults: assert "(default: /some/path/to/a/file)" in helptext assert "--y {RED,GREEN,BLUE}" in helptext assert "(default: RED)" in helptext + assert "--z STR" in helptext + assert "(default: %)" in helptext def test_multiline_helptext() -> None: diff --git a/tyro/_argparse_formatter.py b/tyro/_argparse_formatter.py index 1e7f35aa2..035e7b4fb 100644 --- a/tyro/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -380,7 +380,6 @@ def _tyro_format_nonroot(self): description_part = None item_parts = [] for func, args in self.items: - item_content = func(*args) if ( getattr(func, "__func__", None) is TyroArgparseHelpFormatter._format_action @@ -390,6 +389,7 @@ def _tyro_format_nonroot(self): item_parts.extend(self._format_action(action)) else: + item_content = func(*args) assert isinstance(item_content, str) if item_content.strip() != "": assert ( From 52d997534e1480bf637642f2339dacfd052b2f4f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 10 Mar 2023 23:54:47 -0800 Subject: [PATCH 272/491] Extra test for subcommands with bad defaults --- tests/test_nested.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_nested.py b/tests/test_nested.py index 3b97e164a..ab0b6b9fd 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -345,6 +345,24 @@ class DefaultSubparser: ) +def test_subparser_with_default_bad_alt() -> None: + @dataclasses.dataclass + class A: + a: int + + @tyro.conf.configure(tyro.conf.subcommand(name="bbbb")) + @dataclasses.dataclass + class B: + b: int + + @dataclasses.dataclass + class C: + c: int + + with pytest.warns(UserWarning): + assert tyro.cli(Union[A, B], default=C(3), args=["--c", "2"]) == C(2) + + def test_optional_subparser() -> None: @dataclasses.dataclass class OptionalHTTPServer: From 44e837bf499c569530497dfe8c95870d47f3eac2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 11 Mar 2023 20:20:36 -0800 Subject: [PATCH 273/491] Fix mypy, more aggressive caching --- tests/test_nested.py | 2 +- tyro/_docstrings.py | 13 +++++-------- tyro/_fields.py | 10 +++++++++- tyro/_parsers.py | 1 + tyro/_resolver.py | 5 ++++- tyro/_subcommand_matching.py | 5 ++++- tyro/_unsafe_cache.py | 37 ++++++++++++++++++++++++++++++++++++ 7 files changed, 61 insertions(+), 12 deletions(-) create mode 100644 tyro/_unsafe_cache.py diff --git a/tests/test_nested.py b/tests/test_nested.py index ab0b6b9fd..66cd6dacd 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -360,7 +360,7 @@ class C: c: int with pytest.warns(UserWarning): - assert tyro.cli(Union[A, B], default=C(3), args=["--c", "2"]) == C(2) + assert tyro.cli(Union[A, B], default=C(3), args=["--c", "2"]) == C(2) # type: ignore def test_optional_subparser() -> None: diff --git a/tyro/_docstrings.py b/tyro/_docstrings.py index 4c7139d4c..ac172c7be 100644 --- a/tyro/_docstrings.py +++ b/tyro/_docstrings.py @@ -22,13 +22,10 @@ import docstring_parser from typing_extensions import get_origin, is_typeddict -from . import _resolver, _strings +from . import _resolver, _strings, _unsafe_cache T = TypeVar("T", bound=Callable) -# Cast for making types more lenient. -_cache = cast(Callable[[int], Callable[[T], T]], functools.lru_cache) - @dataclasses.dataclass(frozen=True) class _Token: @@ -54,7 +51,7 @@ class _ClassTokenization: field_data_from_name: Dict[str, _FieldData] @staticmethod - @_cache(64) + @_unsafe_cache.unsafe_cache(64) def make(clz) -> "_ClassTokenization": """Parse the source code of a class, and cache some tokenization information.""" readline = io.BytesIO(inspect.getsource(clz).encode("utf-8")).readline @@ -124,7 +121,7 @@ def make(clz) -> "_ClassTokenization": ) -@_cache(256) +@_unsafe_cache.unsafe_cache(1024) def get_class_tokenization_with_field( cls: Type, field_name: str ) -> Optional[_ClassTokenization]: @@ -167,7 +164,7 @@ def get_class_tokenization_with_field( return tokenization -@_cache(256) +@_unsafe_cache.unsafe_cache(1024) def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: """Get docstring for a field in a class.""" @@ -279,7 +276,7 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: ) -@_cache(256) +@_unsafe_cache.unsafe_cache(1024) def get_callable_description(f: Callable) -> str: """Get description associated with a callable via docstring parsing. diff --git a/tyro/_fields.py b/tyro/_fields.py index f327c4386..6366f45c1 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -32,7 +32,14 @@ from typing_extensions import get_args, get_type_hints, is_typeddict from . import conf # Avoid circular import. -from . import _docstrings, _instantiators, _resolver, _singleton, _strings +from . import ( + _docstrings, + _instantiators, + _resolver, + _singleton, + _strings, + _unsafe_cache, +) from ._typing import TypeForm from .conf import _confstruct, _markers @@ -163,6 +170,7 @@ class UnsupportedNestedTypeMessage: message: str +@_unsafe_cache.unsafe_cache(maxsize=1024) def is_nested_type(typ: TypeForm[Any], default_instance: DefaultInstance) -> bool: """Determine whether a type should be treated as a 'nested type', where a single type can be broken down into multiple fields (eg for nested dataclasses or diff --git a/tyro/_parsers.py b/tyro/_parsers.py index c91e742dc..f492c8214 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -30,6 +30,7 @@ _resolver, _strings, _subcommand_matching, + _unsafe_cache, ) from ._typing import TypeForm from .conf import _confstruct, _markers diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 278573a40..cc468898d 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -23,7 +23,7 @@ from typing_extensions import Annotated, get_args, get_origin, get_type_hints -from . import _fields +from . import _fields, _unsafe_cache from ._typing import TypeForm TypeOrCallable = TypeVar("TypeOrCallable", TypeForm, Callable) @@ -80,6 +80,7 @@ def resolve_generic_types( return cls, type_from_typevar +@_unsafe_cache.unsafe_cache(maxsize=1024) def resolved_fields(cls: TypeForm) -> List[dataclasses.Field]: """Similar to dataclasses.fields(), but includes dataclasses.InitVar types and resolves forward references.""" @@ -129,6 +130,7 @@ def type_from_typevar_constraints(typ: TypeOrCallable) -> TypeOrCallable: return typ +@_unsafe_cache.unsafe_cache(maxsize=1024) def narrow_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCallable: """Type narrowing: if we annotate as Animal but specify a default instance of Cat, we should parse as Cat. @@ -244,6 +246,7 @@ def apply_type_from_typevar( return typ +@_unsafe_cache.unsafe_cache(maxsize=1024) def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCallable: """Narrow union types. diff --git a/tyro/_subcommand_matching.py b/tyro/_subcommand_matching.py index 5026a1c5b..c7bf6bfc1 100644 --- a/tyro/_subcommand_matching.py +++ b/tyro/_subcommand_matching.py @@ -5,7 +5,7 @@ from typing_extensions import get_args, get_origin -from . import _fields, _instantiators, _resolver, _typing +from . import _fields, _instantiators, _resolver, _typing, _unsafe_cache from .conf import _confstruct @@ -14,6 +14,9 @@ def match_subcommand( subcommand_config_from_name: Dict[str, _confstruct._SubcommandConfiguration], subcommand_type_from_name: Dict[str, type], ) -> Optional[str]: + """Given a subcommand mapping and a default, return which subcommand the default + corresponds to.""" + # It's really hard to concretize a generic type at runtime, so we just... # don't. :-) if hasattr(type(default), "__parameters__"): diff --git a/tyro/_unsafe_cache.py b/tyro/_unsafe_cache.py new file mode 100644 index 000000000..3bcb8f6bc --- /dev/null +++ b/tyro/_unsafe_cache.py @@ -0,0 +1,37 @@ +import functools +from typing import Any, Callable, Dict, TypeVar + +CallableType = TypeVar("CallableType", bound=Callable) + + +def unsafe_cache(maxsize: int) -> Callable[[CallableType], CallableType]: + """Cache decorator that relies object IDs when arguments are unhashable. Assumes + immutability.""" + cache: Dict[Any, Any] = {} + + def inner(f: CallableType) -> CallableType: + @functools.wraps(f) + def wrapped_f(*args, **kwargs): + key = tuple(unsafe_hash(arg) for arg in args) + tuple( + ("__kwarg__", k, unsafe_hash(v)) for k, v in kwargs.items() + ) + + if key in cache: + return cache[key] + + out = f(*args, **kwargs) + cache[key] = out + if len(cache) > maxsize: + cache.pop(next(iter(cache))) + return out + + return wrapped_f # type: ignore + + return inner + + +def unsafe_hash(obj: Any) -> Any: + try: + return hash(obj) + except TypeError: + return id(obj) From 7d1da5af7f53f4e414ceb0effee43425752f6fb7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 11 Mar 2023 20:44:30 -0800 Subject: [PATCH 274/491] Bump version, add cache cleanup --- pyproject.toml | 2 +- tyro/_cli.py | 10 ++++++++++ tyro/_unsafe_cache.py | 24 +++++++++++++++++------- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 603854b7f..ecf7644e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.4.0" +version = "0.4.1" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] diff --git a/tyro/_cli.py b/tyro/_cli.py index dd7187e9b..683a36129 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -27,6 +27,7 @@ _fields, _parsers, _strings, + _unsafe_cache, conf, ) from ._typing import TypeForm @@ -168,6 +169,11 @@ def cli( equivalent. If `return_unknown_args` is True, returns a tuple of the output of `f(...)` and a list of unknown arguments. """ + + # Make sure we start on a clean slate. Some tests may fail without this due to + # memory address conflicts. + _unsafe_cache.clear_cache() + output = _cli_impl( f, prog=prog, @@ -178,6 +184,10 @@ def cli( return_unknown_args=return_unknown_args, **deprecated_kwargs, ) + + # Prevent unnecessary memory usage. + _unsafe_cache.clear_cache() + if return_unknown_args: return cast(Tuple[OutT, List[str]], output) else: diff --git a/tyro/_unsafe_cache.py b/tyro/_unsafe_cache.py index 3bcb8f6bc..b47a3ac5f 100644 --- a/tyro/_unsafe_cache.py +++ b/tyro/_unsafe_cache.py @@ -1,13 +1,23 @@ import functools -from typing import Any, Callable, Dict, TypeVar +from typing import Any, Callable, Dict, List, TypeVar CallableType = TypeVar("CallableType", bound=Callable) +_cache_list: List[Dict[Any, Any]] = [] + + +def clear_cache() -> None: + for c in _cache_list: + c.clear() + + def unsafe_cache(maxsize: int) -> Callable[[CallableType], CallableType]: """Cache decorator that relies object IDs when arguments are unhashable. Assumes immutability.""" - cache: Dict[Any, Any] = {} + + _cache_list.append({}) + local_cache = _cache_list[-1] def inner(f: CallableType) -> CallableType: @functools.wraps(f) @@ -16,13 +26,13 @@ def wrapped_f(*args, **kwargs): ("__kwarg__", k, unsafe_hash(v)) for k, v in kwargs.items() ) - if key in cache: - return cache[key] + if key in local_cache: + return local_cache[key] out = f(*args, **kwargs) - cache[key] = out - if len(cache) > maxsize: - cache.pop(next(iter(cache))) + local_cache[key] = out + if len(local_cache) > maxsize: + local_cache.pop(next(iter(local_cache))) return out return wrapped_f # type: ignore From 620073c442db1efa4317614787209a79993541cb Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 11 Mar 2023 20:55:09 -0800 Subject: [PATCH 275/491] Add cache + container tests, use 3.10 for coverage --- .github/workflows/coverage.yml | 4 ++-- tests/test_errors.py | 8 ++++++++ tests/test_unsafe_cache.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 tests/test_unsafe_cache.py diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index e6eda0614..be421f157 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.10 uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.10" - name: Install dependencies run: | curl -sSL https://install.python-poetry.org | python3 - diff --git a/tests/test_errors.py b/tests/test_errors.py index 97b5bba48..45faf3237 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -131,3 +131,11 @@ def main(value: tyro.conf.Suppress[int]) -> int: with pytest.raises(tyro.UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) + + +def test_ambiguous_sequence() -> None: + def main(value: list) -> None: + return None + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) diff --git a/tests/test_unsafe_cache.py b/tests/test_unsafe_cache.py new file mode 100644 index 000000000..35cb41741 --- /dev/null +++ b/tests/test_unsafe_cache.py @@ -0,0 +1,31 @@ +from tyro import _unsafe_cache + + +def test_unsafe_cache(): + x = 0 + + @_unsafe_cache.unsafe_cache(maxsize=2) + def f(dummy: int): + nonlocal x + x += 1 + + f(0) + f(0) + f(0) + assert x == 1 + f(1) + f(1) + f(1) + assert x == 2 + f(0) + f(0) + f(0) + assert x == 2 + f(2) + f(2) + f(2) + assert x == 3 + f(0) + f(0) + f(0) + assert x == 4 From 8ccd20b683d12cc72de5382d5c7dfd694413c11f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 11 Mar 2023 22:19:28 -0800 Subject: [PATCH 276/491] Support container edge case in subcommands --- tests/test_nested.py | 25 +++++++++++++++++++++++++ tyro/_calling.py | 6 +++++- tyro/_cli.py | 2 +- tyro/_docstrings.py | 12 +----------- tyro/_parsers.py | 2 -- tyro/_strings.py | 17 +++++++++++++++-- tyro/_subcommand_matching.py | 2 +- 7 files changed, 48 insertions(+), 18 deletions(-) diff --git a/tests/test_nested.py b/tests/test_nested.py index 66cd6dacd..e8c90a57e 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -726,6 +726,31 @@ def main(x: Tuple[Tuple[Color], Location, float]): ) == ((Color(255, 0, 0),), Location(5.0, 0.0, 2.0), 4.0) +def test_tuple_nesting_union() -> None: + @dataclasses.dataclass(frozen=True) + class Color: + r: int + g: int + b: int + + @dataclasses.dataclass(frozen=True) + class Location: + x: float + y: float + z: float + + def main(x: Union[Tuple[Tuple[Color], Location, float], Tuple[Color, Color]]): + return x + + assert tyro.cli( + main, + args=( + "x:tuple-tuple-color-location-float --x.0.0.r 255 --x.0.0.g 0 --x.0.0.b 0" + " --x.1.x 5.0 --x.1.y 0.0 --x.1.z 2.0 --x.2 4.0".split(" ") + ), + ) == ((Color(255, 0, 0),), Location(5.0, 0.0, 2.0), 4.0) + + def test_generic_subparsers() -> None: T = TypeVar("T") diff --git a/tyro/_calling.py b/tyro/_calling.py index ec6b38ac1..e38dc3861 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -155,7 +155,11 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: value, consumed_keywords_child = call_from_args( chosen_f, subparser_def.parser_from_name[subparser_name], - field.default if type(field.default) is chosen_f else None, + ( + field.default + if type(field.default) is chosen_f + else _fields.MISSING_NONPROP + ), value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, ) diff --git a/tyro/_cli.py b/tyro/_cli.py index 683a36129..591c3a789 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -326,7 +326,7 @@ def _cli_impl( and modified_args[fixed] != arg ): raise RuntimeError( - f"Ambiguous arguments: " + modified_args[fixed] + " and " + arg + "Ambiguous arguments: " + modified_args[fixed] + " and " + arg ) modified_args[fixed] = arg args[index] = fixed diff --git a/tyro/_docstrings.py b/tyro/_docstrings.py index ac172c7be..265562ce2 100644 --- a/tyro/_docstrings.py +++ b/tyro/_docstrings.py @@ -7,17 +7,7 @@ import io import itertools import tokenize -from typing import ( - Callable, - Dict, - Generic, - Hashable, - List, - Optional, - Type, - TypeVar, - cast, -) +from typing import Callable, Dict, Generic, Hashable, List, Optional, Type, TypeVar import docstring_parser from typing_extensions import get_origin, is_typeddict diff --git a/tyro/_parsers.py b/tyro/_parsers.py index f492c8214..b56f4d932 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -30,7 +30,6 @@ _resolver, _strings, _subcommand_matching, - _unsafe_cache, ) from ._typing import TypeForm from .conf import _confstruct, _markers @@ -355,7 +354,6 @@ def from_field( option, found_subcommand_configs = _resolver.unwrap_annotated( option, _confstruct._SubcommandConfiguration ) - default_hash = None if len(found_subcommand_configs) != 0: # Explicitly annotated default. assert len(found_subcommand_configs) == 1, ( diff --git a/tyro/_strings.py b/tyro/_strings.py index 8b31fc503..7ca994e73 100644 --- a/tyro/_strings.py +++ b/tyro/_strings.py @@ -5,6 +5,8 @@ import textwrap from typing import Iterable, List, Sequence, Tuple, Type, Union +from typing_extensions import get_args, get_origin + from . import _resolver dummy_field_name = "__tyro_dummy_field__" @@ -78,9 +80,20 @@ def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: return found_name, prefix_name # Subparser name from class name. + def get_name(cls: Type) -> str: + if hasattr(cls, "__name__"): + return hyphen_separated_from_camel_case(cls.__name__) + elif hasattr(get_origin(cls), "__name__"): + parts = [get_origin(cls).__name__] # type: ignore + parts.extend(map(get_name, get_args(cls))) + return "-".join(parts) + else: + raise AssertionError( + f"Tried to interpret {cls} as a subcommand, but could not infer name" + ) + if len(type_from_typevar) == 0: - assert hasattr(cls, "__name__") - return hyphen_separated_from_camel_case(cls.__name__), prefix_name # type: ignore + return get_name(cls), prefix_name # type: ignore return ( "-".join( diff --git a/tyro/_subcommand_matching.py b/tyro/_subcommand_matching.py index c7bf6bfc1..b7e6c6b36 100644 --- a/tyro/_subcommand_matching.py +++ b/tyro/_subcommand_matching.py @@ -5,7 +5,7 @@ from typing_extensions import get_args, get_origin -from . import _fields, _instantiators, _resolver, _typing, _unsafe_cache +from . import _fields, _instantiators, _resolver, _typing from .conf import _confstruct From 6ed7282c8fcc9eae8795f19b01255e631077f7a2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 11 Mar 2023 22:58:55 -0800 Subject: [PATCH 277/491] Fix container subcommands for Python 3.10 --- pyproject.toml | 3 ++- tyro/_strings.py | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ecf7644e7..b4527aaa8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.4.1" +version = "0.4.2" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] @@ -66,6 +66,7 @@ exclude_lines = [ # or assert statements & errors "assert", + "raise AssertionError", # or anything that's not implemented "NotImplementedError()", diff --git a/tyro/_strings.py b/tyro/_strings.py index 7ca994e73..620106e36 100644 --- a/tyro/_strings.py +++ b/tyro/_strings.py @@ -81,12 +81,13 @@ def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: # Subparser name from class name. def get_name(cls: Type) -> str: - if hasattr(cls, "__name__"): - return hyphen_separated_from_camel_case(cls.__name__) - elif hasattr(get_origin(cls), "__name__"): - parts = [get_origin(cls).__name__] # type: ignore + orig = get_origin(cls) + if orig is not None and hasattr(orig, "__name__"): + parts = [orig.__name__] # type: ignore parts.extend(map(get_name, get_args(cls))) return "-".join(parts) + elif hasattr(cls, "__name__"): + return hyphen_separated_from_camel_case(cls.__name__) else: raise AssertionError( f"Tried to interpret {cls} as a subcommand, but could not infer name" From 45494a9efe875493a3db691a56d82b0d9c95113d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 13 Mar 2023 22:56:56 -0700 Subject: [PATCH 278/491] Typo --- tyro/conf/_markers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index df0a4d57a..279223583 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -117,7 +117,7 @@ def configure(*markers: Marker) -> Callable[[CallableType], CallableType]: This decorator makes markers applicable to general functions as well: ```python - # Recursively apply FlagConversionOff to all field in `main()`. + # Recursively apply FlagConversionOff to all fields in `main()`. @tyro.conf.configure(tyro.conf.FlagConversionOff) def main(field: bool) -> None: ... From c3fd21406f61bc92ff23e52a6a53ab8ab38b09b6 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 18 Mar 2023 22:13:36 -0700 Subject: [PATCH 279/491] Tweak docs for type generation helpers --- .../03_config_systems/01_base_configs.rst | 25 ++++---- examples/03_config_systems/01_base_configs.py | 18 +++--- tyro/extras/_base_configs.py | 60 +++++-------------- tyro/extras/_choices_type.py | 22 +++---- 4 files changed, 46 insertions(+), 79 deletions(-) diff --git a/docs/source/examples/03_config_systems/01_base_configs.rst b/docs/source/examples/03_config_systems/01_base_configs.rst index 968222cba..783ee3ac6 100644 --- a/docs/source/examples/03_config_systems/01_base_configs.rst +++ b/docs/source/examples/03_config_systems/01_base_configs.rst @@ -9,10 +9,8 @@ We can integrate ``tyro.cli()`` into common configuration patterns: here, we sel one of multiple possible base configurations, create a subcommand for each one, and then use the CLI to either override (existing) or fill in (missing) values. -Note that our interfaces don't prescribe any of the mechanics used for storing -base configurations. A Hydra-style YAML approach could just as easily -be used for the config library (although we generally prefer to avoid YAMLs; staying in -Python is convenient for autocompletion and type checking). +This example is verbose; to shorten, consider using +:func:`tyro.extras.subcommand_type_from_defaults()`. @@ -79,6 +77,7 @@ Python is convenient for autocompletion and type checking). activation=nn.ReLU, ), description="Train a smaller model.", + prefix_name=False, ), ] BigConfig = Annotated[ @@ -96,10 +95,12 @@ Python is convenient for autocompletion and type checking). activation=nn.GELU, ), description="Train a bigger model.", + prefix_name=False, ), ] + @tyro.conf.configure(tyro.conf.ConsolidateSubcommandArgs) def main( config: Union[SmallConfig, BigConfig], restore_checkpoint: bool = False, @@ -123,30 +124,30 @@ Python is convenient for autocompletion and type checking). .. raw:: html - python 03_config_systems/01_base_configs.py config:small --help + python 03_config_systems/01_base_configs.py small --help -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py config:small --help +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py small --help ------------ .. raw:: html - python 03_config_systems/01_base_configs.py config:small --config.seed 94720 + python 03_config_systems/01_base_configs.py small --config.seed 94720 -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py config:small --config.seed 94720 +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py small --config.seed 94720 ------------ .. raw:: html - python 03_config_systems/01_base_configs.py config:big --help + python 03_config_systems/01_base_configs.py big --help -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py config:big --help +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py big --help ------------ .. raw:: html - python 03_config_systems/01_base_configs.py config:big --config.seed 94720 + python 03_config_systems/01_base_configs.py big --config.seed 94720 -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py config:big --config.seed 94720 +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py big --config.seed 94720 diff --git a/examples/03_config_systems/01_base_configs.py b/examples/03_config_systems/01_base_configs.py index 1f34f8ddb..b1507a16c 100644 --- a/examples/03_config_systems/01_base_configs.py +++ b/examples/03_config_systems/01_base_configs.py @@ -4,17 +4,16 @@ one of multiple possible base configurations, create a subcommand for each one, and then use the CLI to either override (existing) or fill in (missing) values. -Note that our interfaces don't prescribe any of the mechanics used for storing -base configurations. A Hydra-style YAML approach could just as easily -be used for the config library (although we generally prefer to avoid YAMLs; staying in -Python is convenient for autocompletion and type checking). +This example is verbose; to shorten, consider using +:func:`tyro.extras.subcommand_type_from_defaults()`. + Usage: `python ./10_base_configs.py --help` -`python ./10_base_configs.py config:small --help` -`python ./10_base_configs.py config:small --config.seed 94720` -`python ./10_base_configs.py config:big --help` -`python ./10_base_configs.py config:big --config.seed 94720` +`python ./10_base_configs.py small --help` +`python ./10_base_configs.py small --config.seed 94720` +`python ./10_base_configs.py big --help` +`python ./10_base_configs.py big --config.seed 94720` """ from dataclasses import dataclass @@ -76,6 +75,7 @@ class ExperimentConfig: activation=nn.ReLU, ), description="Train a smaller model.", + prefix_name=False, ), ] BigConfig = Annotated[ @@ -93,10 +93,12 @@ class ExperimentConfig: activation=nn.GELU, ), description="Train a bigger model.", + prefix_name=False, ), ] +@tyro.conf.configure(tyro.conf.ConsolidateSubcommandArgs) def main( config: Union[SmallConfig, BigConfig], restore_checkpoint: bool = False, diff --git a/tyro/extras/_base_configs.py b/tyro/extras/_base_configs.py index 792782091..2ed3e5b35 100644 --- a/tyro/extras/_base_configs.py +++ b/tyro/extras/_base_configs.py @@ -16,35 +16,6 @@ def subcommand_type_from_defaults( ) -> TypeForm[T]: """Construct a Union type for defining subcommands that choose between defaults. - .. warning:: - - Use of this helper is discouraged. It will likely be deprecated. - - Using the the returned type is understood as an annotation by ``pyright`` and - ``pylance`` (with ``from __future__ import annotations``), but it relies on - behavior that isn't defined by the Python language specifications. - - At the cost of verbosity, using :func:`tyro.conf.subcommand()` directly is - better supported by tools like ``mypy``. - - Alternatively, we can work around this limitation with an ``if TYPE_CHECKING`` - guard: - - .. code-block:: python - - from typing import TYPE_CHECKING - - if TYPE_CHECKING: - # Static type seen by mypy, language servers, etc. - SelectableConfig = Config - else: - # Runtime type used by tyro. - SelectableConfig = subcommand_type_from_defaults(...) - - - This can most commonly be used to create a "base configuration" pattern: - https://brentyi.github.io/tyro/examples/10_base_configs/ - For example, when `defaults` is set to: ```python @@ -69,26 +40,27 @@ def subcommand_type_from_defaults( ] ``` - The resulting type can be used directly in tyro.cli: + Direct use of `typing.Union` and `tyro.conf.subcommand()` should generally be + preferred, but this function can be helpful for succinictness. - ```python - config = tyro.cli(subcommand_type_from_defaults(default_from_name)) - reveal_type(config) # Should be correct! - ``` - Or to generate annotations for classes and functions: + .. warning:: - ```python - SelectableConfig = subcommand_type_from_defaults(default_from_name) + The type returned by this function can be safely used as an input to + `tyro.cli()`, but for static analysis when used for annotations we recommend + applying a TYPE_CHECKING guard: - def train( - config: SelectableConfig, - checkpoint_path: Optional[pathlib.Path] = None, - ) -> None: - ... + .. code-block:: python + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + # Static type seen by language servers, type checkers, etc. + SelectableConfig = Config + else: + # Runtime type used by tyro. + SelectableConfig = subcommand_type_from_defaults(...) - tyro.cli(train) - ``` """ return Union.__getitem__( # type: ignore tuple( diff --git a/tyro/extras/_choices_type.py b/tyro/extras/_choices_type.py index d10819d62..68da8cb61 100644 --- a/tyro/extras/_choices_type.py +++ b/tyro/extras/_choices_type.py @@ -11,33 +11,25 @@ def literal_type_from_choices(choices: Iterable[T]) -> TypeForm[T]: """Generate a `typing.Literal[]` type that constrains values to a set of choices. - .. warning:: - - Use of this helper is discouraged. It will likely be deprecated. + Using `Literal[...]` directly should generally be preferred, but this function can be + helpful when choices are generated dynamically. - The the returned type is understood as an annotation by ``pyright`` and - ``pylance`` (with ``from __future__ import annotations``), but it relies on - behavior that isn't defined by the Python language specifications. - At the cost of verbosity, using ``typing.Literal[]`` directly is better supported - by tools like ``mypy``. + .. warning:: - Alternatively, we can work around this limitation with an ``if TYPE_CHECKING`` - guard: + The type returned by this function can be safely used as an input to + `tyro.cli()`, but for static analysis when used for annotations we recommend + applying a TYPE_CHECKING guard: .. code-block:: python from typing import TYPE_CHECKING if TYPE_CHECKING: - # Static type seen by mypy, language servers, etc. + # Static type seen by language servers, type checkers, etc. Color = str else: # Runtime type used by tyro. Color = literal_type_from_choices(["red", "green", "blue"]) - - Using `Literal[...]` directly should generally be preferred, but this helper can be - used in the rare case that choices are generated dynamically. (for example, the keys - of a dictionary) """ return Literal.__getitem__(tuple(choices)) # type: ignore From b956fff62ecc367cfadffd84926f8111b46f3580 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 28 Apr 2023 00:23:34 -0700 Subject: [PATCH 280/491] Avoid unnecessary flax imports --- pyproject.toml | 2 +- tyro/_cli.py | 5 +---- tyro/_fields.py | 14 ++++++++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b4527aaa8..87ee2786b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.4.2" +version = "0.4.3" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] diff --git a/tyro/_cli.py b/tyro/_cli.py index 591c3a789..7efa87438 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -255,10 +255,7 @@ def _cli_impl( return_unknown_args: bool, **deprecated_kwargs, ) -> Union[OutT, argparse.ArgumentParser, Tuple[OutT, List[str]],]: - """Helper for stitching the `tyro` pipeline together. - - Converts `f` into a - """ + """Helper for stitching the `tyro` pipeline together.""" if "default_instance" in deprecated_kwargs: warnings.warn( "`default_instance=` is deprecated! use `default=` instead.", stacklevel=2 diff --git a/tyro/_fields.py b/tyro/_fields.py index 6366f45c1..1e72862c5 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -10,6 +10,7 @@ import inspect import itertools import os +import sys import typing import warnings from typing import ( @@ -394,13 +395,18 @@ def _field_list_from_namedtuple( def _field_list_from_dataclass( cls: TypeForm[Any], default_instance: DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - # Check if dataclass is a flax module. is_flax_module = False try: - import flax + # Check if dataclass is a flax module. This is only possible if flax is already + # loaded. + # + # We generally want to avoid importing flax, since it requires a lot of heavy + # imports. + if "flax" in sys.modules.keys(): + import flax - if issubclass(cls, flax.linen.Module): - is_flax_module = True + if issubclass(cls, flax.linen.Module): + is_flax_module = True except ImportError: pass From 5a20d69e8c5a7c3509ac13cb54076dabff1f7588 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 4 May 2023 10:05:37 -0700 Subject: [PATCH 281/491] Append action for variable-length containers (#47) * Add marker for append actions * Tweak naming, docs * Fix edge cases * Support variable-length nesting * Fix mypy errors --- .../04_additional/02_dictionaries.rst | 10 - examples/04_additional/02_dictionaries.py | 12 +- tests/test_conf.py | 156 ++++++++++- tests/test_errors.py | 11 +- tests/test_nested.py | 4 +- tyro/_arguments.py | 63 ++++- tyro/_docstrings.py | 2 +- tyro/_fields.py | 4 +- tyro/_instantiators.py | 262 +++++++++++------- tyro/conf/__init__.py | 2 + tyro/conf/_markers.py | 13 + 11 files changed, 402 insertions(+), 137 deletions(-) diff --git a/docs/source/examples/04_additional/02_dictionaries.rst b/docs/source/examples/04_additional/02_dictionaries.rst index 2c52b34ac..99add94e9 100644 --- a/docs/source/examples/04_additional/02_dictionaries.rst +++ b/docs/source/examples/04_additional/02_dictionaries.rst @@ -16,8 +16,6 @@ Dictionary inputs can be specified using either a standard ``Dict[K, V]`` annota from typing import Dict, Mapping, Tuple, TypedDict - from frozendict import frozendict # type: ignore - import tyro @@ -37,19 +35,11 @@ Dictionary inputs can be specified using either a standard ``Dict[K, V]`` annota "beta1": 0.9, "beta2": 0.999, }, - frozen_dict: Mapping[str, float] = frozendict( - { - "num_epochs": 20, - "batch_size": 64, - } - ), ) -> None: assert isinstance(typed_dict, dict) assert isinstance(standard_dict, dict) - assert isinstance(frozen_dict, frozendict) print("Typed dict:", typed_dict) print("Standard dict:", standard_dict) - print("Frozen dict:", frozen_dict) if __name__ == "__main__": diff --git a/examples/04_additional/02_dictionaries.py b/examples/04_additional/02_dictionaries.py index a5894fa5d..d09641ba0 100644 --- a/examples/04_additional/02_dictionaries.py +++ b/examples/04_additional/02_dictionaries.py @@ -9,9 +9,7 @@ `python ./02_dictionaries.py --typed-dict.betas 0.9 0.999` """ -from typing import Dict, Mapping, Tuple, TypedDict - -from frozendict import frozendict # type: ignore +from typing import Dict, Tuple, TypedDict import tyro @@ -32,19 +30,11 @@ def main( "beta1": 0.9, "beta2": 0.999, }, - frozen_dict: Mapping[str, float] = frozendict( - { - "num_epochs": 20, - "batch_size": 64, - } - ), ) -> None: assert isinstance(typed_dict, dict) assert isinstance(standard_dict, dict) - assert isinstance(frozen_dict, frozendict) print("Typed dict:", typed_dict) print("Standard dict:", standard_dict) - print("Frozen dict:", frozen_dict) if __name__ == "__main__": diff --git a/tests/test_conf.py b/tests/test_conf.py index 2b75bc905..99f34853e 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Any, Generic, TypeVar, Union +from typing import Any, Dict, Generic, List, Tuple, TypeVar, Union import pytest from helptext_utils import get_helptext @@ -668,3 +668,157 @@ def func(parent: DefaultInstanceSubparser) -> DefaultInstanceSubparser: "8", ], ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + + +def test_append_lists() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[List[int]] + + assert tyro.cli(A, args="--x 1 --x 2 --x 3".split(" ")) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=[]) == A(x=[]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3"]) + + +def test_append_tuple() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Tuple[int, ...]] + + assert tyro.cli(A, args="--x 1 --x 2 --x 3".split(" ")) == A(x=(1, 2, 3)) + assert tyro.cli(A, args=[]) == A(x=()) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3"]) + + +def test_append_tuple_with_default() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Tuple[int, ...]] = (1, 2) + + assert tyro.cli(A, args="--x 1 --x 2 --x 3".split(" ")) == A(x=(1, 2, 1, 2, 3)) + assert tyro.cli(A, args=[]) == A(x=(1, 2)) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3"]) + + +def test_append_nested_tuple_fixed_length() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Tuple[Tuple[str, int], ...]] + + assert tyro.cli(A, args="--x 1 1 --x 2 2 --x 3 3".split(" ")) == A( + x=(("1", 1), ("2", 2), ("3", 3)) + ) + assert tyro.cli(A, args=[]) == A(x=()) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3"]) + + +def test_append_nested_tuple_with_default_fixed_length() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Tuple[Tuple[str, int], ...]] = (("1", 1), ("2", 2)) + + assert tyro.cli(A, args="--x 1 1 --x 2 2 --x 3 3".split(" ")) == A( + x=(("1", 1), ("2", 2), ("1", 1), ("2", 2), ("3", 3)) + ) + assert tyro.cli(A, args=[]) == A(x=(("1", 1), ("2", 2))) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3"]) + + +def test_append_dict() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Dict[str, int]] + + assert tyro.cli(A, args="--x 1 1 --x 2 2 --x 3 3".split(" ")) == A( + x={"1": 1, "2": 2, "3": 3} + ) + assert tyro.cli(A, args=[]) == A(x={}) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3"]) + + +def test_append_dict_with_default() -> None: + """Append has no impact when a dictionary has a default value.""" + + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Dict[str, int]] = dataclasses.field( + default_factory=lambda: {"1": 1} + ) + + assert tyro.cli(A, args=[]) == A(x={"1": 1}) + assert tyro.cli(A, args=["--x.1", "2"]) == A(x={"1": 2}) + with pytest.raises(SystemExit): + assert tyro.cli(A, args="--x 2 2 --x 3 3".split(" ")) == A( + x={"1": 1, "2": 2, "3": 3} + ) + + +def test_append_nested_tuple() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Tuple[Tuple[str, ...], ...]] + + assert tyro.cli(A, args="--x 1 2 3 --x 4 5".split(" ")) == A( + x=(("1", "2", "3"), ("4", "5")) + ) + assert tyro.cli(A, args=[]) == A(x=()) + + +def test_append_nested_tuple_with_default() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Tuple[Tuple[str, ...], ...]] = (("1", "2"),) + + assert tyro.cli(A, args="--x 1 2 3 --x 4 5".split(" ")) == A( + x=(("1", "2"), ("1", "2", "3"), ("4", "5")) + ) + assert tyro.cli(A, args=[]) == A(x=(("1", "2"),)) + + +def test_append_nested_list() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[List[List[int]]] + + assert tyro.cli(A, args="--x 1 2 3 --x 4 5".split(" ")) == A(x=[[1, 2, 3], [4, 5]]) + assert tyro.cli(A, args=[]) == A(x=[]) + + +def test_append_nested_dict() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[List[Dict[str, int]]] + + assert tyro.cli(A, args="--x 1 2 3 4 --x 4 5".split(" ")) == A( + x=[{"1": 2, "3": 4}, {"4": 5}] + ) + assert tyro.cli(A, args=[]) == A(x=[]) + + +def test_append_nested_dict_double() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Dict[str, Dict[str, int]]] + + assert tyro.cli(A, args="--x 0 1 2 3 4 --x 4 5 6".split(" ")) == A( + x={"0": {"1": 2, "3": 4}, "4": {"5": 6}} + ) + assert tyro.cli(A, args=[]) == A(x={}) diff --git a/tests/test_errors.py b/tests/test_errors.py index 45faf3237..ced3abaa1 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -16,6 +16,15 @@ class A: def test_ambiguous_collection_1() -> None: + @dataclasses.dataclass + class A: + x: List[List[int]] + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(A, args=["--x", "0", "1"]) + + +def test_ambiguous_collection_2() -> None: def main(x: Tuple[List[str], List[str]]) -> None: pass @@ -23,7 +32,7 @@ def main(x: Tuple[List[str], List[str]]) -> None: tyro.cli(main, args=["--help"]) -def test_ambiguous_collection_2() -> None: +def test_ambiguous_collection_3() -> None: def main(x: List[Union[Tuple[int, int], Tuple[int, int, int]]]) -> None: pass diff --git a/tests/test_nested.py b/tests/test_nested.py index e8c90a57e..584e14119 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -829,7 +829,7 @@ class Parent: def test_frozen_dict() -> None: def main( - x: Mapping[str, float] = frozendict( + x: Mapping[str, float] = frozendict( # type: ignore { "num_epochs": 20, "batch_size": 64, @@ -839,7 +839,7 @@ def main( return x assert hash(tyro.cli(main, args="--x.num-epochs 10".split(" "))) == hash( - frozendict({"num_epochs": 10, "batch_size": 64}) + frozendict({"num_epochs": 10, "batch_size": 64}) # type: ignore ) diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 8653d30f7..64c9892ed 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -41,7 +41,7 @@ def add_argument( """Add a defined argument to a parser.""" # Get keyword arguments, with None values removed. - kwargs = dataclasses.asdict(self.lowered) + kwargs = dataclasses.asdict(self.lowered) # type: ignore kwargs.pop("instantiator") kwargs = {k: v for k, v in kwargs.items() if v is not None} name_or_flag = kwargs.pop("name_or_flag") @@ -52,7 +52,10 @@ def add_argument( # MISSING value will be detected in _calling.py and the field default will # directly be used. This helps reduce the likelihood of issues with converting # the field default to a string format, then back to the desired type. - kwargs["default"] = _fields.MISSING_NONPROP + if kwargs.get("action", None) != "append": + kwargs["default"] = _fields.MISSING_NONPROP + else: + kwargs["default"] = [] # Apply overrides in our arg configuration object. # Note that the `name` field is applied when the field object is instantiated! @@ -215,8 +218,9 @@ def _rule_recursive_instantiator_from_type( return lowered try: instantiator, metadata = _instantiators.instantiator_from_type( - arg.field.typ, # type: ignore + arg.field.typ, arg.type_from_typevar, + arg.field.markers, ) except _instantiators.UnsupportedTypeAnnotationError as e: if arg.field.default in _fields.MISSING_SINGLETONS: @@ -235,13 +239,36 @@ def _rule_recursive_instantiator_from_type( default=_fields.MISSING_PROP, ) - return dataclasses.replace( - lowered, - instantiator=instantiator, - choices=metadata.choices, - nargs=metadata.nargs, - metavar=metadata.metavar, - ) + if metadata.action == "append": + + def append_instantiator(x: Any) -> Any: + out = instantiator(x) + if arg.field.default in _fields.MISSING_SINGLETONS: + return instantiator(x) + + return type(out)(arg.field.default) + out + + return out + + return dataclasses.replace( + lowered, + instantiator=append_instantiator, + default=None, + choices=metadata.choices, + nargs=metadata.nargs, + metavar=metadata.metavar, + action=metadata.action, + required=False, + ) + else: + return dataclasses.replace( + lowered, + instantiator=instantiator, + choices=metadata.choices, + nargs=metadata.nargs, + metavar=metadata.metavar, + action=metadata.action, + ) def _rule_convert_defaults_to_strings( @@ -312,10 +339,9 @@ def _rule_generate_helptext( help_parts.append(_rich_tag_if_enabled(primary_help, "helptext")) default = lowered.default - if lowered.is_fixed(): - # For fixed args, we'll be missing the lowered default. Use field default - # instead. - assert default in _fields.MISSING_SINGLETONS + if lowered.is_fixed() or lowered.action == "append": + # Cases where we'll be missing the lowered default. Use field default instead. + assert default in _fields.MISSING_SINGLETONS or default is None default = arg.field.default if not lowered.required: @@ -330,6 +356,15 @@ def _rule_generate_helptext( default_text = f"(sets: {arg.field.name}=True)" elif lowered.action == "store_false": default_text = f"(sets: {arg.field.name}=False)" + elif lowered.action == "append" and ( + arg.field.default in _fields.MISSING_SINGLETONS + or len(arg.field.default) == 0 + ): + default_text = "(repeatable)" + elif lowered.action == "append" and len(arg.field.default) > 0: + assert default is not None # Just for type checker. + default_parts = map(shlex.quote, map(str, default)) + default_text = f"(repeatable, appends: {' '.join(default_parts)})" elif arg.field.default is _fields.EXCLUDE_FROM_CALL: default_text = "(unset by default)" elif lowered.nargs is not None and hasattr(default, "__iter__"): diff --git a/tyro/_docstrings.py b/tyro/_docstrings.py index 265562ce2..9f0fe4e1b 100644 --- a/tyro/_docstrings.py +++ b/tyro/_docstrings.py @@ -308,7 +308,7 @@ def get_callable_description(f: Callable) -> str: docstring = _strings.dedent(docstring) if dataclasses.is_dataclass(f): - default_doc = f.__name__ + str(inspect.signature(f)).replace(" -> None", "") + default_doc = f.__name__ + str(inspect.signature(f)).replace(" -> None", "") # type: ignore if docstring == default_doc: return "" diff --git a/tyro/_fields.py b/tyro/_fields.py index 1e72862c5..f4df8f1ee 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -402,8 +402,8 @@ def _field_list_from_dataclass( # # We generally want to avoid importing flax, since it requires a lot of heavy # imports. - if "flax" in sys.modules.keys(): - import flax + if "flax.linen" in sys.modules.keys(): + import flax.linen if issubclass(cls, flax.linen.Module): is_flax_module = True diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 8ad012671..4e4794ab7 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -41,6 +41,7 @@ Any, Callable, Dict, + FrozenSet, Hashable, Iterable, List, @@ -63,14 +64,16 @@ from . import _strings from ._typing import TypeForm +from .conf import _markers _StandardInstantiator = Callable[[List[str]], Any] +_AppendNargsInstantiator = Callable[[List[List[str]]], Any] # Special case: the only time that argparse doesn't give us a string is when the # argument action is set to `store_true` or `store_false`. In this case, we get # a bool directly, and the field action can be a no-op. _FlagInstantiator = Callable[[bool], bool] -Instantiator = Union[_StandardInstantiator, _FlagInstantiator] +Instantiator = Union[_StandardInstantiator, _AppendNargsInstantiator, _FlagInstantiator] NoneType = type(None) @@ -79,11 +82,12 @@ class InstantiatorMetadata: # Unlike in vanilla argparse, we never set nargs to None. To make things simpler, we # instead use nargs=1. - nargs: Union[int, Literal["+"]] + nargs: Optional[Union[int, Literal["+"]]] # Unlike in vanilla argparse, our metavar is always a string. We handle # sequences, multiple arguments, etc, manually. metavar: str choices: Optional[Tuple[str, ...]] + action: Optional[Literal["append"]] def check_choices(self, strings: List[str]) -> None: if self.choices is not None and any(s not in self.choices for s in strings): @@ -136,7 +140,9 @@ def is_type_string_converter(typ: Union[Callable, TypeForm[Any]]) -> bool: def instantiator_from_type( - typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] + typ: TypeForm, + type_from_typevar: Dict[TypeVar, TypeForm[Any]], + markers: FrozenSet[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: """Recursive helper for parsing type annotations. @@ -163,6 +169,7 @@ def instantiator(strings: List[str]) -> None: nargs=1, metavar="{None}", choices=("None",), + action=None, ) # Instantiate os.PathLike annotations using pathlib.Path. @@ -173,7 +180,7 @@ def instantiator(strings: List[str]) -> None: # Address container types. If a matching container is found, this will recursively # call instantiator_from_type(). - container_out = _instantiator_from_container_type(typ, type_from_typevar) + container_out = _instantiator_from_container_type(typ, type_from_typevar, markers) if container_out is not None: return container_out @@ -233,6 +240,7 @@ def instantiator_base_case(strings: List[str]) -> Any: else "{" + ",".join(map(str, auto_choices)) + "}" ), choices=auto_choices, + action=None, ) @@ -241,6 +249,7 @@ def _instantiator_from_type_inner( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal["fixed_length"], + markers: FrozenSet[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: ... @@ -250,6 +259,7 @@ def _instantiator_from_type_inner( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal[False], + markers: FrozenSet[_markers.Marker], ) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: ... @@ -259,6 +269,7 @@ def _instantiator_from_type_inner( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal[True], + markers: FrozenSet[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: ... @@ -267,11 +278,12 @@ def _instantiator_from_type_inner( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal["fixed_length", True, False], + markers: FrozenSet[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: """Thin wrapper over instantiator_from_type, with some extra asserts for catching errors.""" - out = instantiator_from_type(typ, type_from_typevar) - if out[1].nargs is not None: + out = instantiator_from_type(typ, type_from_typevar, markers) + if out[1].nargs == "+": # We currently only use allow_sequences=False for options in Literal types, # which are evaluated using `type()`. It should not be possible to hit this # condition from polling a runtime type. @@ -284,7 +296,9 @@ def _instantiator_from_type_inner( def _instantiator_from_container_type( - typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] + typ: TypeForm, + type_from_typevar: Dict[TypeVar, TypeForm[Any]], + markers: FrozenSet[_markers.Marker], ) -> Optional[Tuple[Instantiator, InstantiatorMetadata]]: """Attempt to create an instantiator from a container type. Returns `None` is no container type is found.""" @@ -296,7 +310,7 @@ def _instantiator_from_container_type( # Unwrap Annotated and Final types. if type_origin in (Annotated, Final): contained_type = get_args(typ)[0] - return instantiator_from_type(contained_type, type_from_typevar) + return instantiator_from_type(contained_type, type_from_typevar, markers) for make, matched_origins in { _instantiator_from_sequence: ( @@ -312,7 +326,7 @@ def _instantiator_from_container_type( _instantiator_from_literal: (Literal,), }.items(): if type_origin in matched_origins: - return make(typ, type_from_typevar) + return make(typ, type_from_typevar, markers) raise UnsupportedTypeAnnotationError( # pragma: no cover f"Unsupported type {typ} with origin {type_origin}" @@ -320,7 +334,9 @@ def _instantiator_from_container_type( def _instantiator_from_tuple( - typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] + typ: TypeForm, + type_from_typevar: Dict[TypeVar, TypeForm[Any]], + markers: FrozenSet[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: types = get_args(typ) typeset = set(types) # Note that sets are unordered. @@ -330,7 +346,7 @@ def _instantiator_from_tuple( # Ellipsis: variable argument counts. When an ellipsis is used, tuples must # contain only one type. assert len(typeset_no_ellipsis) == 1 - return _instantiator_from_sequence(typ, type_from_typevar) + return _instantiator_from_sequence(typ, type_from_typevar, markers) else: instantiators: List[_StandardInstantiator] = [] @@ -338,9 +354,7 @@ def _instantiator_from_tuple( nargs = 0 for t in types: a, b = _instantiator_from_type_inner( - t, - type_from_typevar, - allow_sequences="fixed_length", + t, type_from_typevar, allow_sequences="fixed_length", markers=markers ) instantiators.append(a) # type: ignore metas.append(b) @@ -364,6 +378,7 @@ def fixed_length_tuple_instantiator(strings: List[str]) -> Any: nargs=nargs, metavar=" ".join(m.metavar for m in metas), choices=None, + action=None, ) @@ -404,7 +419,9 @@ def _join_union_metavars(metavars: Iterable[str]) -> str: def _instantiator_from_union( - typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] + typ: TypeForm, + type_from_typevar: Dict[TypeVar, TypeForm[Any]], + markers: FrozenSet[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: options = list(get_args(typ)) if NoneType in options: @@ -418,13 +435,11 @@ def _instantiator_from_union( # right. instantiators = [] metas = [] - nargs: Union[int, Literal["+"]] = 1 + nargs: Optional[Union[int, Literal["+"]]] = 1 first = True for t in options: a, b = _instantiator_from_type_inner( - t, - type_from_typevar, - allow_sequences=True, + t, type_from_typevar, allow_sequences=True, markers=markers ) instantiators.append(a) metas.append(b) @@ -476,67 +491,96 @@ def union_instantiator(strings: List[str]) -> Any: nargs=nargs, metavar=metavar, choices=None, + action=None, ) def _instantiator_from_dict( - typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] + typ: TypeForm, + type_from_typevar: Dict[TypeVar, TypeForm[Any]], + markers: FrozenSet[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: key_type, val_type = get_args(typ) key_instantiator, key_meta = _instantiator_from_type_inner( - key_type, - type_from_typevar, - allow_sequences="fixed_length", - ) - val_instantiator, val_meta = _instantiator_from_type_inner( - val_type, - type_from_typevar, - allow_sequences="fixed_length", + key_type, type_from_typevar, allow_sequences="fixed_length", markers=markers ) - key_nargs = cast(int, key_meta.nargs) # Casts needed for mypy but not pyright! - val_nargs = cast(int, val_meta.nargs) - assert isinstance(key_nargs, int) - assert isinstance(val_nargs, int) - pair_nargs = key_nargs + val_nargs - - def dict_instantiator(strings: List[str]) -> Any: - out = {} - if len(strings) % pair_nargs != 0: - raise ValueError("incomplete set of key value pairs!") - - index = 0 - for _ in range(len(strings) // pair_nargs): - k = strings[index : index + key_nargs] - index += key_nargs - v = strings[index : index + val_nargs] - index += val_nargs - - if key_meta.choices is not None and any( - kj not in key_meta.choices for kj in k - ): - raise ValueError( - f"invalid choice: {k} (choose from {key_meta.choices}))" - ) - if val_meta.choices is not None and any( - vj not in val_meta.choices for vj in v - ): - raise ValueError( - f"invalid choice: {v} (choose from {val_meta.choices}))" - ) - out[key_instantiator(k)] = val_instantiator(v) # type: ignore - return out - - pair_metavar = f"{key_meta.metavar} {val_meta.metavar}" - return dict_instantiator, InstantiatorMetadata( - nargs="+", - metavar=_strings.multi_metavar_from_single(pair_metavar), - choices=None, - ) + if _markers.UseAppendAction in markers: + val_instantiator, val_meta = _instantiator_from_type_inner( + val_type, + type_from_typevar, + allow_sequences=True, + markers=markers - {_markers.UseAppendAction}, + ) + pair_metavar = f"{key_meta.metavar} {val_meta.metavar}" + key_nargs = cast(int, key_meta.nargs) # Casts needed for mypy but not pyright! + val_nargs = val_meta.nargs + assert isinstance(key_nargs, int) + + def append_dict_instantiator(strings: List[List[str]]) -> Any: + out = {} + for s in strings: + out[key_instantiator(s[:key_nargs])] = val_instantiator(s[key_nargs:]) # type: ignore + return out + + return append_dict_instantiator, InstantiatorMetadata( + nargs=key_nargs + val_nargs if isinstance(val_nargs, int) else "+", + metavar=pair_metavar, + choices=None, + action="append", + ) + else: + val_instantiator, val_meta = _instantiator_from_type_inner( + val_type, type_from_typevar, allow_sequences="fixed_length", markers=markers + ) + pair_metavar = f"{key_meta.metavar} {val_meta.metavar}" + key_nargs = cast(int, key_meta.nargs) # Casts needed for mypy but not pyright! + val_nargs = cast(int, val_meta.nargs) + assert isinstance(key_nargs, int) + assert isinstance(val_nargs, int) + pair_nargs = key_nargs + val_nargs + + def dict_instantiator(strings: List[str]) -> Any: + out = {} + if len(strings) % pair_nargs != 0: + raise ValueError("incomplete set of key value pairs!") + + index = 0 + for _ in range(len(strings) // pair_nargs): + assert isinstance(key_nargs, int) + assert isinstance(val_nargs, int) + k = strings[index : index + key_nargs] + index += key_nargs + v = strings[index : index + val_nargs] + index += val_nargs + + if key_meta.choices is not None and any( + kj not in key_meta.choices for kj in k + ): + raise ValueError( + f"invalid choice: {k} (choose from {key_meta.choices}))" + ) + if val_meta.choices is not None and any( + vj not in val_meta.choices for vj in v + ): + raise ValueError( + f"invalid choice: {v} (choose from {val_meta.choices}))" + ) + out[key_instantiator(k)] = val_instantiator(v) # type: ignore + return out + + return dict_instantiator, InstantiatorMetadata( + nargs="+", + metavar=_strings.multi_metavar_from_single(pair_metavar), + choices=None, + action=None, + ) def _instantiator_from_sequence( - typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] + typ: TypeForm, + type_from_typevar: Dict[TypeVar, TypeForm[Any]], + markers: FrozenSet[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: """Instantiator for variable-length sequences: list, sets, Tuple[T, ...], etc.""" container_type = get_origin(typ) @@ -549,38 +593,65 @@ def _instantiator_from_sequence( else: (contained_type,) = get_args(typ) - make, inner_meta = _instantiator_from_type_inner( - contained_type, - type_from_typevar, - allow_sequences="fixed_length", - ) + if _markers.UseAppendAction in markers: + make, inner_meta = _instantiator_from_type_inner( + contained_type, + type_from_typevar, + allow_sequences=True, + markers=markers - {_markers.UseAppendAction}, + ) - def sequence_instantiator(strings: List[str]) -> Any: - # Validate nargs. - assert type(inner_meta.nargs) in (int, NoneType) - if isinstance(inner_meta.nargs, int) and len(strings) % inner_meta.nargs != 0: - raise ValueError( - f"input {strings} is of length {len(strings)}, which is not divisible" - f" by {inner_meta.nargs}." - ) + def append_sequence_instantiator(strings: Optional[List[List[str]]]) -> Any: + if strings is None: + assert container_type is not None + return container_type() + return container_type(make(s) for s in strings) # type: ignore + + return append_sequence_instantiator, InstantiatorMetadata( + nargs=inner_meta.nargs, + metavar=inner_meta.metavar, + choices=inner_meta.choices, + action="append", + ) + else: + make, inner_meta = _instantiator_from_type_inner( + contained_type, + type_from_typevar, + allow_sequences="fixed_length", + markers=markers, + ) - # Make tuple. - out = [] - step = inner_meta.nargs if isinstance(inner_meta.nargs, int) else 1 - for i in range(0, len(strings), step): - out.append(make(strings[i : i + inner_meta.nargs])) # type: ignore - assert container_type is not None - return container_type(out) - - return sequence_instantiator, InstantiatorMetadata( - nargs="+", - metavar=_strings.multi_metavar_from_single(inner_meta.metavar), - choices=inner_meta.choices, - ) + def sequence_instantiator(strings: List[str]) -> Any: + # Validate nargs. + if ( + isinstance(inner_meta.nargs, int) + and len(strings) % inner_meta.nargs != 0 + ): + raise ValueError( + f"input {strings} is of length {len(strings)}, which is not divisible" + f" by {inner_meta.nargs}." + ) + + # Make tuple. + out = [] + step = inner_meta.nargs if isinstance(inner_meta.nargs, int) else 1 + for i in range(0, len(strings), step): + out.append(make(strings[i : i + inner_meta.nargs])) # type: ignore + assert container_type is not None + return container_type(out) + + return sequence_instantiator, InstantiatorMetadata( + nargs="+", + metavar=_strings.multi_metavar_from_single(inner_meta.metavar), + choices=inner_meta.choices, + action=None, + ) def _instantiator_from_literal( - typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]] + typ: TypeForm, + type_from_typevar: Dict[TypeVar, TypeForm[Any]], + markers: FrozenSet[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: choices = get_args(typ) str_choices = tuple(x.name if isinstance(x, enum.Enum) else str(x) for x in choices) @@ -592,5 +663,6 @@ def _instantiator_from_literal( nargs=1, metavar="{" + ",".join(str_choices) + "}", choices=str_choices, + action=None, ), ) diff --git a/tyro/conf/__init__.py b/tyro/conf/__init__.py index 3fa3556cf..472dadaa9 100644 --- a/tyro/conf/__init__.py +++ b/tyro/conf/__init__.py @@ -18,6 +18,7 @@ Positional, Suppress, SuppressFixed, + UseAppendAction, configure, ) @@ -32,5 +33,6 @@ "Positional", "Suppress", "SuppressFixed", + "UseAppendAction", "configure", ] diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index 279223583..f36e93b40 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -88,6 +88,19 @@ If subcommand prefixes are omitted, we would instead simply have `--arg`. """ +UseAppendAction = Annotated[T, None] +"""Use "append" actions for variable-length arguments. + +Given an annotation like `x: List[int]`, this means that `x = [0, 1, 2]` can be set via +the CLI syntax `--x 0 --x 1 --x 2` instead of the default of `--x 0 1 2`. + +The resulting syntax may be more user-friendly; for `tyro`, it also enables support for +otherwise ambiguous annotations like `List[List[int]]`. + +Can be applied to all variable-length sequences (`List[T]`, `Sequence[T]`, +`Tuple[T, ...]`, etc), including dictionaries without default values. +""" + CallableType = TypeVar("CallableType", bound=Callable) # Dynamically generate marker singletons. From 863eb120d5c9f766c943e60b5fcf90e42b9c802c Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 4 May 2023 23:29:37 -0700 Subject: [PATCH 282/491] More predictable boolean flag behavior (#50) * Backport argparse.BooleanOptionalAction * Backport usage functionality * No coverage for argparse backport --- tests/test_boolean_optional.py | 67 ++++++++++++++++++ tests/test_helptext.py | 12 ++++ tyro/_argparse_formatter.py | 122 +++++++++++++++++++++++++++++++++ tyro/_arguments.py | 99 ++++++++++++++++++++------ 4 files changed, 279 insertions(+), 21 deletions(-) create mode 100644 tests/test_boolean_optional.py diff --git a/tests/test_boolean_optional.py b/tests/test_boolean_optional.py new file mode 100644 index 000000000..e3358bcf2 --- /dev/null +++ b/tests/test_boolean_optional.py @@ -0,0 +1,67 @@ +import dataclasses + +import tyro + + +def test_flag_default_false() -> None: + """Test for argparse.BooleanOptionalAction-style usage.""" + + @dataclasses.dataclass + class A: + x: bool + + assert tyro.cli( + A, + args=["--x"], + default=A(False), + ) == A(True) + + assert tyro.cli( + A, + args=["--no-x"], + default=A(False), + ) == A(False) + + assert tyro.cli( + A, + args=[], + default=A(False), + ) == A(False) + + assert tyro.cli( + tyro.conf.FlagConversionOff[A], + args=["--x", "True"], + default=A(False), + ) == A(True) + + +def test_flag_default_true() -> None: + """Test for argparse.BooleanOptionalAction-style usage.""" + + @dataclasses.dataclass + class A: + x: bool + + assert tyro.cli( + A, + args=["--x"], + default=A(True), + ) == A(True) + + assert tyro.cli( + A, + args=["--no-x"], + default=A(True), + ) == A(False) + + assert tyro.cli( + A, + args=[], + default=A(True), + ) == A(True) + + assert tyro.cli( + tyro.conf.FlagConversionOff[A], + args=["--x", "True"], + default=A(False), + ) == A(True) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 506e287bd..a308c435a 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -574,3 +574,15 @@ def main(x: os.PathLike) -> None: helptext = get_helptext(main) assert "--x PATH " in helptext + + +def test_nested_bool() -> None: + @dataclasses.dataclass + class Child: + x: bool = False + + def main(child: Child) -> None: + pass + + helptext = get_helptext(main) + assert "--child.x | --child.no-x" in helptext diff --git a/tyro/_argparse_formatter.py b/tyro/_argparse_formatter.py index 035e7b4fb..874947a03 100644 --- a/tyro/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -7,11 +7,14 @@ This is largely built by fussing around in argparse implementation details, and is by far the hackiest part of `tyro`. + +TODO: we may want to just maintain our own fork of argparse. """ import argparse import contextlib import dataclasses import itertools +import re as _re import shutil from typing import Any, ContextManager, Generator, List, Optional @@ -458,3 +461,122 @@ def _tyro_format_nonroot(self): border_style=THEME.border, # padding=(1, 1, 0, 1), ) + + def _format_actions_usage(self, actions, groups): # pragma: no cover + """Backporting from Python 3.10, primarily to call format_usage() on actions.""" + + # find group indices and identify actions in groups + group_actions = set() + inserts = {} + for group in groups: + if not group._group_actions: + raise ValueError(f"empty group {group}") + + try: + start = actions.index(group._group_actions[0]) # type: ignore + except ValueError: + continue + else: + group_action_count = len(group._group_actions) + end = start + group_action_count + if actions[start:end] == group._group_actions: # type: ignore + suppressed_actions_count = 0 + for action in group._group_actions: + group_actions.add(action) + if action.help is argparse.SUPPRESS: + suppressed_actions_count += 1 + + exposed_actions_count = ( + group_action_count - suppressed_actions_count + ) + + if not group.required: + if start in inserts: + inserts[start] += " [" + else: + inserts[start] = "[" + if end in inserts: + inserts[end] += "]" + else: + inserts[end] = "]" + elif exposed_actions_count > 1: + if start in inserts: + inserts[start] += " (" + else: + inserts[start] = "(" + if end in inserts: + inserts[end] += ")" + else: + inserts[end] = ")" + for i in range(start + 1, end): + inserts[i] = "|" + + # collect all actions format strings + parts = [] + for i, action in enumerate(actions): + # suppressed arguments are marked with None + # remove | separators for suppressed arguments + if action.help is argparse.SUPPRESS: + parts.append(None) + if inserts.get(i) == "|": + inserts.pop(i) + elif inserts.get(i + 1) == "|": + inserts.pop(i + 1) + + # produce all arg strings + elif not action.option_strings: + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) + + # if it's in a group, strip the outer [] + if action in group_actions: + if part[0] == "[" and part[-1] == "]": + part = part[1:-1] + + # add the action string to the list + parts.append(part) + + # produce the first way to invoke the option in brackets + else: + option_string = action.option_strings[0] + + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = ( + action.format_usage() + if hasattr(action, "format_usage") + else "%s" % option_string + ) + + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + part = "%s %s" % (option_string, args_string) + + # make it look optional if it's not required or in a group + if not action.required and action not in group_actions: + part = "[%s]" % part + + # add the action string to the list + parts.append(part) + + # insert things at the necessary indices + for i in sorted(inserts, reverse=True): + parts[i:i] = [inserts[i]] + + # join all the action items with spaces + text = " ".join([item for item in parts if item is not None]) + + # clean up separators for mutually exclusive groups + open = r"[\[(]" + close = r"[\])]" + text = _re.sub(r"(%s) " % open, r"\1", text) + text = _re.sub(r" (%s)" % close, r"\1", text) + text = _re.sub(r"%s *%s" % (open, close), r"", text) + text = text.strip() + + # return the text + return text diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 64c9892ed..e0b47f6a7 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -8,7 +8,19 @@ import functools import itertools import shlex -from typing import Any, Dict, Mapping, Optional, Sequence, Set, Tuple, TypeVar, Union +from typing import ( + Any, + Callable, + Dict, + Iterable, + Mapping, + Optional, + Sequence, + Set, + Tuple, + TypeVar, + Union, +) import rich.markup import shtab @@ -25,6 +37,64 @@ from backports.cached_property import cached_property # type: ignore +_T = TypeVar("_T") + + +# TODO: refactor! +class BooleanOptionalAction(argparse.Action): + """Adapted from https://github.com/python/cpython/pull/27672""" + + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: _T | str | None = None, + type: Callable[[str], _T] | argparse.FileType | None = None, + choices: Iterable[_T] | None = None, + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = None, + ) -> None: + _option_strings = [] + self._no_strings = set() + for option_string in option_strings: + _option_strings.append(option_string) + + if option_string.startswith("--"): + if "." not in option_string: + option_string = "--no-" + option_string[2:] + else: + # Loose heuristic for where to add the no- prefix. + left, _, right = option_string.rpartition(".") + option_string = left + ".no-" + right + self._no_strings.add(option_string) + + _option_strings.append(option_string) + + super().__init__( + option_strings=_option_strings, + dest=dest, + nargs=0, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar, + ) + + def __call__(self, parser, namespace, values, option_string=None): + if option_string in self.option_strings: + assert option_string is not None + print(self._no_strings) + setattr(namespace, self.dest, option_string not in self._no_strings) + + # Typically only supported in Python 3.10, but we backport some functionality in + # _argparse_formatters.py + def format_usage(self): + return " | ".join(self.option_strings) + + @dataclasses.dataclass(frozen=True) class ArgumentDefinition: """Structure containing everything needed to define an argument.""" @@ -52,8 +122,11 @@ def add_argument( # MISSING value will be detected in _calling.py and the field default will # directly be used. This helps reduce the likelihood of issues with converting # the field default to a string format, then back to the desired type. - if kwargs.get("action", None) != "append": + action = kwargs.get("action", None) + if action != "append": kwargs["default"] = _fields.MISSING_NONPROP + elif action == BooleanOptionalAction: + pass else: kwargs["default"] = [] @@ -136,7 +209,7 @@ def is_fixed(self) -> bool: default: Optional[Any] = None dest: Optional[str] = None required: bool = False - action: Optional[str] = None + action: Optional[Any] = None nargs: Optional[Union[int, str]] = None choices: Optional[Set[Any]] = None # Note: unlike in vanilla argparse, our metavar is always a string. We handle @@ -173,18 +246,11 @@ def _rule_handle_boolean_flags( ): # Treat bools as a normal parameter. return lowered - elif arg.field.default is False: + elif arg.field.default in (True, False): # Default `False` => --flag passed in flips to `True`. return dataclasses.replace( lowered, - action="store_true", - instantiator=lambda x: x, # argparse will directly give us a bool! - ) - elif arg.field.default is True: - # Default `True` => --no-flag passed in flips to `False`. - return dataclasses.replace( - lowered, - action="store_false", + action=BooleanOptionalAction, instantiator=lambda x: x, # argparse will directly give us a bool! ) @@ -352,10 +418,6 @@ def _rule_generate_helptext( # Intentionally not quoted via shlex, since this can't actually be passed # in via the commandline. default_text = f"(fixed to: {str(arg.field.default)})" - elif lowered.action == "store_true": - default_text = f"(sets: {arg.field.name}=True)" - elif lowered.action == "store_false": - default_text = f"(sets: {arg.field.name}=False)" elif lowered.action == "append" and ( arg.field.default in _fields.MISSING_SINGLETONS or len(arg.field.default) == 0 @@ -393,11 +455,6 @@ def _rule_set_name_or_flag_and_dest( # Positional arguments: no -- prefix. if arg.field.is_positional(): name_or_flag = _strings.make_field_name([arg.name_prefix, arg.field.name]) - # Negated booleans. - elif lowered.action == "store_false": - name_or_flag = "--" + _strings.make_field_name( - [arg.name_prefix, "no-" + arg.field.name] - ) # Prefix keyword arguments with --. else: name_or_flag = "--" + _strings.make_field_name( From bfcd212ab5838fb627a49b572ad1ed0687c284fc Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 4 May 2023 23:42:48 -0700 Subject: [PATCH 283/491] Add `UseAppendAction` note in nested sequence error messages --- tyro/_instantiators.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 4e4794ab7..65989f237 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -290,7 +290,9 @@ def _instantiator_from_type_inner( assert allow_sequences if allow_sequences == "fixed_length" and not isinstance(out[1].nargs, int): raise UnsupportedTypeAnnotationError( - f"Found an unsupported (variable-length) nested sequence of type {typ}." + f"{typ} is a variable-length sequence, which is ambiguous when nested." + " For nesting variable-length sequences (example: List[List[int]])," + " `tyro.conf.UseAppendAction` can help resolve ambiguities." ) return out @@ -628,8 +630,8 @@ def sequence_instantiator(strings: List[str]) -> Any: and len(strings) % inner_meta.nargs != 0 ): raise ValueError( - f"input {strings} is of length {len(strings)}, which is not divisible" - f" by {inner_meta.nargs}." + f"input {strings} is of length {len(strings)}, which is not" + f" divisible by {inner_meta.nargs}." ) # Make tuple. From a7de7e3db04fb2b375885339d42c33bf56ad1d04 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 4 May 2023 23:46:53 -0700 Subject: [PATCH 284/491] Bump to 0.5.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 87ee2786b..f49120257 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.4.3" +version = "0.5.0" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] From f4b9a7088b8afb5418d54cab7f768015a2c0765b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 5 May 2023 00:19:31 -0700 Subject: [PATCH 285/491] Add `prefix_name` field to `tyro.conf.arg` --- tests/test_conf.py | 45 +++++++++++++++++++++++++++++++++++++ tyro/_argparse_formatter.py | 4 +--- tyro/_arguments.py | 26 +++++++++++++-------- tyro/_calling.py | 13 ++++++++--- tyro/_fields.py | 2 +- tyro/_instantiators.py | 3 ++- tyro/_parsers.py | 12 +++++----- tyro/conf/_confstruct.py | 10 +++++++-- 8 files changed, 90 insertions(+), 25 deletions(-) diff --git a/tests/test_conf.py b/tests/test_conf.py index 99f34853e..7c91a6884 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -530,6 +530,35 @@ def main(x: Any = Struct()) -> int: assert tyro.cli(main, args=["--x.nice", "3"]) == 3 +def test_argconf_no_prefix_help() -> None: + @dataclasses.dataclass + class Struct: + a: Annotated[ + int, + tyro.conf.arg( + name="nice", help="Hello world", metavar="NUMBER", prefix_name=False + ), + ] = 5 + b: tyro.conf.Suppress[str] = "7" + + def main(x: Any = Struct()) -> int: + return x.a + + helptext = get_helptext(main) + assert "Hello world" in helptext + assert "INT" not in helptext + assert "NUMBER" in helptext + assert "--x.a" not in helptext + assert "--x.nice" not in helptext + assert "--nice" in helptext + assert "--x.b" not in helptext + + assert tyro.cli(main, args=[]) == 5 + with pytest.raises(SystemExit): + assert tyro.cli(main, args=["--x.nice", "3"]) == 3 + assert tyro.cli(main, args=["--nice", "3"]) == 3 + + def test_positional() -> None: def main(x: tyro.conf.Positional[int], y: int) -> int: return x + y @@ -822,3 +851,19 @@ class A: x={"0": {"1": 2, "3": 4}, "4": {"5": 6}} ) assert tyro.cli(A, args=[]) == A(x={}) + + +def test_duplicated_arg() -> None: + # Loosely inspired by: https://github.com/brentyi/tyro/issues/49 + @dataclasses.dataclass + class ModelConfig: + num_slots: Annotated[int, tyro.conf.arg(name="num_slots", prefix_name=False)] + + @dataclasses.dataclass + class TrainConfig: + num_slots: int + model: ModelConfig + + assert tyro.cli(TrainConfig, args="--num-slots 3".split(" ")) == TrainConfig( + num_slots=3, model=ModelConfig(num_slots=3) + ) diff --git a/tyro/_argparse_formatter.py b/tyro/_argparse_formatter.py index 874947a03..eb9f19787 100644 --- a/tyro/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -56,9 +56,7 @@ def set_accent_color(accent_color: Optional[str]) -> None: THEME.helptext = Style(dim=True) THEME.helptext_required = Style(color="bright_red", bold=True) THEME.helptext_default = Style( - color="cyan" - if accent_color != "cyan" - else "magenta" + color="cyan" if accent_color != "cyan" else "magenta" # Another option: make default color match accent color. This is maybe more # visually consistent, but harder to read. # color=accent_color if accent_color is not None else "cyan", diff --git a/tyro/_arguments.py b/tyro/_arguments.py index e0b47f6a7..ee3ab753e 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -136,7 +136,10 @@ def add_argument( kwargs["metavar"] = self.field.argconf.metavar # Add argument! Note that the name must be passed in as a position argument. - arg = parser.add_argument(name_or_flag, **kwargs) + try: + arg = parser.add_argument(name_or_flag, **kwargs) + except argparse.ArgumentError: + return # Do our best to tab complete paths. # There will be false positives here, but if choices is unset they should be @@ -452,14 +455,15 @@ def _rule_set_name_or_flag_and_dest( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: - # Positional arguments: no -- prefix. - if arg.field.is_positional(): - name_or_flag = _strings.make_field_name([arg.name_prefix, arg.field.name]) + name_or_flag = _strings.make_field_name( + [arg.name_prefix, arg.field.name] + if arg.field.argconf.prefix_name + else [arg.field.name] + ) + # Prefix keyword arguments with --. - else: - name_or_flag = "--" + _strings.make_field_name( - [arg.name_prefix, arg.field.name] - ) + if not arg.field.is_positional(): + name_or_flag = "--" + name_or_flag # Strip. if name_or_flag.startswith("--") and arg.subcommand_prefix != "": @@ -472,7 +476,11 @@ def _rule_set_name_or_flag_and_dest( return dataclasses.replace( lowered, name_or_flag=name_or_flag, - dest=_strings.make_field_name([arg.dest_prefix, arg.field.name]), + dest=( + _strings.make_field_name([arg.dest_prefix, arg.field.name]) + if arg.field.argconf.prefix_name + else arg.field.name + ), ) diff --git a/tyro/_calling.py b/tyro/_calling.py index e38dc3861..ddb2dd7ea 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -42,7 +42,9 @@ def call_from_args( def get_value_from_arg(prefixed_field_name: str) -> Any: """Helper for getting values from `value_from_arg` + doing some extra asserts.""" - assert prefixed_field_name in value_from_prefixed_field_name + assert ( + prefixed_field_name in value_from_prefixed_field_name + ), f"{prefixed_field_name} not in {value_from_prefixed_field_name}" return value_from_prefixed_field_name[prefixed_field_name] arg_from_prefixed_field_name: Dict[str, _arguments.ArgumentDefinition] = {} @@ -63,9 +65,14 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: # Standard arguments. arg = arg_from_prefixed_field_name[prefixed_field_name] - consumed_keywords.add(prefixed_field_name) + name_maybe_prefixed = ( + prefixed_field_name + if field.argconf.prefix_name + else _strings.make_field_name([field.name]) + ) + consumed_keywords.add(name_maybe_prefixed) if not arg.lowered.is_fixed(): - value = get_value_from_arg(prefixed_field_name) + value = get_value_from_arg(name_maybe_prefixed) if value in _fields.MISSING_SINGLETONS: value = arg.field.default diff --git a/tyro/_fields.py b/tyro/_fields.py index f4df8f1ee..198a02c9b 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -80,7 +80,7 @@ def make( # Try to extract argconf overrides from type. _, argconfs = _resolver.unwrap_annotated(typ, _confstruct._ArgConfiguration) if len(argconfs) == 0: - argconf = _confstruct._ArgConfiguration(None, None, None) + argconf = _confstruct._ArgConfiguration(None, None, None, True) else: assert len(argconfs) == 1 (argconf,) = argconfs diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 65989f237..2fddb6237 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -486,7 +486,8 @@ def union_instantiator(strings: List[str]) -> Any: ) raise ValueError( f"no type in {options} could be instantiated from" - f" {strings}.\n\nGot errors: \n- " + "\n- ".join(errors) + f" {strings}.\n\nGot errors: \n- " + + "\n- ".join(errors) ) return union_instantiator, InstantiatorMetadata( diff --git a/tyro/_parsers.py b/tyro/_parsers.py index b56f4d932..727add96e 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -126,9 +126,9 @@ def from_callable_or_type( # Don't make a subparser. field = dataclasses.replace(field, typ=type(field.default)) else: - subparsers_from_prefix[ - subparsers_attempt.prefix - ] = subparsers_attempt + subparsers_from_prefix[subparsers_attempt.prefix] = ( + subparsers_attempt + ) subparsers = add_subparsers_to_leaves( subparsers, subparsers_attempt ) @@ -345,9 +345,9 @@ def from_field( return None # Get subcommand configurations from `tyro.conf.subcommand()`. - subcommand_config_from_name: Dict[ - str, _confstruct._SubcommandConfiguration - ] = {} + subcommand_config_from_name: Dict[str, _confstruct._SubcommandConfiguration] = ( + {} + ) subcommand_type_from_name: Dict[str, type] = {} for option in options_no_none: subcommand_name = _strings.subparser_name_from_type(prefix, option) diff --git a/tyro/conf/_confstruct.py b/tyro/conf/_confstruct.py index 887587dec..1cf4f0454 100644 --- a/tyro/conf/_confstruct.py +++ b/tyro/conf/_confstruct.py @@ -60,7 +60,7 @@ class _ArgConfiguration: name: Optional[str] metavar: Optional[str] help: Optional[str] - # TODO - add prefix_name: bool + prefix_name: bool def arg( @@ -68,6 +68,7 @@ def arg( name: Optional[str] = None, metavar: Optional[str] = None, help: Optional[str] = None, + prefix_name: bool = True, ) -> Any: """Returns a metadata object for configuring arguments with `typing.Annotated`. Useful for aesthetics. @@ -77,4 +78,9 @@ def arg( x: Annotated[int, tyro.conf.arg(...)] ``` """ - return _ArgConfiguration(name=name, metavar=metavar, help=help) + return _ArgConfiguration( + name=name, + metavar=metavar, + help=help, + prefix_name=prefix_name, + ) From 378b41ccb91ffb69be65a2d6c17cf3293db335d7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 5 May 2023 01:00:22 -0700 Subject: [PATCH 286/491] Format --- tyro/_argparse_formatter.py | 4 +++- tyro/_instantiators.py | 3 +-- tyro/_parsers.py | 12 ++++++------ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tyro/_argparse_formatter.py b/tyro/_argparse_formatter.py index eb9f19787..874947a03 100644 --- a/tyro/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -56,7 +56,9 @@ def set_accent_color(accent_color: Optional[str]) -> None: THEME.helptext = Style(dim=True) THEME.helptext_required = Style(color="bright_red", bold=True) THEME.helptext_default = Style( - color="cyan" if accent_color != "cyan" else "magenta" + color="cyan" + if accent_color != "cyan" + else "magenta" # Another option: make default color match accent color. This is maybe more # visually consistent, but harder to read. # color=accent_color if accent_color is not None else "cyan", diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 2fddb6237..65989f237 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -486,8 +486,7 @@ def union_instantiator(strings: List[str]) -> Any: ) raise ValueError( f"no type in {options} could be instantiated from" - f" {strings}.\n\nGot errors: \n- " - + "\n- ".join(errors) + f" {strings}.\n\nGot errors: \n- " + "\n- ".join(errors) ) return union_instantiator, InstantiatorMetadata( diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 727add96e..b56f4d932 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -126,9 +126,9 @@ def from_callable_or_type( # Don't make a subparser. field = dataclasses.replace(field, typ=type(field.default)) else: - subparsers_from_prefix[subparsers_attempt.prefix] = ( - subparsers_attempt - ) + subparsers_from_prefix[ + subparsers_attempt.prefix + ] = subparsers_attempt subparsers = add_subparsers_to_leaves( subparsers, subparsers_attempt ) @@ -345,9 +345,9 @@ def from_field( return None # Get subcommand configurations from `tyro.conf.subcommand()`. - subcommand_config_from_name: Dict[str, _confstruct._SubcommandConfiguration] = ( - {} - ) + subcommand_config_from_name: Dict[ + str, _confstruct._SubcommandConfiguration + ] = {} subcommand_type_from_name: Dict[str, type] = {} for option in options_no_none: subcommand_name = _strings.subparser_name_from_type(prefix, option) From 3c3e9f44cf2ce3b63e97913931c848a4efe563d1 Mon Sep 17 00:00:00 2001 From: Costa Huang Date: Fri, 5 May 2023 16:59:21 -0400 Subject: [PATCH 287/491] Remove debugging print statement (#51) --- tyro/_arguments.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tyro/_arguments.py b/tyro/_arguments.py index ee3ab753e..23f34d671 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -86,7 +86,6 @@ def __init__( def __call__(self, parser, namespace, values, option_string=None): if option_string in self.option_strings: assert option_string is not None - print(self._no_strings) setattr(namespace, self.dest, option_string not in self._no_strings) # Typically only supported in Python 3.10, but we backport some functionality in From 4f9b3d344de1702b12b6b71177725cf3205f9b9b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 5 May 2023 13:59:44 -0700 Subject: [PATCH 288/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f49120257..e88e15378 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.5.0" +version = "0.5.1" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] From 1d27a599c30963c1baf85bd48660499befbdf244 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 9 May 2023 15:18:36 -0700 Subject: [PATCH 289/491] Add `tyro.conf.OmitArgPrefixes` cc https://github.com/openrlbenchmark/openrlbenchmark/issues/15 --- tests/test_conf.py | 28 ++++++++++++++++++++++++---- tyro/_arguments.py | 14 ++++---------- tyro/_calling.py | 6 +----- tyro/conf/__init__.py | 41 +++++++++++++---------------------------- tyro/conf/_markers.py | 11 +++++++++++ 5 files changed, 53 insertions(+), 47 deletions(-) diff --git a/tests/test_conf.py b/tests/test_conf.py index 7c91a6884..fdf685c63 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1,3 +1,4 @@ +import argparse import dataclasses from typing import Any, Dict, Generic, List, Tuple, TypeVar, Union @@ -857,13 +858,32 @@ def test_duplicated_arg() -> None: # Loosely inspired by: https://github.com/brentyi/tyro/issues/49 @dataclasses.dataclass class ModelConfig: - num_slots: Annotated[int, tyro.conf.arg(name="num_slots", prefix_name=False)] + num_slots: Annotated[int, tyro.conf.arg(prefix_name=False)] @dataclasses.dataclass class TrainConfig: num_slots: int model: ModelConfig - assert tyro.cli(TrainConfig, args="--num-slots 3".split(" ")) == TrainConfig( - num_slots=3, model=ModelConfig(num_slots=3) - ) + with pytest.raises(argparse.ArgumentError): + tyro.cli(TrainConfig, args="--num-slots 3".split(" ")) + + +def test_omit_arg_prefixes() -> None: + # Loosely inspired by: https://github.com/brentyi/tyro/issues/49 + @dataclasses.dataclass + class ModelConfig: + num_slots: int + + @dataclasses.dataclass + class TrainConfig: + model: ModelConfig + + assert tyro.cli( + tyro.conf.OmitSubcommandPrefixes[TrainConfig], + args="--model.num-slots 3".split(" "), + ) == TrainConfig(ModelConfig(num_slots=3)) + + assert tyro.cli( + tyro.conf.OmitArgPrefixes[TrainConfig], args="--num-slots 3".split(" ") + ) == TrainConfig(ModelConfig(num_slots=3)) diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 23f34d671..0d503d8f4 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -134,11 +134,8 @@ def add_argument( if self.field.argconf.metavar is not None: kwargs["metavar"] = self.field.argconf.metavar - # Add argument! Note that the name must be passed in as a position argument. - try: - arg = parser.add_argument(name_or_flag, **kwargs) - except argparse.ArgumentError: - return + # Add argument! + arg = parser.add_argument(name_or_flag, **kwargs) # Do our best to tab complete paths. # There will be false positives here, but if choices is unset they should be @@ -457,6 +454,7 @@ def _rule_set_name_or_flag_and_dest( name_or_flag = _strings.make_field_name( [arg.name_prefix, arg.field.name] if arg.field.argconf.prefix_name + and _markers.OmitArgPrefixes not in arg.field.markers else [arg.field.name] ) @@ -475,11 +473,7 @@ def _rule_set_name_or_flag_and_dest( return dataclasses.replace( lowered, name_or_flag=name_or_flag, - dest=( - _strings.make_field_name([arg.dest_prefix, arg.field.name]) - if arg.field.argconf.prefix_name - else arg.field.name - ), + dest=_strings.make_field_name([arg.dest_prefix, arg.field.name]), ) diff --git a/tyro/_calling.py b/tyro/_calling.py index ddb2dd7ea..d26e99429 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -65,11 +65,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: # Standard arguments. arg = arg_from_prefixed_field_name[prefixed_field_name] - name_maybe_prefixed = ( - prefixed_field_name - if field.argconf.prefix_name - else _strings.make_field_name([field.name]) - ) + name_maybe_prefixed = prefixed_field_name consumed_keywords.add(name_maybe_prefixed) if not arg.lowered.is_fixed(): value = get_value_from_arg(name_maybe_prefixed) diff --git a/tyro/conf/__init__.py b/tyro/conf/__init__.py index 472dadaa9..1a1ba8c58 100644 --- a/tyro/conf/__init__.py +++ b/tyro/conf/__init__.py @@ -8,31 +8,16 @@ Features here are supported, but generally unnecessary and should be used sparingly. """ -from ._confstruct import arg, subcommand -from ._markers import ( - AvoidSubcommands, - ConsolidateSubcommandArgs, - Fixed, - FlagConversionOff, - OmitSubcommandPrefixes, - Positional, - Suppress, - SuppressFixed, - UseAppendAction, - configure, -) - -__all__ = [ - "arg", - "subcommand", - "AvoidSubcommands", - "ConsolidateSubcommandArgs", - "Fixed", - "FlagConversionOff", - "OmitSubcommandPrefixes", - "Positional", - "Suppress", - "SuppressFixed", - "UseAppendAction", - "configure", -] +from ._confstruct import arg as arg +from ._confstruct import subcommand as subcommand +from ._markers import AvoidSubcommands as AvoidSubcommands +from ._markers import ConsolidateSubcommandArgs as ConsolidateSubcommandArgs +from ._markers import Fixed as Fixed +from ._markers import FlagConversionOff as FlagConversionOff +from ._markers import OmitArgPrefixes as OmitArgPrefixes +from ._markers import OmitSubcommandPrefixes as OmitSubcommandPrefixes +from ._markers import Positional as Positional +from ._markers import Suppress as Suppress +from ._markers import SuppressFixed as SuppressFixed +from ._markers import UseAppendAction as UseAppendAction +from ._markers import configure as configure diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index f36e93b40..2d8eecb9f 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -88,6 +88,17 @@ If subcommand prefixes are omitted, we would instead simply have `--arg`. """ +OmitArgPrefixes = Annotated[T, None] +"""Make flags used for keyword arguments in arguments shorter by omitting prefixes. + +If we have a structure with the field: + + cmd: NestedType + +By default, `--cmd.arg` may be generated as a flag. If prefixes are omitted, we would +instead simply have `--arg`. +""" + UseAppendAction = Annotated[T, None] """Use "append" actions for variable-length arguments. From e25a0207d42a41c47237e04d57d6148836a0a5ab Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 10 May 2023 00:26:38 -0700 Subject: [PATCH 290/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e88e15378..9dbdcd04a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.5.1" +version = "0.5.2" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] From b92fc8b8a15d44e8a9d9f0ac0e8e1d7245865105 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 10 May 2023 21:38:39 -0700 Subject: [PATCH 291/491] Update _markers.py --- tyro/conf/_markers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index 2d8eecb9f..d791b8bb7 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -78,7 +78,8 @@ """ OmitSubcommandPrefixes = Annotated[T, None] -"""Make flags used for keyword arguments in subcommands shorter by omitting prefixes. +"""Make flags used for keyword arguments in subcommands shorter by omitting the +subcommand-specific portion of the prefix. If we have a structure with the field: @@ -89,7 +90,7 @@ """ OmitArgPrefixes = Annotated[T, None] -"""Make flags used for keyword arguments in arguments shorter by omitting prefixes. +"""Make flags used for keyword arguments shorter by omitting prefixes. If we have a structure with the field: From 3de5c7f8220c0552db85a0fdd26faabe5f1ce31b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 12 May 2023 10:55:51 -0700 Subject: [PATCH 292/491] Set `allow_abbrev=False` --- tyro/_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tyro/_cli.py b/tyro/_cli.py index 7efa87438..af2aa1e06 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -369,6 +369,7 @@ def _cli_impl( parser = argparse.ArgumentParser( prog=prog, formatter_class=_argparse_formatter.TyroArgparseHelpFormatter, + allow_abbrev=False, ) parser_definition.apply(parser) From aa5d1871a4658d35c30fe312da585afa45d75fde Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 15 May 2023 11:09:49 -0700 Subject: [PATCH 293/491] Docs formatting fix --- tyro/extras/_base_configs.py | 17 +++++++++++------ tyro/extras/_choices_type.py | 12 ++++++++---- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/tyro/extras/_base_configs.py b/tyro/extras/_base_configs.py index 2ed3e5b35..083bb2c99 100644 --- a/tyro/extras/_base_configs.py +++ b/tyro/extras/_base_configs.py @@ -40,15 +40,13 @@ def subcommand_type_from_defaults( ] ``` - Direct use of `typing.Union` and `tyro.conf.subcommand()` should generally be - preferred, but this function can be helpful for succinictness. - + Direct use of `typing.Union` and :func:`tyro.conf.subcommand()` should generally be + preferred, but this function can be helpful for succinctness. .. warning:: - The type returned by this function can be safely used as an input to - `tyro.cli()`, but for static analysis when used for annotations we recommend - applying a TYPE_CHECKING guard: + :func:`tyro.cli()`, but for static analysis when used for annotations we + recommend applying a `TYPE_CHECKING` guard: .. code-block:: python @@ -61,6 +59,13 @@ def subcommand_type_from_defaults( # Runtime type used by tyro. SelectableConfig = subcommand_type_from_defaults(...) + Args: + defaults: A dictionary of default subcommand instances. + descriptions: A dictionary conttaining descriptions for helptext. + prefix_names: Whether to prefix subcommand names. + + Returns: + A subcommand type, which can be passed to :func:`tyro.cli`. """ return Union.__getitem__( # type: ignore tuple( diff --git a/tyro/extras/_choices_type.py b/tyro/extras/_choices_type.py index 68da8cb61..57998afc2 100644 --- a/tyro/extras/_choices_type.py +++ b/tyro/extras/_choices_type.py @@ -14,12 +14,10 @@ def literal_type_from_choices(choices: Iterable[T]) -> TypeForm[T]: Using `Literal[...]` directly should generally be preferred, but this function can be helpful when choices are generated dynamically. - .. warning:: - The type returned by this function can be safely used as an input to - `tyro.cli()`, but for static analysis when used for annotations we recommend - applying a TYPE_CHECKING guard: + :func:`tyro.cli()`, but for static analysis when used for annotations we + recommend applying a `TYPE_CHECKING` guard: .. code-block:: python @@ -31,5 +29,11 @@ def literal_type_from_choices(choices: Iterable[T]) -> TypeForm[T]: else: # Runtime type used by tyro. Color = literal_type_from_choices(["red", "green", "blue"]) + + Args: + choices: Options to choose from. + + Returns: + A type that can be passed to :func:`tyro.cli()`. """ return Literal.__getitem__(tuple(choices)) # type: ignore From 3c6e9e2e1944272da9ab54d2990b65485cf3f961 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 27 May 2023 02:24:46 -0700 Subject: [PATCH 294/491] Fix typing_extensions Literal bug --- poetry.lock | 1775 ++++++++++++++++++++-------------------- pyproject.toml | 2 +- tyro/_instantiators.py | 9 +- 3 files changed, 891 insertions(+), 895 deletions(-) diff --git a/poetry.lock b/poetry.lock index eb85007b9..11ef9b2ab 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,56 +1,75 @@ +# This file is automatically @generated by Poetry 1.5.0 and should not be changed by hand. + [[package]] name = "absl-py" -version = "1.3.0" +version = "1.4.0" description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "absl-py-1.4.0.tar.gz", hash = "sha256:d2c244d01048ba476e7c080bd2c6df5e141d211de80223460d5b3b8a2a58433d"}, + {file = "absl_py-1.4.0-py3-none-any.whl", hash = "sha256:0d3fe606adfa4f7db64792dd4c7aee4ee0c38ab75dfd353b7a83ed3e957fcb47"}, +] [[package]] name = "antlr4-python3-runtime" version = "4.9.3" description = "ANTLR 4.9.3 runtime for Python 3.7" -category = "dev" optional = false python-versions = "*" +files = [ + {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, +] [[package]] name = "attrs" version = "21.4.0" description = "Classes Without Boilerplate" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, + {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, +] [package.extras] dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "sphinx", "sphinx-notfound-page", "zope.interface"] docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "zope.interface"] -tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six"] [[package]] -name = "backports.cached-property" +name = "backports-cached-property" version = "1.0.2" description = "cached_property() - computed once per instance, cached as attribute" -category = "main" optional = false python-versions = ">=3.6.0" +files = [ + {file = "backports.cached-property-1.0.2.tar.gz", hash = "sha256:9306f9eed6ec55fd156ace6bc1094e2c86fae5fb2bf07b6a9c00745c656e75dd"}, + {file = "backports.cached_property-1.0.2-py3-none-any.whl", hash = "sha256:baeb28e1cd619a3c9ab8941431fe34e8490861fb998c6c4590693d50171db0cc"}, +] [[package]] name = "cached-property" version = "1.5.2" description = "A decorator for caching properties in classes." -category = "dev" optional = false python-versions = "*" +files = [ + {file = "cached-property-1.5.2.tar.gz", hash = "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130"}, + {file = "cached_property-1.5.2-py2.py3-none-any.whl", hash = "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0"}, +] [[package]] name = "chex" version = "0.1.5" description = "Chex: Testing made fun, in JAX!" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "chex-0.1.5-py3-none-any.whl", hash = "sha256:b3321184850d5fc29b2eca63087cdbdd83a1b3e4f33c1314ff8b3b8bd67abbca"}, + {file = "chex-0.1.5.tar.gz", hash = "sha256:686858320f8f220c82a6c7eeb54dcdcaa4f3d7f66690dacd13a24baa1ee8299e"}, +] [package.dependencies] absl-py = ">=0.9.0" @@ -64,28 +83,71 @@ toolz = ">=0.9.0" name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" - -[[package]] -name = "commonmark" -version = "0.9.1" -description = "Python parser for the CommonMark Markdown spec" -category = "main" -optional = false -python-versions = "*" - -[package.extras] -test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] [[package]] name = "coverage" version = "6.5.0" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, + {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83516205e254a0cb77d2d7bb3632ee019d93d9f4005de31dca0a8c3667d5bc04"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:97117225cdd992a9c2a5515db1f66b59db634f59d0679ca1fa3fe8da32749cae"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1170fa54185845505fbfa672f1c1ab175446c887cce8212c44149581cf2d466"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:11b990d520ea75e7ee8dcab5bc908072aaada194a794db9f6d7d5cfd19661e5a"}, + {file = "coverage-6.5.0-cp310-cp310-win32.whl", hash = "sha256:5dbec3b9095749390c09ab7c89d314727f18800060d8d24e87f01fb9cfb40b32"}, + {file = "coverage-6.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:59f53f1dc5b656cafb1badd0feb428c1e7bc19b867479ff72f7a9dd9b479f10e"}, + {file = "coverage-6.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33a7da4376d5977fbf0a8ed91c4dffaaa8dbf0ddbf4c8eea500a2486d8bc4d7b"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a6b7d95969b8845250586f269e81e5dfdd8ff828ddeb8567a4a2eaa7313460c4"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1ef221513e6f68b69ee9e159506d583d31aa3567e0ae84eaad9d6ec1107dddaa"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cca4435eebea7962a52bdb216dec27215d0df64cf27fc1dd538415f5d2b9da6b"}, + {file = "coverage-6.5.0-cp311-cp311-win32.whl", hash = "sha256:98e8a10b7a314f454d9eff4216a9a94d143a7ee65018dd12442e898ee2310578"}, + {file = "coverage-6.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:bc8ef5e043a2af066fa8cbfc6e708d58017024dc4345a1f9757b329a249f041b"}, + {file = "coverage-6.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4433b90fae13f86fafff0b326453dd42fc9a639a0d9e4eec4d366436d1a41b6d"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4f05d88d9a80ad3cac6244d36dd89a3c00abc16371769f1340101d3cb899fc3"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:94e2565443291bd778421856bc975d351738963071e9b8839ca1fc08b42d4bef"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:027018943386e7b942fa832372ebc120155fd970837489896099f5cfa2890f79"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:255758a1e3b61db372ec2736c8e2a1fdfaf563977eedbdf131de003ca5779b7d"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:851cf4ff24062c6aec510a454b2584f6e998cada52d4cb58c5e233d07172e50c"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:12adf310e4aafddc58afdb04d686795f33f4d7a6fa67a7a9d4ce7d6ae24d949f"}, + {file = "coverage-6.5.0-cp37-cp37m-win32.whl", hash = "sha256:b5604380f3415ba69de87a289a2b56687faa4fe04dbee0754bfcae433489316b"}, + {file = "coverage-6.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4a8dbc1f0fbb2ae3de73eb0bdbb914180c7abfbf258e90b311dcd4f585d44bd2"}, + {file = "coverage-6.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c"}, + {file = "coverage-6.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:20c8ac5386253717e5ccc827caad43ed66fea0efe255727b1053a8154d952398"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dbdb91cd8c048c2b09eb17713b0c12a54fbd587d79adcebad543bc0cd9a3410b"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:de3001a203182842a4630e7b8d1a2c7c07ec1b45d3084a83d5d227a3806f530f"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e07f4a4a9b41583d6eabec04f8b68076ab3cd44c20bd29332c6572dda36f372e"}, + {file = "coverage-6.5.0-cp38-cp38-win32.whl", hash = "sha256:6d4817234349a80dbf03640cec6109cd90cba068330703fa65ddf56b60223a6d"}, + {file = "coverage-6.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:7ccf362abd726b0410bf8911c31fbf97f09f8f1061f8c1cf03dfc4b6372848f6"}, + {file = "coverage-6.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745"}, + {file = "coverage-6.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:265de0fa6778d07de30bcf4d9dc471c3dc4314a23a3c6603d356a3c9abc2dfcf"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7b6be138d61e458e18d8e6ddcddd36dd96215edfe5f1168de0b1b32635839b62"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:42eafe6778551cf006a7c43153af1211c3aaab658d4d66fa5fcc021613d02518"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:723e8130d4ecc8f56e9a611e73b31219595baa3bb252d539206f7bbbab6ffc1f"}, + {file = "coverage-6.5.0-cp39-cp39-win32.whl", hash = "sha256:d9ecf0829c6a62b9b573c7bb6d4dcd6ba8b6f80be9ba4fc7ed50bf4ac9aecd72"}, + {file = "coverage-6.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc2af30ed0d5ae0b1abdb4ebdce598eafd5b35397d4d75deb341a614d333d987"}, + {file = "coverage-6.5.0-pp36.pp37.pp38-none-any.whl", hash = "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a"}, + {file = "coverage-6.5.0.tar.gz", hash = "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84"}, +] [package.dependencies] tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} @@ -97,33 +159,82 @@ toml = ["tomli"] name = "cycler" version = "0.11.0" description = "Composable style cycles" -category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "cycler-0.11.0-py3-none-any.whl", hash = "sha256:3a27e95f763a428a739d2add979fa7494c912a32c17c4c38c4d5f082cad165a3"}, + {file = "cycler-0.11.0.tar.gz", hash = "sha256:9c87405839a19696e837b3b818fed3f5f69f16f1eec1a1ad77e043dcea9c772f"}, +] [[package]] name = "dm-tree" version = "0.1.8" description = "Tree is a library for working with nested data structures." -category = "dev" optional = false python-versions = "*" +files = [ + {file = "dm-tree-0.1.8.tar.gz", hash = "sha256:0fcaabbb14e7980377439e7140bd05552739ca5e515ecb3119f234acee4b9430"}, + {file = "dm_tree-0.1.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:35cc164a79336bfcfafb47e5f297898359123bbd3330c1967f0c4994f9cf9f60"}, + {file = "dm_tree-0.1.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39070ba268c0491af9fe7a58644d99e8b4f2cde6e5884ba3380bddc84ed43d5f"}, + {file = "dm_tree-0.1.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2869228d9c619074de501a3c10dc7f07c75422f8fab36ecdcb859b6f1b1ec3ef"}, + {file = "dm_tree-0.1.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d20f2faa3672b52e5013f4077117bfb99c4cfc0b445d3bde1584c34032b57436"}, + {file = "dm_tree-0.1.8-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5483dca4d7eb1a0d65fe86d3b6a53ae717face83c1f17e0887b1a4a64ae5c410"}, + {file = "dm_tree-0.1.8-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1d7c26e431fc93cc7e0cba867eb000db6a05f6f2b25af11ac4e9dada88fc5bca"}, + {file = "dm_tree-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4d714371bb08839e4e5e29024fc95832d9affe129825ef38836b143028bd144"}, + {file = "dm_tree-0.1.8-cp310-cp310-win_amd64.whl", hash = "sha256:d40fa4106ca6edc66760246a08f500ec0c85ef55c762fb4a363f6ee739ba02ee"}, + {file = "dm_tree-0.1.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad16ceba90a56ec47cf45b21856d14962ac314787975ef786efb5e6e9ca75ec7"}, + {file = "dm_tree-0.1.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:803bfc53b4659f447ac694dbd04235f94a73ef7c1fd1e0df7c84ac41e0bc963b"}, + {file = "dm_tree-0.1.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:378cc8ad93c5fe3590f405a309980721f021c790ca1bdf9b15bb1d59daec57f5"}, + {file = "dm_tree-0.1.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1607ce49aa42f010d1e5e616d92ce899d66835d4d8bea49679582435285515de"}, + {file = "dm_tree-0.1.8-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:343a4a4ebaa127451ff971254a4be4084eb4bdc0b2513c32b46f6f728fd03f9e"}, + {file = "dm_tree-0.1.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d"}, + {file = "dm_tree-0.1.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83b7764de0d855338abefc6e3ee9fe40d301668310aa3baea3f778ff051f4393"}, + {file = "dm_tree-0.1.8-cp311-cp311-win_amd64.whl", hash = "sha256:a5d819c38c03f0bb5b3b3703c60e4b170355a0fc6b5819325bf3d4ceb3ae7e80"}, + {file = "dm_tree-0.1.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8c60a7eadab64c2278861f56bca320b2720f163dca9d7558103c3b77f2416571"}, + {file = "dm_tree-0.1.8-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af4b3d372f2477dcd89a6e717e4a575ca35ccc20cc4454a8a4b6f8838a00672d"}, + {file = "dm_tree-0.1.8-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de287fabc464b8734be251e46e06aa9aa1001f34198da2b6ce07bd197172b9cb"}, + {file = "dm_tree-0.1.8-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:054b461f8176f4bce7a21f7b1870f873a1ced3bdbe1282c816c550bb43c71fa6"}, + {file = "dm_tree-0.1.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f7915660f59c09068e428613c480150180df1060561fd0d1470684ae7007bd1"}, + {file = "dm_tree-0.1.8-cp37-cp37m-win_amd64.whl", hash = "sha256:b9f89a454e98806b44fe9d40ec9eee61f848388f7e79ac2371a55679bd5a3ac6"}, + {file = "dm_tree-0.1.8-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0e9620ccf06393eb6b613b5e366469304622d4ea96ae6540b28a33840e6c89cf"}, + {file = "dm_tree-0.1.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b095ba4f8ca1ba19350fd53cf1f8f3eb0bd406aa28af64a6dfc86707b32a810a"}, + {file = "dm_tree-0.1.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b9bd9b9ccb59409d33d51d84b7668010c04c2af7d4a371632874c1ca356cff3d"}, + {file = "dm_tree-0.1.8-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d3172394079a86c3a759179c65f64c48d1a42b89495fcf38976d11cc3bb952c"}, + {file = "dm_tree-0.1.8-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1612fcaecd79023dbc6a6ae48d51a80beb5c385d6f3f6d71688e57bc8d07de8"}, + {file = "dm_tree-0.1.8-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c5c8c12e3fda754ef6af94161bacdaeda816d941995fac415d6855c6c386af68"}, + {file = "dm_tree-0.1.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:694c3654cfd2a81552c08ec66bb5c4a3d48fa292b9a181880fb081c36c5b9134"}, + {file = "dm_tree-0.1.8-cp38-cp38-win_amd64.whl", hash = "sha256:bb2d109f42190225112da899b9f3d46d0d5f26aef501c61e43529fe9322530b5"}, + {file = "dm_tree-0.1.8-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d16e1f2a073604cfcc09f7131ae8d534674f43c3aef4c25742eae295bc60d04f"}, + {file = "dm_tree-0.1.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:250b692fb75f45f02e2f58fbef9ab338904ef334b90557565621fa251df267cf"}, + {file = "dm_tree-0.1.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81fce77f22a302d7a5968aebdf4efafef4def7ce96528719a354e6990dcd49c7"}, + {file = "dm_tree-0.1.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb"}, + {file = "dm_tree-0.1.8-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fe962015b2fe1282892b28ebe962faed53c7f98d942da9a4625cbf27baef913"}, + {file = "dm_tree-0.1.8-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c52cbf4f8b3dbd0beaedf44f69fa85eec5e9dede612e08035e06ada6ec9426"}, + {file = "dm_tree-0.1.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:181c35521d480d0365f39300542cb6cd7fd2b77351bb43d7acfda15aef63b317"}, + {file = "dm_tree-0.1.8-cp39-cp39-win_amd64.whl", hash = "sha256:8ed3564abed97c806db122c2d3e1a2b64c74a63debe9903aad795167cc301368"}, +] [[package]] name = "docstring-parser" version = "0.14.1" description = "Parse Python docstrings in reST, Google and Numpydoc format" -category = "main" optional = false python-versions = ">=3.6,<4.0" +files = [ + {file = "docstring_parser-0.14.1-py3-none-any.whl", hash = "sha256:14ac6ec1f1ba6905c4d8cb90fd0bc55394f5678183752c90e44812bf28d7a515"}, + {file = "docstring_parser-0.14.1.tar.gz", hash = "sha256:2c77522e31b7c88b1ab457a1f3c9ae38947ad719732260ba77ee8a3deb58622a"}, +] [[package]] name = "etils" version = "0.9.0" description = "Collection of common python utils" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "etils-0.9.0-py3-none-any.whl", hash = "sha256:635d6f7d1c519eb194304228543a4c5c7df0e6b58243302473e34c18cf720588"}, + {file = "etils-0.9.0.tar.gz", hash = "sha256:489103e9e499a566765c60458ee15d185cf0065f2060a4d16a68f8f46962ed0d"}, +] [package.extras] all = ["etils[array-types]", "etils[eapp]", "etils[ecolab]", "etils[edc]", "etils[enp]", "etils[epath]", "etils[epy]", "etils[etqdm]", "etils[etree-dm]", "etils[etree-jax]", "etils[etree-tf]", "etils[etree]"] @@ -136,7 +247,7 @@ enp = ["etils[epy]", "numpy"] epath = ["etils[epy]", "importlib_resources", "typing_extensions", "zipp"] epy = ["typing_extensions"] etqdm = ["absl-py", "etils[epy]", "tqdm"] -etree = ["etils[array_types]", "etils[enp]", "etils[epy]", "etils[etqdm]"] +etree = ["etils[array-types]", "etils[enp]", "etils[epy]", "etils[etqdm]"] etree-dm = ["dm-tree", "etils[etree]"] etree-jax = ["etils[etree]", "jax[cpu]"] etree-tf = ["etils[etree]", "tf-nightly"] @@ -144,22 +255,28 @@ lazy-imports = ["etils[ecolab]"] [[package]] name = "exceptiongroup" -version = "1.1.0" +version = "1.1.1" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, +] [package.extras] test = ["pytest (>=6)"] [[package]] name = "flax" -version = "0.6.3" +version = "0.6.4" description = "Flax: A neural network library for JAX designed for flexibility" -category = "dev" optional = false python-versions = "*" +files = [ + {file = "flax-0.6.4-py3-none-any.whl", hash = "sha256:fe5010525202241fdc960920033d2e4c0b35f06090c1ad9e280b1f4415ae308f"}, + {file = "flax-0.6.4.tar.gz", hash = "sha256:d06465a3e6636c3c23c29f651a13f5367d06c41373b441dc8ec1bfaa4db06a48"}, +] [package.dependencies] jax = ">=0.3.16" @@ -174,15 +291,18 @@ tensorstore = "*" typing-extensions = ">=4.1.1" [package.extras] -testing = ["atari-py (==0.2.5)", "clu", "gym (==0.18.3)", "jaxlib", "jraph (>=0.0.6dev0)", "ml-collections", "opencv-python", "pytest", "pytest-cov", "pytest-custom-exit-code", "pytest-xdist (==1.34.0)", "pytype", "sentencepiece", "tensorflow", "tensorflow-datasets", "tensorflow-text (>=2.4.0)", "torch"] +testing = ["atari-py (==0.2.5)", "clu", "gym (==0.18.3)", "jaxlib", "jraph (>=0.0.6dev0)", "ml-collections", "mypy", "opencv-python", "pytest", "pytest-cov", "pytest-custom-exit-code", "pytest-xdist (==1.34.0)", "pytype", "sentencepiece", "tensorflow", "tensorflow-datasets", "tensorflow-text (>=2.4.0)", "torch"] [[package]] name = "fonttools" version = "4.38.0" description = "Tools to manipulate font files" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "fonttools-4.38.0-py3-none-any.whl", hash = "sha256:820466f43c8be8c3009aef8b87e785014133508f0de64ec469e4efb643ae54fb"}, + {file = "fonttools-4.38.0.zip", hash = "sha256:2bb244009f9bf3fa100fc3ead6aeb99febe5985fa20afbfbaa2f8946c2fbdaf1"}, +] [package.extras] all = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "fs (>=2.2.0,<3)", "lxml (>=4.0,<5)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres", "scipy", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=14.0.0)", "xattr", "zopfli (>=0.1.4)"] @@ -200,19 +320,60 @@ woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] [[package]] name = "frozendict" -version = "2.3.4" +version = "2.3.8" description = "A simple immutable dictionary" -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "frozendict-2.3.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d188d062084fba0e4bf32719ff7380b26c050b932ff164043ce82ab90587c52b"}, + {file = "frozendict-2.3.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f2a4e818ac457f6354401dcb631527af25e5a20fcfc81e6b5054b45fc245caca"}, + {file = "frozendict-2.3.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a506d807858fa961aaa5b48dab6154fdc6bd045bbe9310788bbff141bb42d13"}, + {file = "frozendict-2.3.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:750632cc890d8ee9484fe6d31b261159144b6efacc08e1317fe46accd1410373"}, + {file = "frozendict-2.3.8-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7ee5fe2658a8ac9a57f748acaf563f6a47f80b8308cbf0a04fac0ba057d41f75"}, + {file = "frozendict-2.3.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23c4bb46e6b8246e1e7e49b5593c2bc09221db0d8f31f7c092be8dfb42b9e620"}, + {file = "frozendict-2.3.8-cp310-cp310-win_amd64.whl", hash = "sha256:c31abc8acea309b132dde441856829f6003a3d242da8b54bce4c0f2a3c8c63f0"}, + {file = "frozendict-2.3.8-cp310-cp310-win_arm64.whl", hash = "sha256:9ea5520e85447ff8d4681e181941e482662817ccba921b7cb3f87922056d892a"}, + {file = "frozendict-2.3.8-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f83fed36497af9562ead5e9fb8443224ba2781786bd3b92b1087cb7d0ff20135"}, + {file = "frozendict-2.3.8-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e27c5c1d29d0eda7979253ec88abc239da1313b38f39f4b16984db3b3e482300"}, + {file = "frozendict-2.3.8-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4c785de7f1a13f15963945f400656b18f057c2fc76c089dacf127a2bb188c03"}, + {file = "frozendict-2.3.8-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8cf35ddd25513428ec152614def9696afb93ae5ec0eb54fa6aa6206eda77ac4c"}, + {file = "frozendict-2.3.8-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:ffc684773de7c88724788fa9787d0016fd75830412d58acbd9ed1a04762c675b"}, + {file = "frozendict-2.3.8-cp36-cp36m-win_amd64.whl", hash = "sha256:4c258aab9c8488338634f2ec670ef049dbf0ab0e7a2fa9bc2c7b5009cb614801"}, + {file = "frozendict-2.3.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:47fc26468407fdeb428cfc89495b7921419e670355c21b383765482fdf6c5c14"}, + {file = "frozendict-2.3.8-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ea638228692db2bf94bce40ea4b25f4077588497b516bd16576575560094bd9"}, + {file = "frozendict-2.3.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a75bf87e76c4386caecdbdd02a99e53ad43a6b5c38fb3d5a634a9fc9ce41462"}, + {file = "frozendict-2.3.8-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ed5a6c5c7a0f57269577c2a338a6002949aea21a23b7b7d06da7e7dced8b605b"}, + {file = "frozendict-2.3.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d086440328a465dea9bef2dbad7548d75d1a0a0d21f43a08c03e1ec79ac5240e"}, + {file = "frozendict-2.3.8-cp37-cp37m-win_amd64.whl", hash = "sha256:0bc4767e2f83db5b701c787e22380296977368b0c57e485ca71b2eedfa11c4a3"}, + {file = "frozendict-2.3.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:638cf363d3cbca31a341503cf2219eac52a5f5140449676fae3d9644cd3c5487"}, + {file = "frozendict-2.3.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2b2fd8ce36277919b36e3c834d2389f3cd7ac068ae730c312671dd4439a5dd65"}, + {file = "frozendict-2.3.8-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3957d52f1906b0c85f641a1911d214255873f6408ab4e5ad657cc27a247fb145"}, + {file = "frozendict-2.3.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72cfe08ab8ae524e54848fa90b22d02c1b1ecfb3064438696bcaa4b953f18772"}, + {file = "frozendict-2.3.8-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4742e76c4111bd09198d3ab66cef94be8506212311338f9182d6ef5f5cb60493"}, + {file = "frozendict-2.3.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:313ed8d9ba6bac35d7635cd9580ee5721a0fb016f4d2d20f0efa05dbecbdb1be"}, + {file = "frozendict-2.3.8-cp38-cp38-win_amd64.whl", hash = "sha256:d3c6ce943946c2a61501c8cf116fff0892d11dd579877eb36e2aea2c27fddfef"}, + {file = "frozendict-2.3.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0f573dc4861dd7ec9e055c8cceaf45355e894e749f621f199aab7b311ac4bdb"}, + {file = "frozendict-2.3.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2b3435e5f1ca5ae68a5e95e64b09d6d5c645cadd6b87569a0b3019dd248c8d00"}, + {file = "frozendict-2.3.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:145afd033ebfade28416093335261b8ec1af5cccc593482309e7add062ec8668"}, + {file = "frozendict-2.3.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da98427de26b5a2865727947480cbb53860089c4d195baa29c539da811cea617"}, + {file = "frozendict-2.3.8-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5e82befa7c385a668d569cebbebbdf49cee6fea4083f08e869a1b08cfb640a9f"}, + {file = "frozendict-2.3.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:80abe81d36e889ceec665e06ec764a7638000fa3e7be09786ac4d3ddc64b76db"}, + {file = "frozendict-2.3.8-cp39-cp39-win_amd64.whl", hash = "sha256:8ccc94ac781710db44e142e1a11ff9b31d02c032c01c6868d51fcbef73086225"}, + {file = "frozendict-2.3.8-cp39-cp39-win_arm64.whl", hash = "sha256:e72dbc1bcc2203cef38d205f692396f5505921a5680f66aa9a7e8bb71fd38f28"}, + {file = "frozendict-2.3.8-py311-none-any.whl", hash = "sha256:ba41a7ed019bd03b62d63ed3f8dea35b8243d1936f7c9ed4b5298ca45a01928e"}, + {file = "frozendict-2.3.8.tar.gz", hash = "sha256:5526559eca8f1780a4ee5146896f59afc31435313560208dd394a3a5e537d3ff"}, +] [[package]] name = "importlib-metadata" -version = "5.2.0" +version = "6.6.0" description = "Read metadata from Python packages" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "importlib_metadata-6.6.0-py3-none-any.whl", hash = "sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed"}, + {file = "importlib_metadata-6.6.0.tar.gz", hash = "sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705"}, +] [package.dependencies] typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} @@ -225,34 +386,42 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag [[package]] name = "importlib-resources" -version = "5.10.1" +version = "5.12.0" description = "Read resources from Python packages" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, + {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, +] [package.dependencies] zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [[package]] name = "iniconfig" -version = "1.1.1" -description = "iniconfig: brain-dead simple config-ini parsing" -category = "dev" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" optional = false -python-versions = "*" +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] [[package]] name = "jax" version = "0.3.25" description = "Differentiate, compile, and transform Numpy code." -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "jax-0.3.25.tar.gz", hash = "sha256:18bea69321cb95ea5ea913adfe5e2c1d453cade9d4cfd0dc814ecba9fc0cb6e3"}, +] [package.dependencies] numpy = ">=1.20" @@ -265,8 +434,8 @@ australis = ["protobuf (>=3.13,<4)"] ci = ["jaxlib (==0.3.24)"] cpu = ["jaxlib (==0.3.25)"] cuda = ["jaxlib (==0.3.25+cuda11.cudnn82)"] -cuda11_cudnn805 = ["jaxlib (==0.3.25+cuda11.cudnn805)"] -cuda11_cudnn82 = ["jaxlib (==0.3.25+cuda11.cudnn82)"] +cuda11-cudnn805 = ["jaxlib (==0.3.25+cuda11.cudnn805)"] +cuda11-cudnn82 = ["jaxlib (==0.3.25+cuda11.cudnn82)"] minimum-jaxlib = ["jaxlib (==0.3.22)"] tpu = ["jaxlib (==0.3.25)", "libtpu-nightly (==0.1.dev20221109)", "requests"] @@ -274,9 +443,24 @@ tpu = ["jaxlib (==0.3.25)", "libtpu-nightly (==0.1.dev20221109)", "requests"] name = "jaxlib" version = "0.3.25" description = "XLA library for JAX" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "jaxlib-0.3.25-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:09508f7000c0fa958fba29267338e8de75b31d7ea29bd79719a568c38f0f8d31"}, + {file = "jaxlib-0.3.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c75c8efd3702687968820446e3fb9ff997f8a2a07ab92e33b80e2f12eab3d9a"}, + {file = "jaxlib-0.3.25-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:6e2f4e51041b8371aa3976b5a3a9cdcdccb1bd7b040c9b1345cbf24bd28a8d19"}, + {file = "jaxlib-0.3.25-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:f2d517635fd77e2729c0ab7863be0d290927d01b2abb2f5dc955c821f8b0d53e"}, + {file = "jaxlib-0.3.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:13446a8382aa9ed944c16af636ca111d0afbbead91eed5cc2dc71195045e71b3"}, + {file = "jaxlib-0.3.25-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:71866aeaafbc9a12b59dcbe443353772ef235b40c53f8bd7403d39311822c276"}, + {file = "jaxlib-0.3.25-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:1e59ba58c9e93c1e1cef243f2609ec0b0c0a81160c20b9555aecdea32ccd6a78"}, + {file = "jaxlib-0.3.25-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:5295354ed5db111e6f3e88cdfa4010d11c33dd926ac61735b9096b4e0746aa7b"}, + {file = "jaxlib-0.3.25-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:9f3116389ee834b3cdeb30001b085f4b55d7741366034f041c1d377154aa5afa"}, + {file = "jaxlib-0.3.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:78b29c72d0680829db9377ed9be326875849258a60b8173b4a388b34ad18bc78"}, + {file = "jaxlib-0.3.25-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:2e008e0a6c10aa7e949555e98dc0471e0d550d5d7c109771e38a971b49480538"}, + {file = "jaxlib-0.3.25-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:fec043cdd55f3257d02e9d8880b33860eacadcae1bd5e26f43fdd08ada23614d"}, + {file = "jaxlib-0.3.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50193ba0cbf879021c9d73d7bcfa7eafb9138895d057b774c301aac3701f9a5"}, + {file = "jaxlib-0.3.25-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:1f1448f102a9d05186f579b6931fa0c607783ecc915fdfaa482c19538affa180"}, +] [package.dependencies] numpy = ">=1.20" @@ -286,20 +470,150 @@ scipy = ">=1.5" name = "kiwisolver" version = "1.4.4" description = "A fast implementation of the Cassowary constraint solver" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "kiwisolver-1.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2f5e60fabb7343a836360c4f0919b8cd0d6dbf08ad2ca6b9cf90bf0c76a3c4f6"}, + {file = "kiwisolver-1.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:10ee06759482c78bdb864f4109886dff7b8a56529bc1609d4f1112b93fe6423c"}, + {file = "kiwisolver-1.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c79ebe8f3676a4c6630fd3f777f3cfecf9289666c84e775a67d1d358578dc2e3"}, + {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:abbe9fa13da955feb8202e215c4018f4bb57469b1b78c7a4c5c7b93001699938"}, + {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7577c1987baa3adc4b3c62c33bd1118c3ef5c8ddef36f0f2c950ae0b199e100d"}, + {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8ad8285b01b0d4695102546b342b493b3ccc6781fc28c8c6a1bb63e95d22f09"}, + {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ed58b8acf29798b036d347791141767ccf65eee7f26bde03a71c944449e53de"}, + {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a68b62a02953b9841730db7797422f983935aeefceb1679f0fc85cbfbd311c32"}, + {file = "kiwisolver-1.4.4-cp310-cp310-win32.whl", hash = "sha256:e92a513161077b53447160b9bd8f522edfbed4bd9759e4c18ab05d7ef7e49408"}, + {file = "kiwisolver-1.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:3fe20f63c9ecee44560d0e7f116b3a747a5d7203376abeea292ab3152334d004"}, + {file = "kiwisolver-1.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e0ea21f66820452a3f5d1655f8704a60d66ba1191359b96541eaf457710a5fc6"}, + {file = "kiwisolver-1.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bc9db8a3efb3e403e4ecc6cd9489ea2bac94244f80c78e27c31dcc00d2790ac2"}, + {file = "kiwisolver-1.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d5b61785a9ce44e5a4b880272baa7cf6c8f48a5180c3e81c59553ba0cb0821ca"}, + {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2dbb44c3f7e6c4d3487b31037b1bdbf424d97687c1747ce4ff2895795c9bf69"}, + {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6295ecd49304dcf3bfbfa45d9a081c96509e95f4b9d0eb7ee4ec0530c4a96514"}, + {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4bd472dbe5e136f96a4b18f295d159d7f26fd399136f5b17b08c4e5f498cd494"}, + {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bf7d9fce9bcc4752ca4a1b80aabd38f6d19009ea5cbda0e0856983cf6d0023f5"}, + {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d6601aed50c74e0ef02f4204da1816147a6d3fbdc8b3872d263338a9052c51"}, + {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:877272cf6b4b7e94c9614f9b10140e198d2186363728ed0f701c6eee1baec1da"}, + {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:db608a6757adabb32f1cfe6066e39b3706d8c3aa69bbc353a5b61edad36a5cb4"}, + {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5853eb494c71e267912275e5586fe281444eb5e722de4e131cddf9d442615626"}, + {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:f0a1dbdb5ecbef0d34eb77e56fcb3e95bbd7e50835d9782a45df81cc46949750"}, + {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:283dffbf061a4ec60391d51e6155e372a1f7a4f5b15d59c8505339454f8989e4"}, + {file = "kiwisolver-1.4.4-cp311-cp311-win32.whl", hash = "sha256:d06adcfa62a4431d404c31216f0f8ac97397d799cd53800e9d3efc2fbb3cf14e"}, + {file = "kiwisolver-1.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:e7da3fec7408813a7cebc9e4ec55afed2d0fd65c4754bc376bf03498d4e92686"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:62ac9cc684da4cf1778d07a89bf5f81b35834cb96ca523d3a7fb32509380cbf6"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41dae968a94b1ef1897cb322b39360a0812661dba7c682aa45098eb8e193dbdf"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02f79693ec433cb4b5f51694e8477ae83b3205768a6fb48ffba60549080e295b"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0611a0a2a518464c05ddd5a3a1a0e856ccc10e67079bb17f265ad19ab3c7597"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:db5283d90da4174865d520e7366801a93777201e91e79bacbac6e6927cbceede"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1041feb4cda8708ce73bb4dcb9ce1ccf49d553bf87c3954bdfa46f0c3f77252c"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-win32.whl", hash = "sha256:a553dadda40fef6bfa1456dc4be49b113aa92c2a9a9e8711e955618cd69622e3"}, + {file = "kiwisolver-1.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:03baab2d6b4a54ddbb43bba1a3a2d1627e82d205c5cf8f4c924dc49284b87166"}, + {file = "kiwisolver-1.4.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:841293b17ad704d70c578f1f0013c890e219952169ce8a24ebc063eecf775454"}, + {file = "kiwisolver-1.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f4f270de01dd3e129a72efad823da90cc4d6aafb64c410c9033aba70db9f1ff0"}, + {file = "kiwisolver-1.4.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f9f39e2f049db33a908319cf46624a569b36983c7c78318e9726a4cb8923b26c"}, + {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c97528e64cb9ebeff9701e7938653a9951922f2a38bd847787d4a8e498cc83ae"}, + {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d1573129aa0fd901076e2bfb4275a35f5b7aa60fbfb984499d661ec950320b0"}, + {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad881edc7ccb9d65b0224f4e4d05a1e85cf62d73aab798943df6d48ab0cd79a1"}, + {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b428ef021242344340460fa4c9185d0b1f66fbdbfecc6c63eff4b7c29fad429d"}, + {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2e407cb4bd5a13984a6c2c0fe1845e4e41e96f183e5e5cd4d77a857d9693494c"}, + {file = "kiwisolver-1.4.4-cp38-cp38-win32.whl", hash = "sha256:75facbe9606748f43428fc91a43edb46c7ff68889b91fa31f53b58894503a191"}, + {file = "kiwisolver-1.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:5bce61af018b0cb2055e0e72e7d65290d822d3feee430b7b8203d8a855e78766"}, + {file = "kiwisolver-1.4.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8c808594c88a025d4e322d5bb549282c93c8e1ba71b790f539567932722d7bd8"}, + {file = "kiwisolver-1.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0a71d85ecdd570ded8ac3d1c0f480842f49a40beb423bb8014539a9f32a5897"}, + {file = "kiwisolver-1.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b533558eae785e33e8c148a8d9921692a9fe5aa516efbdff8606e7d87b9d5824"}, + {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:efda5fc8cc1c61e4f639b8067d118e742b812c930f708e6667a5ce0d13499e29"}, + {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7c43e1e1206cd421cd92e6b3280d4385d41d7166b3ed577ac20444b6995a445f"}, + {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc8d3bd6c72b2dd9decf16ce70e20abcb3274ba01b4e1c96031e0c4067d1e7cd"}, + {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ea39b0ccc4f5d803e3337dd46bcce60b702be4d86fd0b3d7531ef10fd99a1ac"}, + {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:968f44fdbf6dd757d12920d63b566eeb4d5b395fd2d00d29d7ef00a00582aac9"}, + {file = "kiwisolver-1.4.4-cp39-cp39-win32.whl", hash = "sha256:da7e547706e69e45d95e116e6939488d62174e033b763ab1496b4c29b76fabea"}, + {file = "kiwisolver-1.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:ba59c92039ec0a66103b1d5fe588fa546373587a7d68f5c96f743c3396afc04b"}, + {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:91672bacaa030f92fc2f43b620d7b337fd9a5af28b0d6ed3f77afc43c4a64b5a"}, + {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:787518a6789009c159453da4d6b683f468ef7a65bbde796bcea803ccf191058d"}, + {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da152d8cdcab0e56e4f45eb08b9aea6455845ec83172092f09b0e077ece2cf7a"}, + {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ecb1fa0db7bf4cff9dac752abb19505a233c7f16684c5826d1f11ebd9472b871"}, + {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:28bc5b299f48150b5f822ce68624e445040595a4ac3d59251703779836eceff9"}, + {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:81e38381b782cc7e1e46c4e14cd997ee6040768101aefc8fa3c24a4cc58e98f8"}, + {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2a66fdfb34e05b705620dd567f5a03f239a088d5a3f321e7b6ac3239d22aa286"}, + {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872b8ca05c40d309ed13eb2e582cab0c5a05e81e987ab9c521bf05ad1d5cf5cb"}, + {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:70e7c2e7b750585569564e2e5ca9845acfaa5da56ac46df68414f29fea97be9f"}, + {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9f85003f5dfa867e86d53fac6f7e6f30c045673fa27b603c397753bebadc3008"}, + {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e307eb9bd99801f82789b44bb45e9f541961831c7311521b13a6c85afc09767"}, + {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1792d939ec70abe76f5054d3f36ed5656021dcad1322d1cc996d4e54165cef9"}, + {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6cb459eea32a4e2cf18ba5fcece2dbdf496384413bc1bae15583f19e567f3b2"}, + {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36dafec3d6d6088d34e2de6b85f9d8e2324eb734162fba59d2ba9ed7a2043d5b"}, + {file = "kiwisolver-1.4.4.tar.gz", hash = "sha256:d41997519fcba4a1e46eb4a2fe31bc12f0ff957b2b81bac28db24744f333e955"}, +] [package.dependencies] typing-extensions = {version = "*", markers = "python_version < \"3.8\""} +[[package]] +name = "markdown-it-py" +version = "2.2.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.7" +files = [ + {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"}, + {file = "markdown_it_py-2.2.0-py3-none-any.whl", hash = "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" +typing_extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + [[package]] name = "matplotlib" version = "3.5.3" description = "Python plotting package" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "matplotlib-3.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a206a1b762b39398efea838f528b3a6d60cdb26fe9d58b48265787e29cd1d693"}, + {file = "matplotlib-3.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cd45a6f3e93a780185f70f05cf2a383daed13c3489233faad83e81720f7ede24"}, + {file = "matplotlib-3.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d62880e1f60e5a30a2a8484432bcb3a5056969dc97258d7326ad465feb7ae069"}, + {file = "matplotlib-3.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ab29589cef03bc88acfa3a1490359000c18186fc30374d8aa77d33cc4a51a4a"}, + {file = "matplotlib-3.5.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2886cc009f40e2984c083687251821f305d811d38e3df8ded414265e4583f0c5"}, + {file = "matplotlib-3.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c995f7d9568f18b5db131ab124c64e51b6820a92d10246d4f2b3f3a66698a15b"}, + {file = "matplotlib-3.5.3-cp310-cp310-win32.whl", hash = "sha256:6bb93a0492d68461bd458eba878f52fdc8ac7bdb6c4acdfe43dba684787838c2"}, + {file = "matplotlib-3.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:2e6d184ebe291b9e8f7e78bbab7987d269c38ea3e062eace1fe7d898042ef804"}, + {file = "matplotlib-3.5.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6ea6aef5c4338e58d8d376068e28f80a24f54e69f09479d1c90b7172bad9f25b"}, + {file = "matplotlib-3.5.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:839d47b8ead7ad9669aaacdbc03f29656dc21f0d41a6fea2d473d856c39c8b1c"}, + {file = "matplotlib-3.5.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3b4fa56159dc3c7f9250df88f653f085068bcd32dcd38e479bba58909254af7f"}, + {file = "matplotlib-3.5.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:94ff86af56a3869a4ae26a9637a849effd7643858a1a04dd5ee50e9ab75069a7"}, + {file = "matplotlib-3.5.3-cp37-cp37m-win32.whl", hash = "sha256:35a8ad4dddebd51f94c5d24bec689ec0ec66173bf614374a1244c6241c1595e0"}, + {file = "matplotlib-3.5.3-cp37-cp37m-win_amd64.whl", hash = "sha256:43e9d3fa077bf0cc95ded13d331d2156f9973dce17c6f0c8b49ccd57af94dbd9"}, + {file = "matplotlib-3.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:22227c976ad4dc8c5a5057540421f0d8708c6560744ad2ad638d48e2984e1dbc"}, + {file = "matplotlib-3.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bf618a825deb6205f015df6dfe6167a5d9b351203b03fab82043ae1d30f16511"}, + {file = "matplotlib-3.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9befa5954cdbc085e37d974ff6053da269474177921dd61facdad8023c4aeb51"}, + {file = "matplotlib-3.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3840c280ebc87a48488a46f760ea1c0c0c83fcf7abbe2e6baf99d033fd35fd8"}, + {file = "matplotlib-3.5.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dacddf5bfcec60e3f26ec5c0ae3d0274853a258b6c3fc5ef2f06a8eb23e042be"}, + {file = "matplotlib-3.5.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b428076a55fb1c084c76cb93e68006f27d247169f056412607c5c88828d08f88"}, + {file = "matplotlib-3.5.3-cp38-cp38-win32.whl", hash = "sha256:874df7505ba820e0400e7091199decf3ff1fde0583652120c50cd60d5820ca9a"}, + {file = "matplotlib-3.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:b28de401d928890187c589036857a270a032961411934bdac4cf12dde3d43094"}, + {file = "matplotlib-3.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3211ba82b9f1518d346f6309df137b50c3dc4421b4ed4815d1d7eadc617f45a1"}, + {file = "matplotlib-3.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6fe807e8a22620b4cd95cfbc795ba310dc80151d43b037257250faf0bfcd82bc"}, + {file = "matplotlib-3.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5c096363b206a3caf43773abebdbb5a23ea13faef71d701b21a9c27fdcef72f4"}, + {file = "matplotlib-3.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bcdfcb0f976e1bac6721d7d457c17be23cf7501f977b6a38f9d38a3762841f7"}, + {file = "matplotlib-3.5.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1e64ac9be9da6bfff0a732e62116484b93b02a0b4d4b19934fb4f8e7ad26ad6a"}, + {file = "matplotlib-3.5.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:73dd93dc35c85dece610cca8358003bf0760d7986f70b223e2306b4ea6d1406b"}, + {file = "matplotlib-3.5.3-cp39-cp39-win32.whl", hash = "sha256:879c7e5fce4939c6aa04581dfe08d57eb6102a71f2e202e3314d5fbc072fd5a0"}, + {file = "matplotlib-3.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:ab8d26f07fe64f6f6736d635cce7bfd7f625320490ed5bfc347f2cdb4fae0e56"}, + {file = "matplotlib-3.5.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:99482b83ebf4eb6d5fc6813d7aacdefdd480f0d9c0b52dcf9f1cc3b2c4b3361a"}, + {file = "matplotlib-3.5.3-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f814504e459c68118bf2246a530ed953ebd18213dc20e3da524174d84ed010b2"}, + {file = "matplotlib-3.5.3-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57f1b4e69f438a99bb64d7f2c340db1b096b41ebaa515cf61ea72624279220ce"}, + {file = "matplotlib-3.5.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:d2484b350bf3d32cae43f85dcfc89b3ed7bd2bcd781ef351f93eb6fb2cc483f9"}, + {file = "matplotlib-3.5.3.tar.gz", hash = "sha256:339cac48b80ddbc8bfd05daae0a3a73414651a8596904c2a881cfd1edb65f26c"}, +] [package.dependencies] cycler = ">=0.10" @@ -310,23 +624,128 @@ packaging = ">=20.0" pillow = ">=6.2.0" pyparsing = ">=2.2.1" python-dateutil = ">=2.7" -setuptools_scm = ">=4,<7" [[package]] -name = "msgpack" -version = "1.0.4" +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + +[[package]] +name = "msgpack" +version = "1.0.5" description = "MessagePack serializer" -category = "dev" optional = false python-versions = "*" +files = [ + {file = "msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9"}, + {file = "msgpack-1.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f8d8b3bf1ff2672567d6b5c725a1b347fe838b912772aa8ae2bf70338d5a198"}, + {file = "msgpack-1.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdc793c50be3f01106245a61b739328f7dccc2c648b501e237f0699fe1395b81"}, + {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cb47c21a8a65b165ce29f2bec852790cbc04936f502966768e4aae9fa763cb7"}, + {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e42b9594cc3bf4d838d67d6ed62b9e59e201862a25e9a157019e171fbe672dd3"}, + {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:55b56a24893105dc52c1253649b60f475f36b3aa0fc66115bffafb624d7cb30b"}, + {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1967f6129fc50a43bfe0951c35acbb729be89a55d849fab7686004da85103f1c"}, + {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:20a97bf595a232c3ee6d57ddaadd5453d174a52594bf9c21d10407e2a2d9b3bd"}, + {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d25dd59bbbbb996eacf7be6b4ad082ed7eacc4e8f3d2df1ba43822da9bfa122a"}, + {file = "msgpack-1.0.5-cp310-cp310-win32.whl", hash = "sha256:382b2c77589331f2cb80b67cc058c00f225e19827dbc818d700f61513ab47bea"}, + {file = "msgpack-1.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:4867aa2df9e2a5fa5f76d7d5565d25ec76e84c106b55509e78c1ede0f152659a"}, + {file = "msgpack-1.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9f5ae84c5c8a857ec44dc180a8b0cc08238e021f57abdf51a8182e915e6299f0"}, + {file = "msgpack-1.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e6ca5d5699bcd89ae605c150aee83b5321f2115695e741b99618f4856c50898"}, + {file = "msgpack-1.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5494ea30d517a3576749cad32fa27f7585c65f5f38309c88c6d137877fa28a5a"}, + {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ab2f3331cb1b54165976a9d976cb251a83183631c88076613c6c780f0d6e45a"}, + {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28592e20bbb1620848256ebc105fc420436af59515793ed27d5c77a217477705"}, + {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe5c63197c55bce6385d9aee16c4d0641684628f63ace85f73571e65ad1c1e8d"}, + {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ed40e926fa2f297e8a653c954b732f125ef97bdd4c889f243182299de27e2aa9"}, + {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b2de4c1c0538dcb7010902a2b97f4e00fc4ddf2c8cda9749af0e594d3b7fa3d7"}, + {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bf22a83f973b50f9d38e55c6aade04c41ddda19b00c4ebc558930d78eecc64ed"}, + {file = "msgpack-1.0.5-cp311-cp311-win32.whl", hash = "sha256:c396e2cc213d12ce017b686e0f53497f94f8ba2b24799c25d913d46c08ec422c"}, + {file = "msgpack-1.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:6c4c68d87497f66f96d50142a2b73b97972130d93677ce930718f68828b382e2"}, + {file = "msgpack-1.0.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a2b031c2e9b9af485d5e3c4520f4220d74f4d222a5b8dc8c1a3ab9448ca79c57"}, + {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f837b93669ce4336e24d08286c38761132bc7ab29782727f8557e1eb21b2080"}, + {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1d46dfe3832660f53b13b925d4e0fa1432b00f5f7210eb3ad3bb9a13c6204a6"}, + {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:366c9a7b9057e1547f4ad51d8facad8b406bab69c7d72c0eb6f529cf76d4b85f"}, + {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4c075728a1095efd0634a7dccb06204919a2f67d1893b6aa8e00497258bf926c"}, + {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:f933bbda5a3ee63b8834179096923b094b76f0c7a73c1cfe8f07ad608c58844b"}, + {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:36961b0568c36027c76e2ae3ca1132e35123dcec0706c4b7992683cc26c1320c"}, + {file = "msgpack-1.0.5-cp36-cp36m-win32.whl", hash = "sha256:b5ef2f015b95f912c2fcab19c36814963b5463f1fb9049846994b007962743e9"}, + {file = "msgpack-1.0.5-cp36-cp36m-win_amd64.whl", hash = "sha256:288e32b47e67f7b171f86b030e527e302c91bd3f40fd9033483f2cacc37f327a"}, + {file = "msgpack-1.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:137850656634abddfb88236008339fdaba3178f4751b28f270d2ebe77a563b6c"}, + {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c05a4a96585525916b109bb85f8cb6511db1c6f5b9d9cbcbc940dc6b4be944b"}, + {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56a62ec00b636583e5cb6ad313bbed36bb7ead5fa3a3e38938503142c72cba4f"}, + {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef8108f8dedf204bb7b42994abf93882da1159728a2d4c5e82012edd92c9da9f"}, + {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1835c84d65f46900920b3708f5ba829fb19b1096c1800ad60bae8418652a951d"}, + {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e57916ef1bd0fee4f21c4600e9d1da352d8816b52a599c46460e93a6e9f17086"}, + {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:17358523b85973e5f242ad74aa4712b7ee560715562554aa2134d96e7aa4cbbf"}, + {file = "msgpack-1.0.5-cp37-cp37m-win32.whl", hash = "sha256:cb5aaa8c17760909ec6cb15e744c3ebc2ca8918e727216e79607b7bbce9c8f77"}, + {file = "msgpack-1.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:ab31e908d8424d55601ad7075e471b7d0140d4d3dd3272daf39c5c19d936bd82"}, + {file = "msgpack-1.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b72d0698f86e8d9ddf9442bdedec15b71df3598199ba33322d9711a19f08145c"}, + {file = "msgpack-1.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:379026812e49258016dd84ad79ac8446922234d498058ae1d415f04b522d5b2d"}, + {file = "msgpack-1.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:332360ff25469c346a1c5e47cbe2a725517919892eda5cfaffe6046656f0b7bb"}, + {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:476a8fe8fae289fdf273d6d2a6cb6e35b5a58541693e8f9f019bfe990a51e4ba"}, + {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9985b214f33311df47e274eb788a5893a761d025e2b92c723ba4c63936b69b1"}, + {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48296af57cdb1d885843afd73c4656be5c76c0c6328db3440c9601a98f303d87"}, + {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:addab7e2e1fcc04bd08e4eb631c2a90960c340e40dfc4a5e24d2ff0d5a3b3edb"}, + {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:916723458c25dfb77ff07f4c66aed34e47503b2eb3188b3adbec8d8aa6e00f48"}, + {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:821c7e677cc6acf0fd3f7ac664c98803827ae6de594a9f99563e48c5a2f27eb0"}, + {file = "msgpack-1.0.5-cp38-cp38-win32.whl", hash = "sha256:1c0f7c47f0087ffda62961d425e4407961a7ffd2aa004c81b9c07d9269512f6e"}, + {file = "msgpack-1.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:bae7de2026cbfe3782c8b78b0db9cbfc5455e079f1937cb0ab8d133496ac55e1"}, + {file = "msgpack-1.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:20c784e66b613c7f16f632e7b5e8a1651aa5702463d61394671ba07b2fc9e025"}, + {file = "msgpack-1.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:266fa4202c0eb94d26822d9bfd7af25d1e2c088927fe8de9033d929dd5ba24c5"}, + {file = "msgpack-1.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18334484eafc2b1aa47a6d42427da7fa8f2ab3d60b674120bce7a895a0a85bdd"}, + {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57e1f3528bd95cc44684beda696f74d3aaa8a5e58c816214b9046512240ef437"}, + {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:586d0d636f9a628ddc6a17bfd45aa5b5efaf1606d2b60fa5d87b8986326e933f"}, + {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a740fa0e4087a734455f0fc3abf5e746004c9da72fbd541e9b113013c8dc3282"}, + {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3055b0455e45810820db1f29d900bf39466df96ddca11dfa6d074fa47054376d"}, + {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a61215eac016f391129a013c9e46f3ab308db5f5ec9f25811e811f96962599a8"}, + {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:362d9655cd369b08fda06b6657a303eb7172d5279997abe094512e919cf74b11"}, + {file = "msgpack-1.0.5-cp39-cp39-win32.whl", hash = "sha256:ac9dd47af78cae935901a9a500104e2dea2e253207c924cc95de149606dc43cc"}, + {file = "msgpack-1.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:06f5174b5f8ed0ed919da0e62cbd4ffde676a374aba4020034da05fab67b9164"}, + {file = "msgpack-1.0.5.tar.gz", hash = "sha256:c075544284eadc5cddc70f4757331d99dcbc16b2bbd4849d15f8aae4cf36d31c"}, +] [[package]] name = "mypy" version = "0.991" description = "Optional static typing for Python" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "mypy-0.991-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7d17e0a9707d0772f4a7b878f04b4fd11f6f5bcb9b3813975a9b13c9332153ab"}, + {file = "mypy-0.991-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0714258640194d75677e86c786e80ccf294972cc76885d3ebbb560f11db0003d"}, + {file = "mypy-0.991-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c8f3be99e8a8bd403caa8c03be619544bc2c77a7093685dcf308c6b109426c6"}, + {file = "mypy-0.991-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc9ec663ed6c8f15f4ae9d3c04c989b744436c16d26580eaa760ae9dd5d662eb"}, + {file = "mypy-0.991-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4307270436fd7694b41f913eb09210faff27ea4979ecbcd849e57d2da2f65305"}, + {file = "mypy-0.991-cp310-cp310-win_amd64.whl", hash = "sha256:901c2c269c616e6cb0998b33d4adbb4a6af0ac4ce5cd078afd7bc95830e62c1c"}, + {file = "mypy-0.991-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d13674f3fb73805ba0c45eb6c0c3053d218aa1f7abead6e446d474529aafc372"}, + {file = "mypy-0.991-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c8cd4fb70e8584ca1ed5805cbc7c017a3d1a29fb450621089ffed3e99d1857f"}, + {file = "mypy-0.991-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:209ee89fbb0deed518605edddd234af80506aec932ad28d73c08f1400ef80a33"}, + {file = "mypy-0.991-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37bd02ebf9d10e05b00d71302d2c2e6ca333e6c2a8584a98c00e038db8121f05"}, + {file = "mypy-0.991-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:26efb2fcc6b67e4d5a55561f39176821d2adf88f2745ddc72751b7890f3194ad"}, + {file = "mypy-0.991-cp311-cp311-win_amd64.whl", hash = "sha256:3a700330b567114b673cf8ee7388e949f843b356a73b5ab22dd7cff4742a5297"}, + {file = "mypy-0.991-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1f7d1a520373e2272b10796c3ff721ea1a0712288cafaa95931e66aa15798813"}, + {file = "mypy-0.991-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:641411733b127c3e0dab94c45af15fea99e4468f99ac88b39efb1ad677da5711"}, + {file = "mypy-0.991-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3d80e36b7d7a9259b740be6d8d906221789b0d836201af4234093cae89ced0cd"}, + {file = "mypy-0.991-cp37-cp37m-win_amd64.whl", hash = "sha256:e62ebaad93be3ad1a828a11e90f0e76f15449371ffeecca4a0a0b9adc99abcef"}, + {file = "mypy-0.991-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b86ce2c1866a748c0f6faca5232059f881cda6dda2a893b9a8373353cfe3715a"}, + {file = "mypy-0.991-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac6e503823143464538efda0e8e356d871557ef60ccd38f8824a4257acc18d93"}, + {file = "mypy-0.991-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0cca5adf694af539aeaa6ac633a7afe9bbd760df9d31be55ab780b77ab5ae8bf"}, + {file = "mypy-0.991-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a12c56bf73cdab116df96e4ff39610b92a348cc99a1307e1da3c3768bbb5b135"}, + {file = "mypy-0.991-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:652b651d42f155033a1967739788c436491b577b6a44e4c39fb340d0ee7f0d70"}, + {file = "mypy-0.991-cp38-cp38-win_amd64.whl", hash = "sha256:4175593dc25d9da12f7de8de873a33f9b2b8bdb4e827a7cae952e5b1a342e243"}, + {file = "mypy-0.991-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:98e781cd35c0acf33eb0295e8b9c55cdbef64fcb35f6d3aa2186f289bed6e80d"}, + {file = "mypy-0.991-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6d7464bac72a85cb3491c7e92b5b62f3dcccb8af26826257760a552a5e244aa5"}, + {file = "mypy-0.991-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c9166b3f81a10cdf9b49f2d594b21b31adadb3d5e9db9b834866c3258b695be3"}, + {file = "mypy-0.991-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8472f736a5bfb159a5e36740847808f6f5b659960115ff29c7cecec1741c648"}, + {file = "mypy-0.991-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e80e758243b97b618cdf22004beb09e8a2de1af481382e4d84bc52152d1c476"}, + {file = "mypy-0.991-cp39-cp39-win_amd64.whl", hash = "sha256:74e259b5c19f70d35fcc1ad3d56499065c601dfe94ff67ae48b85596b9ec1461"}, + {file = "mypy-0.991-py3-none-any.whl", hash = "sha256:de32edc9b0a7e67c2775e574cb061a537660e51210fbf6006b0b36ea695ae9bb"}, + {file = "mypy-0.991.tar.gz", hash = "sha256:3c0165ba8f354a6d9881809ef29f1a9318a236a6d81c690094c5df32107bde06"}, +] [package.dependencies] mypy-extensions = ">=0.4.3" @@ -342,19 +761,25 @@ reports = ["lxml"] [[package]] name = "mypy-extensions" -version = "0.4.3" -description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "main" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." optional = false -python-versions = "*" +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] [[package]] name = "nodeenv" -version = "1.7.0" +version = "1.8.0" description = "Node.js virtual environment builder" -category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" +files = [ + {file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"}, + {file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"}, +] [package.dependencies] setuptools = "*" @@ -363,17 +788,49 @@ setuptools = "*" name = "numpy" version = "1.21.1" description = "NumPy is the fundamental package for array computing with Python." -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e"}, + {file = "numpy-1.21.1-cp37-cp37m-win32.whl", hash = "sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172"}, + {file = "numpy-1.21.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8"}, + {file = "numpy-1.21.1-cp38-cp38-win32.whl", hash = "sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd"}, + {file = "numpy-1.21.1-cp38-cp38-win_amd64.whl", hash = "sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a"}, + {file = "numpy-1.21.1-cp39-cp39-win32.whl", hash = "sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2"}, + {file = "numpy-1.21.1-cp39-cp39-win_amd64.whl", hash = "sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33"}, + {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"}, + {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, +] [[package]] name = "nvidia-cublas-cu11" version = "11.10.3.66" description = "CUBLAS native runtime libraries" -category = "dev" optional = false python-versions = ">=3" +files = [ + {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl", hash = "sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded"}, + {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-win_amd64.whl", hash = "sha256:8ac17ba6ade3ed56ab898a036f9ae0756f1e81052a317bf98f8c6d18dc3ae49e"}, +] [package.dependencies] setuptools = "*" @@ -383,9 +840,13 @@ wheel = "*" name = "nvidia-cuda-nvrtc-cu11" version = "11.7.99" description = "NVRTC native runtime libraries" -category = "dev" optional = false python-versions = ">=3" +files = [ + {file = "nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:9f1562822ea264b7e34ed5930567e89242d266448e936b85bc97a3370feabb03"}, + {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:f7d9610d9b7c331fa0da2d1b2858a4a8315e6d49765091d28711c8946e7425e7"}, + {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:f2effeb1309bdd1b3854fc9b17eaf997808f8b25968ce0c7070945c4265d64a3"}, +] [package.dependencies] setuptools = "*" @@ -395,9 +856,12 @@ wheel = "*" name = "nvidia-cuda-runtime-cu11" version = "11.7.99" description = "CUDA Runtime native Libraries" -category = "dev" optional = false python-versions = ">=3" +files = [ + {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:cc768314ae58d2641f07eac350f40f99dcb35719c4faff4bc458a7cd2b119e31"}, + {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:bc77fa59a7679310df9d5c70ab13c4e34c64ae2124dd1efd7e5474b71be125c7"}, +] [package.dependencies] setuptools = "*" @@ -407,9 +871,12 @@ wheel = "*" name = "nvidia-cudnn-cu11" version = "8.5.0.96" description = "cuDNN runtime libraries" -category = "dev" optional = false python-versions = ">=3" +files = [ + {file = "nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:402f40adfc6f418f9dae9ab402e773cfed9beae52333f6d86ae3107a1b9527e7"}, + {file = "nvidia_cudnn_cu11-8.5.0.96-py3-none-manylinux1_x86_64.whl", hash = "sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108"}, +] [package.dependencies] setuptools = "*" @@ -419,21 +886,27 @@ wheel = "*" name = "omegaconf" version = "2.3.0" description = "A flexible configuration library" -category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b"}, + {file = "omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7"}, +] [package.dependencies] -antlr4-python3-runtime = ">=4.9.0,<4.10.0" +antlr4-python3-runtime = "==4.9.*" PyYAML = ">=5.1.0" [[package]] name = "opt-einsum" version = "3.3.0" description = "Optimizing numpys einsum function" -category = "dev" optional = false python-versions = ">=3.5" +files = [ + {file = "opt_einsum-3.3.0-py3-none-any.whl", hash = "sha256:2455e59e3947d3c275477df7f5205b30635e266fe6dc300e3d9f9646bfcea147"}, + {file = "opt_einsum-3.3.0.tar.gz", hash = "sha256:59f6475f77bbc37dcf7cd748519c0ec60722e91e63ca114e68821c0c54a46549"}, +] [package.dependencies] numpy = ">=1.7" @@ -446,9 +919,12 @@ tests = ["pytest", "pytest-cov", "pytest-pep8"] name = "optax" version = "0.1.4" description = "A gradient processing and optimisation library in JAX." -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "optax-0.1.4-py3-none-any.whl", hash = "sha256:12fcf33bd682f9a162a3deb097f864130c3224d76771af2ba09410de80399a9b"}, + {file = "optax-0.1.4.tar.gz", hash = "sha256:fb7a0550d57a6636164a3de25986a8a19be8ff6431fcdf1225b4e05175810f22"}, +] [package.dependencies] absl-py = ">=0.7.1" @@ -460,11 +936,14 @@ typing-extensions = ">=3.10.0" [[package]] name = "orbax" -version = "0.0.23" +version = "0.1.0" description = "Orbax" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "orbax-0.1.0-py3-none-any.whl", hash = "sha256:df2b5534cb379e1c72a8829bbb62a2c7ebc0e06567ad98e4b06e6aced5d231b1"}, + {file = "orbax-0.1.0.tar.gz", hash = "sha256:317fae101b504dfea9204d667fd79ad7039395b123356995e13a01332e996595"}, +] [package.dependencies] absl-py = "*" @@ -484,31 +963,104 @@ dev = ["pytest-xdist"] [[package]] name = "packaging" -version = "22.0" +version = "23.1" description = "Core utilities for Python packages" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] [[package]] -name = "Pillow" -version = "9.3.0" +name = "pillow" +version = "9.5.0" description = "Python Imaging Library (Fork)" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "Pillow-9.5.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:ace6ca218308447b9077c14ea4ef381ba0b67ee78d64046b3f19cf4e1139ad16"}, + {file = "Pillow-9.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d3d403753c9d5adc04d4694d35cf0391f0f3d57c8e0030aac09d7678fa8030aa"}, + {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ba1b81ee69573fe7124881762bb4cd2e4b6ed9dd28c9c60a632902fe8db8b38"}, + {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe7e1c262d3392afcf5071df9afa574544f28eac825284596ac6db56e6d11062"}, + {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f36397bf3f7d7c6a3abdea815ecf6fd14e7fcd4418ab24bae01008d8d8ca15e"}, + {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:252a03f1bdddce077eff2354c3861bf437c892fb1832f75ce813ee94347aa9b5"}, + {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:85ec677246533e27770b0de5cf0f9d6e4ec0c212a1f89dfc941b64b21226009d"}, + {file = "Pillow-9.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b416f03d37d27290cb93597335a2f85ed446731200705b22bb927405320de903"}, + {file = "Pillow-9.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1781a624c229cb35a2ac31cc4a77e28cafc8900733a864870c49bfeedacd106a"}, + {file = "Pillow-9.5.0-cp310-cp310-win32.whl", hash = "sha256:8507eda3cd0608a1f94f58c64817e83ec12fa93a9436938b191b80d9e4c0fc44"}, + {file = "Pillow-9.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:d3c6b54e304c60c4181da1c9dadf83e4a54fd266a99c70ba646a9baa626819eb"}, + {file = "Pillow-9.5.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:7ec6f6ce99dab90b52da21cf0dc519e21095e332ff3b399a357c187b1a5eee32"}, + {file = "Pillow-9.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:560737e70cb9c6255d6dcba3de6578a9e2ec4b573659943a5e7e4af13f298f5c"}, + {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96e88745a55b88a7c64fa49bceff363a1a27d9a64e04019c2281049444a571e3"}, + {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d9c206c29b46cfd343ea7cdfe1232443072bbb270d6a46f59c259460db76779a"}, + {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cfcc2c53c06f2ccb8976fb5c71d448bdd0a07d26d8e07e321c103416444c7ad1"}, + {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:a0f9bb6c80e6efcde93ffc51256d5cfb2155ff8f78292f074f60f9e70b942d99"}, + {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:8d935f924bbab8f0a9a28404422da8af4904e36d5c33fc6f677e4c4485515625"}, + {file = "Pillow-9.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fed1e1cf6a42577953abbe8e6cf2fe2f566daebde7c34724ec8803c4c0cda579"}, + {file = "Pillow-9.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c1170d6b195555644f0616fd6ed929dfcf6333b8675fcca044ae5ab110ded296"}, + {file = "Pillow-9.5.0-cp311-cp311-win32.whl", hash = "sha256:54f7102ad31a3de5666827526e248c3530b3a33539dbda27c6843d19d72644ec"}, + {file = "Pillow-9.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:cfa4561277f677ecf651e2b22dc43e8f5368b74a25a8f7d1d4a3a243e573f2d4"}, + {file = "Pillow-9.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:965e4a05ef364e7b973dd17fc765f42233415974d773e82144c9bbaaaea5d089"}, + {file = "Pillow-9.5.0-cp312-cp312-win32.whl", hash = "sha256:22baf0c3cf0c7f26e82d6e1adf118027afb325e703922c8dfc1d5d0156bb2eeb"}, + {file = "Pillow-9.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:432b975c009cf649420615388561c0ce7cc31ce9b2e374db659ee4f7d57a1f8b"}, + {file = "Pillow-9.5.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:5d4ebf8e1db4441a55c509c4baa7a0587a0210f7cd25fcfe74dbbce7a4bd1906"}, + {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:375f6e5ee9620a271acb6820b3d1e94ffa8e741c0601db4c0c4d3cb0a9c224bf"}, + {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99eb6cafb6ba90e436684e08dad8be1637efb71c4f2180ee6b8f940739406e78"}, + {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dfaaf10b6172697b9bceb9a3bd7b951819d1ca339a5ef294d1f1ac6d7f63270"}, + {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:763782b2e03e45e2c77d7779875f4432e25121ef002a41829d8868700d119392"}, + {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:35f6e77122a0c0762268216315bf239cf52b88865bba522999dc38f1c52b9b47"}, + {file = "Pillow-9.5.0-cp37-cp37m-win32.whl", hash = "sha256:aca1c196f407ec7cf04dcbb15d19a43c507a81f7ffc45b690899d6a76ac9fda7"}, + {file = "Pillow-9.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322724c0032af6692456cd6ed554bb85f8149214d97398bb80613b04e33769f6"}, + {file = "Pillow-9.5.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:a0aa9417994d91301056f3d0038af1199eb7adc86e646a36b9e050b06f526597"}, + {file = "Pillow-9.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f8286396b351785801a976b1e85ea88e937712ee2c3ac653710a4a57a8da5d9c"}, + {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c830a02caeb789633863b466b9de10c015bded434deb3ec87c768e53752ad22a"}, + {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fbd359831c1657d69bb81f0db962905ee05e5e9451913b18b831febfe0519082"}, + {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8fc330c3370a81bbf3f88557097d1ea26cd8b019d6433aa59f71195f5ddebbf"}, + {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:7002d0797a3e4193c7cdee3198d7c14f92c0836d6b4a3f3046a64bd1ce8df2bf"}, + {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:229e2c79c00e85989a34b5981a2b67aa079fd08c903f0aaead522a1d68d79e51"}, + {file = "Pillow-9.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9adf58f5d64e474bed00d69bcd86ec4bcaa4123bfa70a65ce72e424bfb88ed96"}, + {file = "Pillow-9.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:662da1f3f89a302cc22faa9f14a262c2e3951f9dbc9617609a47521c69dd9f8f"}, + {file = "Pillow-9.5.0-cp38-cp38-win32.whl", hash = "sha256:6608ff3bf781eee0cd14d0901a2b9cc3d3834516532e3bd673a0a204dc8615fc"}, + {file = "Pillow-9.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:e49eb4e95ff6fd7c0c402508894b1ef0e01b99a44320ba7d8ecbabefddcc5569"}, + {file = "Pillow-9.5.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:482877592e927fd263028c105b36272398e3e1be3269efda09f6ba21fd83ec66"}, + {file = "Pillow-9.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3ded42b9ad70e5f1754fb7c2e2d6465a9c842e41d178f262e08b8c85ed8a1d8e"}, + {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c446d2245ba29820d405315083d55299a796695d747efceb5717a8b450324115"}, + {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aca1152d93dcc27dc55395604dcfc55bed5f25ef4c98716a928bacba90d33a3"}, + {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:608488bdcbdb4ba7837461442b90ea6f3079397ddc968c31265c1e056964f1ef"}, + {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:60037a8db8750e474af7ffc9faa9b5859e6c6d0a50e55c45576bf28be7419705"}, + {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:07999f5834bdc404c442146942a2ecadd1cb6292f5229f4ed3b31e0a108746b1"}, + {file = "Pillow-9.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a127ae76092974abfbfa38ca2d12cbeddcdeac0fb71f9627cc1135bedaf9d51a"}, + {file = "Pillow-9.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:489f8389261e5ed43ac8ff7b453162af39c3e8abd730af8363587ba64bb2e865"}, + {file = "Pillow-9.5.0-cp39-cp39-win32.whl", hash = "sha256:9b1af95c3a967bf1da94f253e56b6286b50af23392a886720f563c547e48e964"}, + {file = "Pillow-9.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:77165c4a5e7d5a284f10a6efaa39a0ae8ba839da344f20b111d62cc932fa4e5d"}, + {file = "Pillow-9.5.0-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:833b86a98e0ede388fa29363159c9b1a294b0905b5128baf01db683672f230f5"}, + {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aaf305d6d40bd9632198c766fb64f0c1a83ca5b667f16c1e79e1661ab5060140"}, + {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0852ddb76d85f127c135b6dd1f0bb88dbb9ee990d2cd9aa9e28526c93e794fba"}, + {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:91ec6fe47b5eb5a9968c79ad9ed78c342b1f97a091677ba0e012701add857829"}, + {file = "Pillow-9.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cb841572862f629b99725ebaec3287fc6d275be9b14443ea746c1dd325053cbd"}, + {file = "Pillow-9.5.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:c380b27d041209b849ed246b111b7c166ba36d7933ec6e41175fd15ab9eb1572"}, + {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c9af5a3b406a50e313467e3565fc99929717f780164fe6fbb7704edba0cebbe"}, + {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5671583eab84af046a397d6d0ba25343c00cd50bce03787948e0fff01d4fd9b1"}, + {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:84a6f19ce086c1bf894644b43cd129702f781ba5751ca8572f08aa40ef0ab7b7"}, + {file = "Pillow-9.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1e7723bd90ef94eda669a3c2c19d549874dd5badaeefabefd26053304abe5799"}, + {file = "Pillow-9.5.0.tar.gz", hash = "sha256:bf548479d336726d7a0eceb6e767e179fbde37833ae42794602631a070d630f1"}, +] [package.extras] -docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-issues (>=3.0.1)", "sphinx-removed-in", "sphinxext-opengraph"] +docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] [[package]] name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] [package.dependencies] importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} @@ -519,26 +1071,66 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pydantic" -version = "1.10.2" +version = "1.10.8" description = "Data validation and settings management using python type hints" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1243d28e9b05003a89d72e7915fdb26ffd1d39bdd39b00b7dbe4afae4b557f9d"}, + {file = "pydantic-1.10.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0ab53b609c11dfc0c060d94335993cc2b95b2150e25583bec37a49b2d6c6c3f"}, + {file = "pydantic-1.10.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9613fadad06b4f3bc5db2653ce2f22e0de84a7c6c293909b48f6ed37b83c61f"}, + {file = "pydantic-1.10.8-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df7800cb1984d8f6e249351139667a8c50a379009271ee6236138a22a0c0f319"}, + {file = "pydantic-1.10.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0c6fafa0965b539d7aab0a673a046466d23b86e4b0e8019d25fd53f4df62c277"}, + {file = "pydantic-1.10.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e82d4566fcd527eae8b244fa952d99f2ca3172b7e97add0b43e2d97ee77f81ab"}, + {file = "pydantic-1.10.8-cp310-cp310-win_amd64.whl", hash = "sha256:ab523c31e22943713d80d8d342d23b6f6ac4b792a1e54064a8d0cf78fd64e800"}, + {file = "pydantic-1.10.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:666bdf6066bf6dbc107b30d034615d2627e2121506c555f73f90b54a463d1f33"}, + {file = "pydantic-1.10.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:35db5301b82e8661fa9c505c800d0990bc14e9f36f98932bb1d248c0ac5cada5"}, + {file = "pydantic-1.10.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f90c1e29f447557e9e26afb1c4dbf8768a10cc676e3781b6a577841ade126b85"}, + {file = "pydantic-1.10.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93e766b4a8226e0708ef243e843105bf124e21331694367f95f4e3b4a92bbb3f"}, + {file = "pydantic-1.10.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88f195f582851e8db960b4a94c3e3ad25692c1c1539e2552f3df7a9e972ef60e"}, + {file = "pydantic-1.10.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:34d327c81e68a1ecb52fe9c8d50c8a9b3e90d3c8ad991bfc8f953fb477d42fb4"}, + {file = "pydantic-1.10.8-cp311-cp311-win_amd64.whl", hash = "sha256:d532bf00f381bd6bc62cabc7d1372096b75a33bc197a312b03f5838b4fb84edd"}, + {file = "pydantic-1.10.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7d5b8641c24886d764a74ec541d2fc2c7fb19f6da2a4001e6d580ba4a38f7878"}, + {file = "pydantic-1.10.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b1f6cb446470b7ddf86c2e57cd119a24959af2b01e552f60705910663af09a4"}, + {file = "pydantic-1.10.8-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c33b60054b2136aef8cf190cd4c52a3daa20b2263917c49adad20eaf381e823b"}, + {file = "pydantic-1.10.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1952526ba40b220b912cdc43c1c32bcf4a58e3f192fa313ee665916b26befb68"}, + {file = "pydantic-1.10.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bb14388ec45a7a0dc429e87def6396f9e73c8c77818c927b6a60706603d5f2ea"}, + {file = "pydantic-1.10.8-cp37-cp37m-win_amd64.whl", hash = "sha256:16f8c3e33af1e9bb16c7a91fc7d5fa9fe27298e9f299cff6cb744d89d573d62c"}, + {file = "pydantic-1.10.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1ced8375969673929809d7f36ad322934c35de4af3b5e5b09ec967c21f9f7887"}, + {file = "pydantic-1.10.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:93e6bcfccbd831894a6a434b0aeb1947f9e70b7468f274154d03d71fabb1d7c6"}, + {file = "pydantic-1.10.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:191ba419b605f897ede9892f6c56fb182f40a15d309ef0142212200a10af4c18"}, + {file = "pydantic-1.10.8-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:052d8654cb65174d6f9490cc9b9a200083a82cf5c3c5d3985db765757eb3b375"}, + {file = "pydantic-1.10.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ceb6a23bf1ba4b837d0cfe378329ad3f351b5897c8d4914ce95b85fba96da5a1"}, + {file = "pydantic-1.10.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f2e754d5566f050954727c77f094e01793bcb5725b663bf628fa6743a5a9108"}, + {file = "pydantic-1.10.8-cp38-cp38-win_amd64.whl", hash = "sha256:6a82d6cda82258efca32b40040228ecf43a548671cb174a1e81477195ed3ed56"}, + {file = "pydantic-1.10.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e59417ba8a17265e632af99cc5f35ec309de5980c440c255ab1ca3ae96a3e0e"}, + {file = "pydantic-1.10.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84d80219c3f8d4cad44575e18404099c76851bc924ce5ab1c4c8bb5e2a2227d0"}, + {file = "pydantic-1.10.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e4148e635994d57d834be1182a44bdb07dd867fa3c2d1b37002000646cc5459"}, + {file = "pydantic-1.10.8-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12f7b0bf8553e310e530e9f3a2f5734c68699f42218bf3568ef49cd9b0e44df4"}, + {file = "pydantic-1.10.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:42aa0c4b5c3025483240a25b09f3c09a189481ddda2ea3a831a9d25f444e03c1"}, + {file = "pydantic-1.10.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17aef11cc1b997f9d574b91909fed40761e13fac438d72b81f902226a69dac01"}, + {file = "pydantic-1.10.8-cp39-cp39-win_amd64.whl", hash = "sha256:66a703d1983c675a6e0fed8953b0971c44dba48a929a2000a493c3772eb61a5a"}, + {file = "pydantic-1.10.8-py3-none-any.whl", hash = "sha256:7456eb22ed9aaa24ff3e7b4757da20d9e5ce2a81018c1b3ebd81a0b88a18f3b2"}, + {file = "pydantic-1.10.8.tar.gz", hash = "sha256:1410275520dfa70effadf4c21811d755e7ef9bb1f1d077a21958153a92c8d9ca"}, +] [package.dependencies] -typing-extensions = ">=4.1.0" +typing-extensions = ">=4.2.0" [package.extras] dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] [[package]] -name = "Pygments" -version = "2.13.0" +name = "pygments" +version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" +files = [ + {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, + {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, +] [package.extras] plugins = ["importlib-metadata"] @@ -547,20 +1139,26 @@ plugins = ["importlib-metadata"] name = "pyparsing" version = "3.0.9" description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "dev" optional = false python-versions = ">=3.6.8" +files = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] [package.extras] diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pyright" -version = "1.1.285" +version = "1.1.310" description = "Command line wrapper for pyright" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pyright-1.1.310-py3-none-any.whl", hash = "sha256:55995ac76bf56cb7a44193b7b1ffafc573abab1f1dbc9f62d327f2a1768b3bda"}, + {file = "pyright-1.1.310.tar.gz", hash = "sha256:9e95335a678db2717eaa0c867d61f9399e916289f4a9f47d993e0df74e7d7391"}, +] [package.dependencies] nodeenv = ">=1.6.0" @@ -572,14 +1170,16 @@ dev = ["twine (>=3.4.1)"] [[package]] name = "pytest" -version = "7.2.0" +version = "7.3.1" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, + {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, +] [package.dependencies] -attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} @@ -589,15 +1189,18 @@ pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] [[package]] name = "pytest-cov" version = "3.0.0" description = "Pytest plugin for measuring coverage." -category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, + {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, +] [package.dependencies] coverage = {version = ">=5.2.1", extras = ["toml"]} @@ -610,101 +1213,177 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] [package.dependencies] six = ">=1.5" [[package]] -name = "PyYAML" +name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] [[package]] name = "rich" -version = "12.6.0" +version = "13.3.5" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "main" optional = false -python-versions = ">=3.6.3,<4.0.0" +python-versions = ">=3.7.0" +files = [ + {file = "rich-13.3.5-py3-none-any.whl", hash = "sha256:69cdf53799e63f38b95b9bf9c875f8c90e78dd62b2f00c13a911c7a3b9fa4704"}, + {file = "rich-13.3.5.tar.gz", hash = "sha256:2d11b9b8dd03868f09b4fffadc84a6a8cda574e40dc90821bd845720ebb8e89c"}, +] [package.dependencies] -commonmark = ">=0.9.0,<0.10.0" -pygments = ">=2.6.0,<3.0.0" +markdown-it-py = ">=2.2.0,<3.0.0" +pygments = ">=2.13.0,<3.0.0" typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} [package.extras] -jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] +jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "scipy" version = "1.6.1" description = "SciPy: Scientific Library for Python" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "scipy-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a15a1f3fc0abff33e792d6049161b7795909b40b97c6cc2934ed54384017ab76"}, + {file = "scipy-1.6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:e79570979ccdc3d165456dd62041d9556fb9733b86b4b6d818af7a0afc15f092"}, + {file = "scipy-1.6.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a423533c55fec61456dedee7b6ee7dce0bb6bfa395424ea374d25afa262be261"}, + {file = "scipy-1.6.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:33d6b7df40d197bdd3049d64e8e680227151673465e5d85723b3b8f6b15a6ced"}, + {file = "scipy-1.6.1-cp37-cp37m-win32.whl", hash = "sha256:6725e3fbb47da428794f243864f2297462e9ee448297c93ed1dcbc44335feb78"}, + {file = "scipy-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:5fa9c6530b1661f1370bcd332a1e62ca7881785cc0f80c0d559b636567fab63c"}, + {file = "scipy-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bd50daf727f7c195e26f27467c85ce653d41df4358a25b32434a50d8870fc519"}, + {file = "scipy-1.6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f46dd15335e8a320b0fb4685f58b7471702234cba8bb3442b69a3e1dc329c345"}, + {file = "scipy-1.6.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0e5b0ccf63155d90da576edd2768b66fb276446c371b73841e3503be1d63fb5d"}, + {file = "scipy-1.6.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2481efbb3740977e3c831edfd0bd9867be26387cacf24eb5e366a6a374d3d00d"}, + {file = "scipy-1.6.1-cp38-cp38-win32.whl", hash = "sha256:68cb4c424112cd4be886b4d979c5497fba190714085f46b8ae67a5e4416c32b4"}, + {file = "scipy-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:5f331eeed0297232d2e6eea51b54e8278ed8bb10b099f69c44e2558c090d06bf"}, + {file = "scipy-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0c8a51d33556bf70367452d4d601d1742c0e806cd0194785914daf19775f0e67"}, + {file = "scipy-1.6.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:83bf7c16245c15bc58ee76c5418e46ea1811edcc2e2b03041b804e46084ab627"}, + {file = "scipy-1.6.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:794e768cc5f779736593046c9714e0f3a5940bc6dcc1dba885ad64cbfb28e9f0"}, + {file = "scipy-1.6.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5da5471aed911fe7e52b86bf9ea32fb55ae93e2f0fac66c32e58897cfb02fa07"}, + {file = "scipy-1.6.1-cp39-cp39-win32.whl", hash = "sha256:8e403a337749ed40af60e537cc4d4c03febddcc56cd26e774c9b1b600a70d3e4"}, + {file = "scipy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a5193a098ae9f29af283dcf0041f762601faf2e595c0db1da929875b7570353f"}, + {file = "scipy-1.6.1.tar.gz", hash = "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11"}, +] [package.dependencies] numpy = ">=1.16.5" [[package]] name = "setuptools" -version = "65.6.3" +version = "67.8.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "setuptools-67.8.0-py3-none-any.whl", hash = "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f"}, + {file = "setuptools-67.8.0.tar.gz", hash = "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102"}, +] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] -[[package]] -name = "setuptools-scm" -version = "6.4.2" -description = "the blessed package to manage your versions by scm tags" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -packaging = ">=20.0" -setuptools = "*" -tomli = ">=1.0.0" - -[package.extras] -test = ["pytest (>=6.2)", "virtualenv (>20)"] -toml = ["setuptools (>=42)"] - [[package]] name = "shtab" -version = "1.5.8" +version = "1.6.1" description = "Automagic shell tab completion for Python CLI applications" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "shtab-1.6.1-py3-none-any.whl", hash = "sha256:db2c41d81a61c7ecefd1dad8212bdc4b667e7449b8172aa6445f557f901810c4"}, + {file = "shtab-1.6.1.tar.gz", hash = "sha256:decc78082c3ffb518c1dfd3a8da99653a2d47e58e3104197bce8ae6507dad78b"}, +] [[package]] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] [[package]] name = "tensorstore" version = "0.1.28" description = "Read and write large, multi-dimensional arrays" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "tensorstore-0.1.28-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:deb32f1e74ab7b836ecec02a759a558fc57522a8dea0db4f19a01a78e93abab5"}, + {file = "tensorstore-0.1.28-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bd8615308ad3e29111238dfdd3aca1f3fb79b55c3dac1c04c96c8a319985addf"}, + {file = "tensorstore-0.1.28-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee16297a9493ed0d6d1000f216e79d29a0b80e7e97646e8520afd44138165839"}, + {file = "tensorstore-0.1.28-cp310-cp310-win_amd64.whl", hash = "sha256:b6f7e0c7455d9e164e6143714519e7c96cc4166b61ed7d268a8ab84a77e0d593"}, + {file = "tensorstore-0.1.28-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:1c53e1e0e7996b8ba41854135d748c2bc8802b94a79122b56448aeb1f77efd9f"}, + {file = "tensorstore-0.1.28-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bd122a44db81369808c84b7739ad2d8453f89c548c5d61d62f2b1f3331dbe9f"}, + {file = "tensorstore-0.1.28-cp37-cp37m-win_amd64.whl", hash = "sha256:197159da3ee97c08d8769fa151ff167328108bf07a13886f688b21f64abfb7d6"}, + {file = "tensorstore-0.1.28-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:e480da4f95f7c95302c4b3e2a7fc32b8cfe1aa51742d72f84a335e49462a2465"}, + {file = "tensorstore-0.1.28-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7948ead0d5a55303ffa97a35659aa56a20c60bd89d9487643f51ac8d2e19b52c"}, + {file = "tensorstore-0.1.28-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:818aeb015b249069a686fb20f0c0b71663410ac6411cee88df18b8e05b567cbc"}, + {file = "tensorstore-0.1.28-cp38-cp38-win_amd64.whl", hash = "sha256:6719eaba4ec2692d890c3c67c389fcae073d40d9e8c27e2dabd27fc9fd745e5c"}, + {file = "tensorstore-0.1.28-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:d7c86f63a6d4d7a84e3201ba9746ca51015612d375d77b9302eb93e9b88bc7aa"}, + {file = "tensorstore-0.1.28-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dc82cbb77255fd2f7aa191f1e030f13276bf7272ea1027fcc6900537b30bfbcb"}, + {file = "tensorstore-0.1.28-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4de09e3fae30f8a9ab7648600c78e109ddd1d14af4280e055439fdd52a6dca9"}, + {file = "tensorstore-0.1.28-cp39-cp39-win_amd64.whl", hash = "sha256:f9270586401ee60ff79a4cd54c62ab5b06831d69b561df1fb9ebbeade4d4929c"}, + {file = "tensorstore-0.1.28.tar.gz", hash = "sha256:cd8d8185136632c58edcd7cf4c43d301bf8ad61a197c632cb495d7d33b7be04e"}, +] [package.dependencies] numpy = ">=1.16.0" @@ -713,25 +1392,53 @@ numpy = ">=1.16.0" name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "main" optional = false python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] [[package]] name = "toolz" version = "0.12.0" description = "List processing tools and functional utilities" -category = "dev" optional = false python-versions = ">=3.5" +files = [ + {file = "toolz-0.12.0-py3-none-any.whl", hash = "sha256:2059bd4148deb1884bb0eb770a3cde70e7f954cfbbdc2285f1f2de01fd21eb6f"}, + {file = "toolz-0.12.0.tar.gz", hash = "sha256:88c570861c440ee3f2f6037c4654613228ff40c93a6c25e0eba70d17282c6194"}, +] [[package]] name = "torch" version = "1.13.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -category = "dev" optional = false python-versions = ">=3.7.0" +files = [ + {file = "torch-1.13.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:fd12043868a34a8da7d490bf6db66991108b00ffbeecb034228bfcbbd4197143"}, + {file = "torch-1.13.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:d9fe785d375f2e26a5d5eba5de91f89e6a3be5d11efb497e76705fdf93fa3c2e"}, + {file = "torch-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:98124598cdff4c287dbf50f53fb455f0c1e3a88022b39648102957f3445e9b76"}, + {file = "torch-1.13.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:393a6273c832e047581063fb74335ff50b4c566217019cc6ace318cd79eb0566"}, + {file = "torch-1.13.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:0122806b111b949d21fa1a5f9764d1fd2fcc4a47cb7f8ff914204fd4fc752ed5"}, + {file = "torch-1.13.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:22128502fd8f5b25ac1cd849ecb64a418382ae81dd4ce2b5cebaa09ab15b0d9b"}, + {file = "torch-1.13.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:76024be052b659ac1304ab8475ab03ea0a12124c3e7626282c9c86798ac7bc11"}, + {file = "torch-1.13.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:ea8dda84d796094eb8709df0fcd6b56dc20b58fdd6bc4e8d7109930dafc8e419"}, + {file = "torch-1.13.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2ee7b81e9c457252bddd7d3da66fb1f619a5d12c24d7074de91c4ddafb832c93"}, + {file = "torch-1.13.1-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:0d9b8061048cfb78e675b9d2ea8503bfe30db43d583599ae8626b1263a0c1380"}, + {file = "torch-1.13.1-cp37-none-macosx_11_0_arm64.whl", hash = "sha256:f402ca80b66e9fbd661ed4287d7553f7f3899d9ab54bf5c67faada1555abde28"}, + {file = "torch-1.13.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:727dbf00e2cf858052364c0e2a496684b9cb5aa01dc8a8bc8bbb7c54502bdcdd"}, + {file = "torch-1.13.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:df8434b0695e9ceb8cc70650afc1310d8ba949e6db2a0525ddd9c3b2b181e5fe"}, + {file = "torch-1.13.1-cp38-cp38-win_amd64.whl", hash = "sha256:5e1e722a41f52a3f26f0c4fcec227e02c6c42f7c094f32e49d4beef7d1e213ea"}, + {file = "torch-1.13.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:33e67eea526e0bbb9151263e65417a9ef2d8fa53cbe628e87310060c9dcfa312"}, + {file = "torch-1.13.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:eeeb204d30fd40af6a2d80879b46a7efbe3cf43cdbeb8838dd4f3d126cc90b2b"}, + {file = "torch-1.13.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:50ff5e76d70074f6653d191fe4f6a42fdbe0cf942fbe2a3af0b75eaa414ac038"}, + {file = "torch-1.13.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:2c3581a3fd81eb1f0f22997cddffea569fea53bafa372b2c0471db373b26aafc"}, + {file = "torch-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:0aa46f0ac95050c604bcf9ef71da9f1172e5037fdf2ebe051962d47b123848e7"}, + {file = "torch-1.13.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:6930791efa8757cb6974af73d4996b6b50c592882a324b8fb0589c6a9ba2ddaf"}, + {file = "torch-1.13.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:e0df902a7c7dd6c795698532ee5970ce898672625635d885eade9976e5a04949"}, +] [package.dependencies] nvidia-cublas-cu11 = {version = "11.10.3.66", markers = "platform_system == \"Linux\""} @@ -747,760 +1454,9 @@ opt-einsum = ["opt-einsum (>=3.3)"] name = "typed-ast" version = "1.5.4" description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "main" optional = false python-versions = ">=3.6" - -[[package]] -name = "typing-extensions" -version = "4.4.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "wheel" -version = "0.38.4" -description = "A built-package format for Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -test = ["pytest (>=3.0.0)"] - -[[package]] -name = "zipp" -version = "3.11.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[metadata] -lock-version = "1.1" -python-versions = "^3.7" -content-hash = "08c358613cee83d6f1992fd715ed61144442e3bebdc746f79a1afe30fef7b58a" - -[metadata.files] -absl-py = [ - {file = "absl-py-1.3.0.tar.gz", hash = "sha256:463c38a08d2e4cef6c498b76ba5bd4858e4c6ef51da1a5a1f27139a022e20248"}, - {file = "absl_py-1.3.0-py3-none-any.whl", hash = "sha256:34995df9bd7a09b3b8749e230408f5a2a2dd7a68a0d33c12a3d0cb15a041a507"}, -] -antlr4-python3-runtime = [ - {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, -] -attrs = [ - {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, - {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, -] -"backports.cached-property" = [ - {file = "backports.cached-property-1.0.2.tar.gz", hash = "sha256:9306f9eed6ec55fd156ace6bc1094e2c86fae5fb2bf07b6a9c00745c656e75dd"}, - {file = "backports.cached_property-1.0.2-py3-none-any.whl", hash = "sha256:baeb28e1cd619a3c9ab8941431fe34e8490861fb998c6c4590693d50171db0cc"}, -] -cached-property = [ - {file = "cached-property-1.5.2.tar.gz", hash = "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130"}, - {file = "cached_property-1.5.2-py2.py3-none-any.whl", hash = "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0"}, -] -chex = [ - {file = "chex-0.1.5-py3-none-any.whl", hash = "sha256:b3321184850d5fc29b2eca63087cdbdd83a1b3e4f33c1314ff8b3b8bd67abbca"}, - {file = "chex-0.1.5.tar.gz", hash = "sha256:686858320f8f220c82a6c7eeb54dcdcaa4f3d7f66690dacd13a24baa1ee8299e"}, -] -colorama = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] -commonmark = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] -coverage = [ - {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, - {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, - {file = "coverage-6.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4"}, - {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83516205e254a0cb77d2d7bb3632ee019d93d9f4005de31dca0a8c3667d5bc04"}, - {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0"}, - {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:97117225cdd992a9c2a5515db1f66b59db634f59d0679ca1fa3fe8da32749cae"}, - {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1170fa54185845505fbfa672f1c1ab175446c887cce8212c44149581cf2d466"}, - {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:11b990d520ea75e7ee8dcab5bc908072aaada194a794db9f6d7d5cfd19661e5a"}, - {file = "coverage-6.5.0-cp310-cp310-win32.whl", hash = "sha256:5dbec3b9095749390c09ab7c89d314727f18800060d8d24e87f01fb9cfb40b32"}, - {file = "coverage-6.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:59f53f1dc5b656cafb1badd0feb428c1e7bc19b867479ff72f7a9dd9b479f10e"}, - {file = "coverage-6.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795"}, - {file = "coverage-6.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75"}, - {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33a7da4376d5977fbf0a8ed91c4dffaaa8dbf0ddbf4c8eea500a2486d8bc4d7b"}, - {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91"}, - {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a6b7d95969b8845250586f269e81e5dfdd8ff828ddeb8567a4a2eaa7313460c4"}, - {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1ef221513e6f68b69ee9e159506d583d31aa3567e0ae84eaad9d6ec1107dddaa"}, - {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cca4435eebea7962a52bdb216dec27215d0df64cf27fc1dd538415f5d2b9da6b"}, - {file = "coverage-6.5.0-cp311-cp311-win32.whl", hash = "sha256:98e8a10b7a314f454d9eff4216a9a94d143a7ee65018dd12442e898ee2310578"}, - {file = "coverage-6.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:bc8ef5e043a2af066fa8cbfc6e708d58017024dc4345a1f9757b329a249f041b"}, - {file = "coverage-6.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4433b90fae13f86fafff0b326453dd42fc9a639a0d9e4eec4d366436d1a41b6d"}, - {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4f05d88d9a80ad3cac6244d36dd89a3c00abc16371769f1340101d3cb899fc3"}, - {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:94e2565443291bd778421856bc975d351738963071e9b8839ca1fc08b42d4bef"}, - {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:027018943386e7b942fa832372ebc120155fd970837489896099f5cfa2890f79"}, - {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:255758a1e3b61db372ec2736c8e2a1fdfaf563977eedbdf131de003ca5779b7d"}, - {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:851cf4ff24062c6aec510a454b2584f6e998cada52d4cb58c5e233d07172e50c"}, - {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:12adf310e4aafddc58afdb04d686795f33f4d7a6fa67a7a9d4ce7d6ae24d949f"}, - {file = "coverage-6.5.0-cp37-cp37m-win32.whl", hash = "sha256:b5604380f3415ba69de87a289a2b56687faa4fe04dbee0754bfcae433489316b"}, - {file = "coverage-6.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4a8dbc1f0fbb2ae3de73eb0bdbb914180c7abfbf258e90b311dcd4f585d44bd2"}, - {file = "coverage-6.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c"}, - {file = "coverage-6.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba"}, - {file = "coverage-6.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e"}, - {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:20c8ac5386253717e5ccc827caad43ed66fea0efe255727b1053a8154d952398"}, - {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b"}, - {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dbdb91cd8c048c2b09eb17713b0c12a54fbd587d79adcebad543bc0cd9a3410b"}, - {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:de3001a203182842a4630e7b8d1a2c7c07ec1b45d3084a83d5d227a3806f530f"}, - {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e07f4a4a9b41583d6eabec04f8b68076ab3cd44c20bd29332c6572dda36f372e"}, - {file = "coverage-6.5.0-cp38-cp38-win32.whl", hash = "sha256:6d4817234349a80dbf03640cec6109cd90cba068330703fa65ddf56b60223a6d"}, - {file = "coverage-6.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:7ccf362abd726b0410bf8911c31fbf97f09f8f1061f8c1cf03dfc4b6372848f6"}, - {file = "coverage-6.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745"}, - {file = "coverage-6.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc"}, - {file = "coverage-6.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe"}, - {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:265de0fa6778d07de30bcf4d9dc471c3dc4314a23a3c6603d356a3c9abc2dfcf"}, - {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5"}, - {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7b6be138d61e458e18d8e6ddcddd36dd96215edfe5f1168de0b1b32635839b62"}, - {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:42eafe6778551cf006a7c43153af1211c3aaab658d4d66fa5fcc021613d02518"}, - {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:723e8130d4ecc8f56e9a611e73b31219595baa3bb252d539206f7bbbab6ffc1f"}, - {file = "coverage-6.5.0-cp39-cp39-win32.whl", hash = "sha256:d9ecf0829c6a62b9b573c7bb6d4dcd6ba8b6f80be9ba4fc7ed50bf4ac9aecd72"}, - {file = "coverage-6.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc2af30ed0d5ae0b1abdb4ebdce598eafd5b35397d4d75deb341a614d333d987"}, - {file = "coverage-6.5.0-pp36.pp37.pp38-none-any.whl", hash = "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a"}, - {file = "coverage-6.5.0.tar.gz", hash = "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84"}, -] -cycler = [ - {file = "cycler-0.11.0-py3-none-any.whl", hash = "sha256:3a27e95f763a428a739d2add979fa7494c912a32c17c4c38c4d5f082cad165a3"}, - {file = "cycler-0.11.0.tar.gz", hash = "sha256:9c87405839a19696e837b3b818fed3f5f69f16f1eec1a1ad77e043dcea9c772f"}, -] -dm-tree = [ - {file = "dm-tree-0.1.8.tar.gz", hash = "sha256:0fcaabbb14e7980377439e7140bd05552739ca5e515ecb3119f234acee4b9430"}, - {file = "dm_tree-0.1.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:35cc164a79336bfcfafb47e5f297898359123bbd3330c1967f0c4994f9cf9f60"}, - {file = "dm_tree-0.1.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39070ba268c0491af9fe7a58644d99e8b4f2cde6e5884ba3380bddc84ed43d5f"}, - {file = "dm_tree-0.1.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2869228d9c619074de501a3c10dc7f07c75422f8fab36ecdcb859b6f1b1ec3ef"}, - {file = "dm_tree-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4d714371bb08839e4e5e29024fc95832d9affe129825ef38836b143028bd144"}, - {file = "dm_tree-0.1.8-cp310-cp310-win_amd64.whl", hash = "sha256:d40fa4106ca6edc66760246a08f500ec0c85ef55c762fb4a363f6ee739ba02ee"}, - {file = "dm_tree-0.1.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad16ceba90a56ec47cf45b21856d14962ac314787975ef786efb5e6e9ca75ec7"}, - {file = "dm_tree-0.1.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:803bfc53b4659f447ac694dbd04235f94a73ef7c1fd1e0df7c84ac41e0bc963b"}, - {file = "dm_tree-0.1.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:378cc8ad93c5fe3590f405a309980721f021c790ca1bdf9b15bb1d59daec57f5"}, - {file = "dm_tree-0.1.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83b7764de0d855338abefc6e3ee9fe40d301668310aa3baea3f778ff051f4393"}, - {file = "dm_tree-0.1.8-cp311-cp311-win_amd64.whl", hash = "sha256:a5d819c38c03f0bb5b3b3703c60e4b170355a0fc6b5819325bf3d4ceb3ae7e80"}, - {file = "dm_tree-0.1.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8c60a7eadab64c2278861f56bca320b2720f163dca9d7558103c3b77f2416571"}, - {file = "dm_tree-0.1.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f7915660f59c09068e428613c480150180df1060561fd0d1470684ae7007bd1"}, - {file = "dm_tree-0.1.8-cp37-cp37m-win_amd64.whl", hash = "sha256:b9f89a454e98806b44fe9d40ec9eee61f848388f7e79ac2371a55679bd5a3ac6"}, - {file = "dm_tree-0.1.8-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0e9620ccf06393eb6b613b5e366469304622d4ea96ae6540b28a33840e6c89cf"}, - {file = "dm_tree-0.1.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b095ba4f8ca1ba19350fd53cf1f8f3eb0bd406aa28af64a6dfc86707b32a810a"}, - {file = "dm_tree-0.1.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b9bd9b9ccb59409d33d51d84b7668010c04c2af7d4a371632874c1ca356cff3d"}, - {file = "dm_tree-0.1.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:694c3654cfd2a81552c08ec66bb5c4a3d48fa292b9a181880fb081c36c5b9134"}, - {file = "dm_tree-0.1.8-cp38-cp38-win_amd64.whl", hash = "sha256:bb2d109f42190225112da899b9f3d46d0d5f26aef501c61e43529fe9322530b5"}, - {file = "dm_tree-0.1.8-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d16e1f2a073604cfcc09f7131ae8d534674f43c3aef4c25742eae295bc60d04f"}, - {file = "dm_tree-0.1.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:250b692fb75f45f02e2f58fbef9ab338904ef334b90557565621fa251df267cf"}, - {file = "dm_tree-0.1.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81fce77f22a302d7a5968aebdf4efafef4def7ce96528719a354e6990dcd49c7"}, - {file = "dm_tree-0.1.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:181c35521d480d0365f39300542cb6cd7fd2b77351bb43d7acfda15aef63b317"}, - {file = "dm_tree-0.1.8-cp39-cp39-win_amd64.whl", hash = "sha256:8ed3564abed97c806db122c2d3e1a2b64c74a63debe9903aad795167cc301368"}, -] -docstring-parser = [ - {file = "docstring_parser-0.14.1-py3-none-any.whl", hash = "sha256:14ac6ec1f1ba6905c4d8cb90fd0bc55394f5678183752c90e44812bf28d7a515"}, - {file = "docstring_parser-0.14.1.tar.gz", hash = "sha256:2c77522e31b7c88b1ab457a1f3c9ae38947ad719732260ba77ee8a3deb58622a"}, -] -etils = [ - {file = "etils-0.9.0-py3-none-any.whl", hash = "sha256:635d6f7d1c519eb194304228543a4c5c7df0e6b58243302473e34c18cf720588"}, - {file = "etils-0.9.0.tar.gz", hash = "sha256:489103e9e499a566765c60458ee15d185cf0065f2060a4d16a68f8f46962ed0d"}, -] -exceptiongroup = [ - {file = "exceptiongroup-1.1.0-py3-none-any.whl", hash = "sha256:327cbda3da756e2de031a3107b81ab7b3770a602c4d16ca618298c526f4bec1e"}, - {file = "exceptiongroup-1.1.0.tar.gz", hash = "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23"}, -] -flax = [ - {file = "flax-0.6.3-py3-none-any.whl", hash = "sha256:0cc0830f76a45c54ebe993aaa751319ea609ca1fb036d418d22259a0074a8759"}, - {file = "flax-0.6.3.tar.gz", hash = "sha256:064f33f24f49ecef01c171cc770d22493fb8f9a36ed29db5e75f82d2052682a9"}, -] -fonttools = [ - {file = "fonttools-4.38.0-py3-none-any.whl", hash = "sha256:820466f43c8be8c3009aef8b87e785014133508f0de64ec469e4efb643ae54fb"}, - {file = "fonttools-4.38.0.zip", hash = "sha256:2bb244009f9bf3fa100fc3ead6aeb99febe5985fa20afbfbaa2f8946c2fbdaf1"}, -] -frozendict = [ - {file = "frozendict-2.3.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4a3b32d47282ae0098b9239a6d53ec539da720258bd762d62191b46f2f87c5fc"}, - {file = "frozendict-2.3.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84c9887179a245a66a50f52afa08d4d92ae0f269839fab82285c70a0fa0dd782"}, - {file = "frozendict-2.3.4-cp310-cp310-win_amd64.whl", hash = "sha256:b98a0d65a59af6da03f794f90b0c3085a7ee14e7bf8f0ef36b079ee8aa992439"}, - {file = "frozendict-2.3.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:3d8042b7dab5e992e30889c9b71b781d5feef19b372d47d735e4d7d45846fd4a"}, - {file = "frozendict-2.3.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25a6d2e8b7cf6b6e5677a1a4b53b4073e5d9ec640d1db30dc679627668d25e90"}, - {file = "frozendict-2.3.4-cp36-cp36m-win_amd64.whl", hash = "sha256:dbbe1339ac2646523e0bb00d1896085d1f70de23780e4927ca82b36ab8a044d3"}, - {file = "frozendict-2.3.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95bac22f7f09d81f378f2b3f672b7a50a974ca180feae1507f5e21bc147e8bc8"}, - {file = "frozendict-2.3.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dae686722c144b333c4dbdc16323a5de11406d26b76d2be1cc175f90afacb5ba"}, - {file = "frozendict-2.3.4-cp37-cp37m-win_amd64.whl", hash = "sha256:389f395a74eb16992217ac1521e689c1dea2d70113bcb18714669ace1ed623b9"}, - {file = "frozendict-2.3.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ccb6450a416c9cc9acef7683e637e28356e3ceeabf83521f74cc2718883076b7"}, - {file = "frozendict-2.3.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aca59108b77cadc13ba7dfea7e8f50811208c7652a13dc6c7f92d7782a24d299"}, - {file = "frozendict-2.3.4-cp38-cp38-win_amd64.whl", hash = "sha256:3ec86ebf143dd685184215c27ec416c36e0ba1b80d81b1b9482f7d380c049b4e"}, - {file = "frozendict-2.3.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5809e6ff6b7257043a486f7a3b73a7da71cf69a38980b4171e4741291d0d9eb3"}, - {file = "frozendict-2.3.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c550ed7fdf1962984bec21630c584d722b3ee5d5f57a0ae2527a0121dc0414a"}, - {file = "frozendict-2.3.4-cp39-cp39-win_amd64.whl", hash = "sha256:3e93aebc6e69a8ef329bbe9afb8342bd33c7b5c7a0c480cb9f7e60b0cbe48072"}, - {file = "frozendict-2.3.4-py3-none-any.whl", hash = "sha256:d722f3d89db6ae35ef35ecc243c40c800eb344848c83dba4798353312cd37b15"}, - {file = "frozendict-2.3.4.tar.gz", hash = "sha256:15b4b18346259392b0d27598f240e9390fafbff882137a9c48a1e0104fb17f78"}, -] -importlib-metadata = [ - {file = "importlib_metadata-5.2.0-py3-none-any.whl", hash = "sha256:0eafa39ba42bf225fc00e67f701d71f85aead9f878569caf13c3724f704b970f"}, - {file = "importlib_metadata-5.2.0.tar.gz", hash = "sha256:404d48d62bba0b7a77ff9d405efd91501bef2e67ff4ace0bed40a0cf28c3c7cd"}, -] -importlib-resources = [ - {file = "importlib_resources-5.10.1-py3-none-any.whl", hash = "sha256:c09b067d82e72c66f4f8eb12332f5efbebc9b007c0b6c40818108c9870adc363"}, - {file = "importlib_resources-5.10.1.tar.gz", hash = "sha256:32bb095bda29741f6ef0e5278c42df98d135391bee5f932841efc0041f748dc3"}, -] -iniconfig = [ - {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, - {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, -] -jax = [ - {file = "jax-0.3.25.tar.gz", hash = "sha256:18bea69321cb95ea5ea913adfe5e2c1d453cade9d4cfd0dc814ecba9fc0cb6e3"}, -] -jaxlib = [ - {file = "jaxlib-0.3.25-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:09508f7000c0fa958fba29267338e8de75b31d7ea29bd79719a568c38f0f8d31"}, - {file = "jaxlib-0.3.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c75c8efd3702687968820446e3fb9ff997f8a2a07ab92e33b80e2f12eab3d9a"}, - {file = "jaxlib-0.3.25-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:6e2f4e51041b8371aa3976b5a3a9cdcdccb1bd7b040c9b1345cbf24bd28a8d19"}, - {file = "jaxlib-0.3.25-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:f2d517635fd77e2729c0ab7863be0d290927d01b2abb2f5dc955c821f8b0d53e"}, - {file = "jaxlib-0.3.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:13446a8382aa9ed944c16af636ca111d0afbbead91eed5cc2dc71195045e71b3"}, - {file = "jaxlib-0.3.25-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:71866aeaafbc9a12b59dcbe443353772ef235b40c53f8bd7403d39311822c276"}, - {file = "jaxlib-0.3.25-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:1e59ba58c9e93c1e1cef243f2609ec0b0c0a81160c20b9555aecdea32ccd6a78"}, - {file = "jaxlib-0.3.25-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:5295354ed5db111e6f3e88cdfa4010d11c33dd926ac61735b9096b4e0746aa7b"}, - {file = "jaxlib-0.3.25-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:9f3116389ee834b3cdeb30001b085f4b55d7741366034f041c1d377154aa5afa"}, - {file = "jaxlib-0.3.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:78b29c72d0680829db9377ed9be326875849258a60b8173b4a388b34ad18bc78"}, - {file = "jaxlib-0.3.25-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:2e008e0a6c10aa7e949555e98dc0471e0d550d5d7c109771e38a971b49480538"}, - {file = "jaxlib-0.3.25-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:fec043cdd55f3257d02e9d8880b33860eacadcae1bd5e26f43fdd08ada23614d"}, - {file = "jaxlib-0.3.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50193ba0cbf879021c9d73d7bcfa7eafb9138895d057b774c301aac3701f9a5"}, - {file = "jaxlib-0.3.25-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:1f1448f102a9d05186f579b6931fa0c607783ecc915fdfaa482c19538affa180"}, -] -kiwisolver = [ - {file = "kiwisolver-1.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2f5e60fabb7343a836360c4f0919b8cd0d6dbf08ad2ca6b9cf90bf0c76a3c4f6"}, - {file = "kiwisolver-1.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:10ee06759482c78bdb864f4109886dff7b8a56529bc1609d4f1112b93fe6423c"}, - {file = "kiwisolver-1.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c79ebe8f3676a4c6630fd3f777f3cfecf9289666c84e775a67d1d358578dc2e3"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:abbe9fa13da955feb8202e215c4018f4bb57469b1b78c7a4c5c7b93001699938"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7577c1987baa3adc4b3c62c33bd1118c3ef5c8ddef36f0f2c950ae0b199e100d"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8ad8285b01b0d4695102546b342b493b3ccc6781fc28c8c6a1bb63e95d22f09"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ed58b8acf29798b036d347791141767ccf65eee7f26bde03a71c944449e53de"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a68b62a02953b9841730db7797422f983935aeefceb1679f0fc85cbfbd311c32"}, - {file = "kiwisolver-1.4.4-cp310-cp310-win32.whl", hash = "sha256:e92a513161077b53447160b9bd8f522edfbed4bd9759e4c18ab05d7ef7e49408"}, - {file = "kiwisolver-1.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:3fe20f63c9ecee44560d0e7f116b3a747a5d7203376abeea292ab3152334d004"}, - {file = "kiwisolver-1.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e0ea21f66820452a3f5d1655f8704a60d66ba1191359b96541eaf457710a5fc6"}, - {file = "kiwisolver-1.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bc9db8a3efb3e403e4ecc6cd9489ea2bac94244f80c78e27c31dcc00d2790ac2"}, - {file = "kiwisolver-1.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d5b61785a9ce44e5a4b880272baa7cf6c8f48a5180c3e81c59553ba0cb0821ca"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2dbb44c3f7e6c4d3487b31037b1bdbf424d97687c1747ce4ff2895795c9bf69"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6295ecd49304dcf3bfbfa45d9a081c96509e95f4b9d0eb7ee4ec0530c4a96514"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4bd472dbe5e136f96a4b18f295d159d7f26fd399136f5b17b08c4e5f498cd494"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bf7d9fce9bcc4752ca4a1b80aabd38f6d19009ea5cbda0e0856983cf6d0023f5"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d6601aed50c74e0ef02f4204da1816147a6d3fbdc8b3872d263338a9052c51"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:877272cf6b4b7e94c9614f9b10140e198d2186363728ed0f701c6eee1baec1da"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:db608a6757adabb32f1cfe6066e39b3706d8c3aa69bbc353a5b61edad36a5cb4"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5853eb494c71e267912275e5586fe281444eb5e722de4e131cddf9d442615626"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:f0a1dbdb5ecbef0d34eb77e56fcb3e95bbd7e50835d9782a45df81cc46949750"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:283dffbf061a4ec60391d51e6155e372a1f7a4f5b15d59c8505339454f8989e4"}, - {file = "kiwisolver-1.4.4-cp311-cp311-win32.whl", hash = "sha256:d06adcfa62a4431d404c31216f0f8ac97397d799cd53800e9d3efc2fbb3cf14e"}, - {file = "kiwisolver-1.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:e7da3fec7408813a7cebc9e4ec55afed2d0fd65c4754bc376bf03498d4e92686"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:62ac9cc684da4cf1778d07a89bf5f81b35834cb96ca523d3a7fb32509380cbf6"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41dae968a94b1ef1897cb322b39360a0812661dba7c682aa45098eb8e193dbdf"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02f79693ec433cb4b5f51694e8477ae83b3205768a6fb48ffba60549080e295b"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0611a0a2a518464c05ddd5a3a1a0e856ccc10e67079bb17f265ad19ab3c7597"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:db5283d90da4174865d520e7366801a93777201e91e79bacbac6e6927cbceede"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1041feb4cda8708ce73bb4dcb9ce1ccf49d553bf87c3954bdfa46f0c3f77252c"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-win32.whl", hash = "sha256:a553dadda40fef6bfa1456dc4be49b113aa92c2a9a9e8711e955618cd69622e3"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:03baab2d6b4a54ddbb43bba1a3a2d1627e82d205c5cf8f4c924dc49284b87166"}, - {file = "kiwisolver-1.4.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:841293b17ad704d70c578f1f0013c890e219952169ce8a24ebc063eecf775454"}, - {file = "kiwisolver-1.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f4f270de01dd3e129a72efad823da90cc4d6aafb64c410c9033aba70db9f1ff0"}, - {file = "kiwisolver-1.4.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f9f39e2f049db33a908319cf46624a569b36983c7c78318e9726a4cb8923b26c"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c97528e64cb9ebeff9701e7938653a9951922f2a38bd847787d4a8e498cc83ae"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d1573129aa0fd901076e2bfb4275a35f5b7aa60fbfb984499d661ec950320b0"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad881edc7ccb9d65b0224f4e4d05a1e85cf62d73aab798943df6d48ab0cd79a1"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b428ef021242344340460fa4c9185d0b1f66fbdbfecc6c63eff4b7c29fad429d"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2e407cb4bd5a13984a6c2c0fe1845e4e41e96f183e5e5cd4d77a857d9693494c"}, - {file = "kiwisolver-1.4.4-cp38-cp38-win32.whl", hash = "sha256:75facbe9606748f43428fc91a43edb46c7ff68889b91fa31f53b58894503a191"}, - {file = "kiwisolver-1.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:5bce61af018b0cb2055e0e72e7d65290d822d3feee430b7b8203d8a855e78766"}, - {file = "kiwisolver-1.4.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8c808594c88a025d4e322d5bb549282c93c8e1ba71b790f539567932722d7bd8"}, - {file = "kiwisolver-1.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0a71d85ecdd570ded8ac3d1c0f480842f49a40beb423bb8014539a9f32a5897"}, - {file = "kiwisolver-1.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b533558eae785e33e8c148a8d9921692a9fe5aa516efbdff8606e7d87b9d5824"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:efda5fc8cc1c61e4f639b8067d118e742b812c930f708e6667a5ce0d13499e29"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7c43e1e1206cd421cd92e6b3280d4385d41d7166b3ed577ac20444b6995a445f"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc8d3bd6c72b2dd9decf16ce70e20abcb3274ba01b4e1c96031e0c4067d1e7cd"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ea39b0ccc4f5d803e3337dd46bcce60b702be4d86fd0b3d7531ef10fd99a1ac"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:968f44fdbf6dd757d12920d63b566eeb4d5b395fd2d00d29d7ef00a00582aac9"}, - {file = "kiwisolver-1.4.4-cp39-cp39-win32.whl", hash = "sha256:da7e547706e69e45d95e116e6939488d62174e033b763ab1496b4c29b76fabea"}, - {file = "kiwisolver-1.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:ba59c92039ec0a66103b1d5fe588fa546373587a7d68f5c96f743c3396afc04b"}, - {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:91672bacaa030f92fc2f43b620d7b337fd9a5af28b0d6ed3f77afc43c4a64b5a"}, - {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:787518a6789009c159453da4d6b683f468ef7a65bbde796bcea803ccf191058d"}, - {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da152d8cdcab0e56e4f45eb08b9aea6455845ec83172092f09b0e077ece2cf7a"}, - {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ecb1fa0db7bf4cff9dac752abb19505a233c7f16684c5826d1f11ebd9472b871"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:28bc5b299f48150b5f822ce68624e445040595a4ac3d59251703779836eceff9"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:81e38381b782cc7e1e46c4e14cd997ee6040768101aefc8fa3c24a4cc58e98f8"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2a66fdfb34e05b705620dd567f5a03f239a088d5a3f321e7b6ac3239d22aa286"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872b8ca05c40d309ed13eb2e582cab0c5a05e81e987ab9c521bf05ad1d5cf5cb"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:70e7c2e7b750585569564e2e5ca9845acfaa5da56ac46df68414f29fea97be9f"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9f85003f5dfa867e86d53fac6f7e6f30c045673fa27b603c397753bebadc3008"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e307eb9bd99801f82789b44bb45e9f541961831c7311521b13a6c85afc09767"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1792d939ec70abe76f5054d3f36ed5656021dcad1322d1cc996d4e54165cef9"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6cb459eea32a4e2cf18ba5fcece2dbdf496384413bc1bae15583f19e567f3b2"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36dafec3d6d6088d34e2de6b85f9d8e2324eb734162fba59d2ba9ed7a2043d5b"}, - {file = "kiwisolver-1.4.4.tar.gz", hash = "sha256:d41997519fcba4a1e46eb4a2fe31bc12f0ff957b2b81bac28db24744f333e955"}, -] -matplotlib = [ - {file = "matplotlib-3.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a206a1b762b39398efea838f528b3a6d60cdb26fe9d58b48265787e29cd1d693"}, - {file = "matplotlib-3.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cd45a6f3e93a780185f70f05cf2a383daed13c3489233faad83e81720f7ede24"}, - {file = "matplotlib-3.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d62880e1f60e5a30a2a8484432bcb3a5056969dc97258d7326ad465feb7ae069"}, - {file = "matplotlib-3.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ab29589cef03bc88acfa3a1490359000c18186fc30374d8aa77d33cc4a51a4a"}, - {file = "matplotlib-3.5.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2886cc009f40e2984c083687251821f305d811d38e3df8ded414265e4583f0c5"}, - {file = "matplotlib-3.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c995f7d9568f18b5db131ab124c64e51b6820a92d10246d4f2b3f3a66698a15b"}, - {file = "matplotlib-3.5.3-cp310-cp310-win32.whl", hash = "sha256:6bb93a0492d68461bd458eba878f52fdc8ac7bdb6c4acdfe43dba684787838c2"}, - {file = "matplotlib-3.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:2e6d184ebe291b9e8f7e78bbab7987d269c38ea3e062eace1fe7d898042ef804"}, - {file = "matplotlib-3.5.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6ea6aef5c4338e58d8d376068e28f80a24f54e69f09479d1c90b7172bad9f25b"}, - {file = "matplotlib-3.5.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:839d47b8ead7ad9669aaacdbc03f29656dc21f0d41a6fea2d473d856c39c8b1c"}, - {file = "matplotlib-3.5.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3b4fa56159dc3c7f9250df88f653f085068bcd32dcd38e479bba58909254af7f"}, - {file = "matplotlib-3.5.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:94ff86af56a3869a4ae26a9637a849effd7643858a1a04dd5ee50e9ab75069a7"}, - {file = "matplotlib-3.5.3-cp37-cp37m-win32.whl", hash = "sha256:35a8ad4dddebd51f94c5d24bec689ec0ec66173bf614374a1244c6241c1595e0"}, - {file = "matplotlib-3.5.3-cp37-cp37m-win_amd64.whl", hash = "sha256:43e9d3fa077bf0cc95ded13d331d2156f9973dce17c6f0c8b49ccd57af94dbd9"}, - {file = "matplotlib-3.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:22227c976ad4dc8c5a5057540421f0d8708c6560744ad2ad638d48e2984e1dbc"}, - {file = "matplotlib-3.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bf618a825deb6205f015df6dfe6167a5d9b351203b03fab82043ae1d30f16511"}, - {file = "matplotlib-3.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9befa5954cdbc085e37d974ff6053da269474177921dd61facdad8023c4aeb51"}, - {file = "matplotlib-3.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3840c280ebc87a48488a46f760ea1c0c0c83fcf7abbe2e6baf99d033fd35fd8"}, - {file = "matplotlib-3.5.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dacddf5bfcec60e3f26ec5c0ae3d0274853a258b6c3fc5ef2f06a8eb23e042be"}, - {file = "matplotlib-3.5.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b428076a55fb1c084c76cb93e68006f27d247169f056412607c5c88828d08f88"}, - {file = "matplotlib-3.5.3-cp38-cp38-win32.whl", hash = "sha256:874df7505ba820e0400e7091199decf3ff1fde0583652120c50cd60d5820ca9a"}, - {file = "matplotlib-3.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:b28de401d928890187c589036857a270a032961411934bdac4cf12dde3d43094"}, - {file = "matplotlib-3.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3211ba82b9f1518d346f6309df137b50c3dc4421b4ed4815d1d7eadc617f45a1"}, - {file = "matplotlib-3.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6fe807e8a22620b4cd95cfbc795ba310dc80151d43b037257250faf0bfcd82bc"}, - {file = "matplotlib-3.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5c096363b206a3caf43773abebdbb5a23ea13faef71d701b21a9c27fdcef72f4"}, - {file = "matplotlib-3.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bcdfcb0f976e1bac6721d7d457c17be23cf7501f977b6a38f9d38a3762841f7"}, - {file = "matplotlib-3.5.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1e64ac9be9da6bfff0a732e62116484b93b02a0b4d4b19934fb4f8e7ad26ad6a"}, - {file = "matplotlib-3.5.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:73dd93dc35c85dece610cca8358003bf0760d7986f70b223e2306b4ea6d1406b"}, - {file = "matplotlib-3.5.3-cp39-cp39-win32.whl", hash = "sha256:879c7e5fce4939c6aa04581dfe08d57eb6102a71f2e202e3314d5fbc072fd5a0"}, - {file = "matplotlib-3.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:ab8d26f07fe64f6f6736d635cce7bfd7f625320490ed5bfc347f2cdb4fae0e56"}, - {file = "matplotlib-3.5.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:99482b83ebf4eb6d5fc6813d7aacdefdd480f0d9c0b52dcf9f1cc3b2c4b3361a"}, - {file = "matplotlib-3.5.3-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f814504e459c68118bf2246a530ed953ebd18213dc20e3da524174d84ed010b2"}, - {file = "matplotlib-3.5.3-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57f1b4e69f438a99bb64d7f2c340db1b096b41ebaa515cf61ea72624279220ce"}, - {file = "matplotlib-3.5.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:d2484b350bf3d32cae43f85dcfc89b3ed7bd2bcd781ef351f93eb6fb2cc483f9"}, - {file = "matplotlib-3.5.3.tar.gz", hash = "sha256:339cac48b80ddbc8bfd05daae0a3a73414651a8596904c2a881cfd1edb65f26c"}, -] -msgpack = [ - {file = "msgpack-1.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4ab251d229d10498e9a2f3b1e68ef64cb393394ec477e3370c457f9430ce9250"}, - {file = "msgpack-1.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:112b0f93202d7c0fef0b7810d465fde23c746a2d482e1e2de2aafd2ce1492c88"}, - {file = "msgpack-1.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:002b5c72b6cd9b4bafd790f364b8480e859b4712e91f43014fe01e4f957b8467"}, - {file = "msgpack-1.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35bc0faa494b0f1d851fd29129b2575b2e26d41d177caacd4206d81502d4c6a6"}, - {file = "msgpack-1.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4733359808c56d5d7756628736061c432ded018e7a1dff2d35a02439043321aa"}, - {file = "msgpack-1.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb514ad14edf07a1dbe63761fd30f89ae79b42625731e1ccf5e1f1092950eaa6"}, - {file = "msgpack-1.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c23080fdeec4716aede32b4e0ef7e213c7b1093eede9ee010949f2a418ced6ba"}, - {file = "msgpack-1.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:49565b0e3d7896d9ea71d9095df15b7f75a035c49be733051c34762ca95bbf7e"}, - {file = "msgpack-1.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:aca0f1644d6b5a73eb3e74d4d64d5d8c6c3d577e753a04c9e9c87d07692c58db"}, - {file = "msgpack-1.0.4-cp310-cp310-win32.whl", hash = "sha256:0dfe3947db5fb9ce52aaea6ca28112a170db9eae75adf9339a1aec434dc954ef"}, - {file = "msgpack-1.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dea20515f660aa6b7e964433b1808d098dcfcabbebeaaad240d11f909298075"}, - {file = "msgpack-1.0.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e83f80a7fec1a62cf4e6c9a660e39c7f878f603737a0cdac8c13131d11d97f52"}, - {file = "msgpack-1.0.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c11a48cf5e59026ad7cb0dc29e29a01b5a66a3e333dc11c04f7e991fc5510a9"}, - {file = "msgpack-1.0.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1276e8f34e139aeff1c77a3cefb295598b504ac5314d32c8c3d54d24fadb94c9"}, - {file = "msgpack-1.0.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c9566f2c39ccced0a38d37c26cc3570983b97833c365a6044edef3574a00c08"}, - {file = "msgpack-1.0.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:fcb8a47f43acc113e24e910399376f7277cf8508b27e5b88499f053de6b115a8"}, - {file = "msgpack-1.0.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:76ee788122de3a68a02ed6f3a16bbcd97bc7c2e39bd4d94be2f1821e7c4a64e6"}, - {file = "msgpack-1.0.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:0a68d3ac0104e2d3510de90a1091720157c319ceeb90d74f7b5295a6bee51bae"}, - {file = "msgpack-1.0.4-cp36-cp36m-win32.whl", hash = "sha256:85f279d88d8e833ec015650fd15ae5eddce0791e1e8a59165318f371158efec6"}, - {file = "msgpack-1.0.4-cp36-cp36m-win_amd64.whl", hash = "sha256:c1683841cd4fa45ac427c18854c3ec3cd9b681694caf5bff04edb9387602d661"}, - {file = "msgpack-1.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a75dfb03f8b06f4ab093dafe3ddcc2d633259e6c3f74bb1b01996f5d8aa5868c"}, - {file = "msgpack-1.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9667bdfdf523c40d2511f0e98a6c9d3603be6b371ae9a238b7ef2dc4e7a427b0"}, - {file = "msgpack-1.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11184bc7e56fd74c00ead4f9cc9a3091d62ecb96e97653add7a879a14b003227"}, - {file = "msgpack-1.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac5bd7901487c4a1dd51a8c58f2632b15d838d07ceedaa5e4c080f7190925bff"}, - {file = "msgpack-1.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1e91d641d2bfe91ba4c52039adc5bccf27c335356055825c7f88742c8bb900dd"}, - {file = "msgpack-1.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2a2df1b55a78eb5f5b7d2a4bb221cd8363913830145fad05374a80bf0877cb1e"}, - {file = "msgpack-1.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:545e3cf0cf74f3e48b470f68ed19551ae6f9722814ea969305794645da091236"}, - {file = "msgpack-1.0.4-cp37-cp37m-win32.whl", hash = "sha256:2cc5ca2712ac0003bcb625c96368fd08a0f86bbc1a5578802512d87bc592fe44"}, - {file = "msgpack-1.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:eba96145051ccec0ec86611fe9cf693ce55f2a3ce89c06ed307de0e085730ec1"}, - {file = "msgpack-1.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7760f85956c415578c17edb39eed99f9181a48375b0d4a94076d84148cf67b2d"}, - {file = "msgpack-1.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:449e57cc1ff18d3b444eb554e44613cffcccb32805d16726a5494038c3b93dab"}, - {file = "msgpack-1.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d603de2b8d2ea3f3bcb2efe286849aa7a81531abc52d8454da12f46235092bcb"}, - {file = "msgpack-1.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f5d88c99f64c456413d74a975bd605a9b0526293218a3b77220a2c15458ba9"}, - {file = "msgpack-1.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6916c78f33602ecf0509cc40379271ba0f9ab572b066bd4bdafd7434dee4bc6e"}, - {file = "msgpack-1.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:81fc7ba725464651190b196f3cd848e8553d4d510114a954681fd0b9c479d7e1"}, - {file = "msgpack-1.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5b5b962221fa2c5d3a7f8133f9abffc114fe218eb4365e40f17732ade576c8e"}, - {file = "msgpack-1.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:77ccd2af37f3db0ea59fb280fa2165bf1b096510ba9fe0cc2bf8fa92a22fdb43"}, - {file = "msgpack-1.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b17be2478b622939e39b816e0aa8242611cc8d3583d1cd8ec31b249f04623243"}, - {file = "msgpack-1.0.4-cp38-cp38-win32.whl", hash = "sha256:2bb8cdf50dd623392fa75525cce44a65a12a00c98e1e37bf0fb08ddce2ff60d2"}, - {file = "msgpack-1.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:26b8feaca40a90cbe031b03d82b2898bf560027160d3eae1423f4a67654ec5d6"}, - {file = "msgpack-1.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:462497af5fd4e0edbb1559c352ad84f6c577ffbbb708566a0abaaa84acd9f3ae"}, - {file = "msgpack-1.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2999623886c5c02deefe156e8f869c3b0aaeba14bfc50aa2486a0415178fce55"}, - {file = "msgpack-1.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f0029245c51fd9473dc1aede1160b0a29f4a912e6b1dd353fa6d317085b219da"}, - {file = "msgpack-1.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed6f7b854a823ea44cf94919ba3f727e230da29feb4a99711433f25800cf747f"}, - {file = "msgpack-1.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0df96d6eaf45ceca04b3f3b4b111b86b33785683d682c655063ef8057d61fd92"}, - {file = "msgpack-1.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a4192b1ab40f8dca3f2877b70e63799d95c62c068c84dc028b40a6cb03ccd0f"}, - {file = "msgpack-1.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0e3590f9fb9f7fbc36df366267870e77269c03172d086fa76bb4eba8b2b46624"}, - {file = "msgpack-1.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1576bd97527a93c44fa856770197dec00d223b0b9f36ef03f65bac60197cedf8"}, - {file = "msgpack-1.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:63e29d6e8c9ca22b21846234913c3466b7e4ee6e422f205a2988083de3b08cae"}, - {file = "msgpack-1.0.4-cp39-cp39-win32.whl", hash = "sha256:fb62ea4b62bfcb0b380d5680f9a4b3f9a2d166d9394e9bbd9666c0ee09a3645c"}, - {file = "msgpack-1.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:4d5834a2a48965a349da1c5a79760d94a1a0172fbb5ab6b5b33cbf8447e109ce"}, - {file = "msgpack-1.0.4.tar.gz", hash = "sha256:f5d869c18f030202eb412f08b28d2afeea553d6613aee89e200d7aca7ef01f5f"}, -] -mypy = [ - {file = "mypy-0.991-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7d17e0a9707d0772f4a7b878f04b4fd11f6f5bcb9b3813975a9b13c9332153ab"}, - {file = "mypy-0.991-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0714258640194d75677e86c786e80ccf294972cc76885d3ebbb560f11db0003d"}, - {file = "mypy-0.991-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c8f3be99e8a8bd403caa8c03be619544bc2c77a7093685dcf308c6b109426c6"}, - {file = "mypy-0.991-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc9ec663ed6c8f15f4ae9d3c04c989b744436c16d26580eaa760ae9dd5d662eb"}, - {file = "mypy-0.991-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4307270436fd7694b41f913eb09210faff27ea4979ecbcd849e57d2da2f65305"}, - {file = "mypy-0.991-cp310-cp310-win_amd64.whl", hash = "sha256:901c2c269c616e6cb0998b33d4adbb4a6af0ac4ce5cd078afd7bc95830e62c1c"}, - {file = "mypy-0.991-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d13674f3fb73805ba0c45eb6c0c3053d218aa1f7abead6e446d474529aafc372"}, - {file = "mypy-0.991-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c8cd4fb70e8584ca1ed5805cbc7c017a3d1a29fb450621089ffed3e99d1857f"}, - {file = "mypy-0.991-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:209ee89fbb0deed518605edddd234af80506aec932ad28d73c08f1400ef80a33"}, - {file = "mypy-0.991-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37bd02ebf9d10e05b00d71302d2c2e6ca333e6c2a8584a98c00e038db8121f05"}, - {file = "mypy-0.991-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:26efb2fcc6b67e4d5a55561f39176821d2adf88f2745ddc72751b7890f3194ad"}, - {file = "mypy-0.991-cp311-cp311-win_amd64.whl", hash = "sha256:3a700330b567114b673cf8ee7388e949f843b356a73b5ab22dd7cff4742a5297"}, - {file = "mypy-0.991-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1f7d1a520373e2272b10796c3ff721ea1a0712288cafaa95931e66aa15798813"}, - {file = "mypy-0.991-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:641411733b127c3e0dab94c45af15fea99e4468f99ac88b39efb1ad677da5711"}, - {file = "mypy-0.991-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3d80e36b7d7a9259b740be6d8d906221789b0d836201af4234093cae89ced0cd"}, - {file = "mypy-0.991-cp37-cp37m-win_amd64.whl", hash = "sha256:e62ebaad93be3ad1a828a11e90f0e76f15449371ffeecca4a0a0b9adc99abcef"}, - {file = "mypy-0.991-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b86ce2c1866a748c0f6faca5232059f881cda6dda2a893b9a8373353cfe3715a"}, - {file = "mypy-0.991-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac6e503823143464538efda0e8e356d871557ef60ccd38f8824a4257acc18d93"}, - {file = "mypy-0.991-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0cca5adf694af539aeaa6ac633a7afe9bbd760df9d31be55ab780b77ab5ae8bf"}, - {file = "mypy-0.991-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a12c56bf73cdab116df96e4ff39610b92a348cc99a1307e1da3c3768bbb5b135"}, - {file = "mypy-0.991-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:652b651d42f155033a1967739788c436491b577b6a44e4c39fb340d0ee7f0d70"}, - {file = "mypy-0.991-cp38-cp38-win_amd64.whl", hash = "sha256:4175593dc25d9da12f7de8de873a33f9b2b8bdb4e827a7cae952e5b1a342e243"}, - {file = "mypy-0.991-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:98e781cd35c0acf33eb0295e8b9c55cdbef64fcb35f6d3aa2186f289bed6e80d"}, - {file = "mypy-0.991-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6d7464bac72a85cb3491c7e92b5b62f3dcccb8af26826257760a552a5e244aa5"}, - {file = "mypy-0.991-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c9166b3f81a10cdf9b49f2d594b21b31adadb3d5e9db9b834866c3258b695be3"}, - {file = "mypy-0.991-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8472f736a5bfb159a5e36740847808f6f5b659960115ff29c7cecec1741c648"}, - {file = "mypy-0.991-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e80e758243b97b618cdf22004beb09e8a2de1af481382e4d84bc52152d1c476"}, - {file = "mypy-0.991-cp39-cp39-win_amd64.whl", hash = "sha256:74e259b5c19f70d35fcc1ad3d56499065c601dfe94ff67ae48b85596b9ec1461"}, - {file = "mypy-0.991-py3-none-any.whl", hash = "sha256:de32edc9b0a7e67c2775e574cb061a537660e51210fbf6006b0b36ea695ae9bb"}, - {file = "mypy-0.991.tar.gz", hash = "sha256:3c0165ba8f354a6d9881809ef29f1a9318a236a6d81c690094c5df32107bde06"}, -] -mypy-extensions = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, -] -nodeenv = [ - {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, - {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, -] -numpy = [ - {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e"}, - {file = "numpy-1.21.1-cp37-cp37m-win32.whl", hash = "sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172"}, - {file = "numpy-1.21.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8"}, - {file = "numpy-1.21.1-cp38-cp38-win32.whl", hash = "sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd"}, - {file = "numpy-1.21.1-cp38-cp38-win_amd64.whl", hash = "sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a"}, - {file = "numpy-1.21.1-cp39-cp39-win32.whl", hash = "sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2"}, - {file = "numpy-1.21.1-cp39-cp39-win_amd64.whl", hash = "sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33"}, - {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"}, - {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, -] -nvidia-cublas-cu11 = [ - {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl", hash = "sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded"}, - {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-win_amd64.whl", hash = "sha256:8ac17ba6ade3ed56ab898a036f9ae0756f1e81052a317bf98f8c6d18dc3ae49e"}, -] -nvidia-cuda-nvrtc-cu11 = [ - {file = "nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:9f1562822ea264b7e34ed5930567e89242d266448e936b85bc97a3370feabb03"}, - {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:f7d9610d9b7c331fa0da2d1b2858a4a8315e6d49765091d28711c8946e7425e7"}, - {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:f2effeb1309bdd1b3854fc9b17eaf997808f8b25968ce0c7070945c4265d64a3"}, -] -nvidia-cuda-runtime-cu11 = [ - {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:cc768314ae58d2641f07eac350f40f99dcb35719c4faff4bc458a7cd2b119e31"}, - {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:bc77fa59a7679310df9d5c70ab13c4e34c64ae2124dd1efd7e5474b71be125c7"}, -] -nvidia-cudnn-cu11 = [ - {file = "nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:402f40adfc6f418f9dae9ab402e773cfed9beae52333f6d86ae3107a1b9527e7"}, - {file = "nvidia_cudnn_cu11-8.5.0.96-py3-none-manylinux1_x86_64.whl", hash = "sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108"}, -] -omegaconf = [ - {file = "omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b"}, - {file = "omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7"}, -] -opt-einsum = [ - {file = "opt_einsum-3.3.0-py3-none-any.whl", hash = "sha256:2455e59e3947d3c275477df7f5205b30635e266fe6dc300e3d9f9646bfcea147"}, - {file = "opt_einsum-3.3.0.tar.gz", hash = "sha256:59f6475f77bbc37dcf7cd748519c0ec60722e91e63ca114e68821c0c54a46549"}, -] -optax = [ - {file = "optax-0.1.4-py3-none-any.whl", hash = "sha256:12fcf33bd682f9a162a3deb097f864130c3224d76771af2ba09410de80399a9b"}, - {file = "optax-0.1.4.tar.gz", hash = "sha256:fb7a0550d57a6636164a3de25986a8a19be8ff6431fcdf1225b4e05175810f22"}, -] -orbax = [ - {file = "orbax-0.0.23-py3-none-any.whl", hash = "sha256:870d31d7e6b2aebb502bd6f90670f3f0e53ca4e4563773d6631629920c884744"}, - {file = "orbax-0.0.23.tar.gz", hash = "sha256:2d009e579b38a5a94f04bafc01f6d71b0220fc6664c3622bd951f2874e6c0cf1"}, -] -packaging = [ - {file = "packaging-22.0-py3-none-any.whl", hash = "sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"}, - {file = "packaging-22.0.tar.gz", hash = "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3"}, -] -Pillow = [ - {file = "Pillow-9.3.0-1-cp37-cp37m-win32.whl", hash = "sha256:e6ea6b856a74d560d9326c0f5895ef8050126acfdc7ca08ad703eb0081e82b74"}, - {file = "Pillow-9.3.0-1-cp37-cp37m-win_amd64.whl", hash = "sha256:32a44128c4bdca7f31de5be641187367fe2a450ad83b833ef78910397db491aa"}, - {file = "Pillow-9.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:0b7257127d646ff8676ec8a15520013a698d1fdc48bc2a79ba4e53df792526f2"}, - {file = "Pillow-9.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b90f7616ea170e92820775ed47e136208e04c967271c9ef615b6fbd08d9af0e3"}, - {file = "Pillow-9.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68943d632f1f9e3dce98908e873b3a090f6cba1cbb1b892a9e8d97c938871fbe"}, - {file = "Pillow-9.3.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be55f8457cd1eac957af0c3f5ece7bc3f033f89b114ef30f710882717670b2a8"}, - {file = "Pillow-9.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d77adcd56a42d00cc1be30843d3426aa4e660cab4a61021dc84467123f7a00c"}, - {file = "Pillow-9.3.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:829f97c8e258593b9daa80638aee3789b7df9da5cf1336035016d76f03b8860c"}, - {file = "Pillow-9.3.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:801ec82e4188e935c7f5e22e006d01611d6b41661bba9fe45b60e7ac1a8f84de"}, - {file = "Pillow-9.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:871b72c3643e516db4ecf20efe735deb27fe30ca17800e661d769faab45a18d7"}, - {file = "Pillow-9.3.0-cp310-cp310-win32.whl", hash = "sha256:655a83b0058ba47c7c52e4e2df5ecf484c1b0b0349805896dd350cbc416bdd91"}, - {file = "Pillow-9.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:9f47eabcd2ded7698106b05c2c338672d16a6f2a485e74481f524e2a23c2794b"}, - {file = "Pillow-9.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:57751894f6618fd4308ed8e0c36c333e2f5469744c34729a27532b3db106ee20"}, - {file = "Pillow-9.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7db8b751ad307d7cf238f02101e8e36a128a6cb199326e867d1398067381bff4"}, - {file = "Pillow-9.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3033fbe1feb1b59394615a1cafaee85e49d01b51d54de0cbf6aa8e64182518a1"}, - {file = "Pillow-9.3.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22b012ea2d065fd163ca096f4e37e47cd8b59cf4b0fd47bfca6abb93df70b34c"}, - {file = "Pillow-9.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9a65733d103311331875c1dca05cb4606997fd33d6acfed695b1232ba1df193"}, - {file = "Pillow-9.3.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:502526a2cbfa431d9fc2a079bdd9061a2397b842bb6bc4239bb176da00993812"}, - {file = "Pillow-9.3.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:90fb88843d3902fe7c9586d439d1e8c05258f41da473952aa8b328d8b907498c"}, - {file = "Pillow-9.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:89dca0ce00a2b49024df6325925555d406b14aa3efc2f752dbb5940c52c56b11"}, - {file = "Pillow-9.3.0-cp311-cp311-win32.whl", hash = "sha256:3168434d303babf495d4ba58fc22d6604f6e2afb97adc6a423e917dab828939c"}, - {file = "Pillow-9.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:18498994b29e1cf86d505edcb7edbe814d133d2232d256db8c7a8ceb34d18cef"}, - {file = "Pillow-9.3.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:772a91fc0e03eaf922c63badeca75e91baa80fe2f5f87bdaed4280662aad25c9"}, - {file = "Pillow-9.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa4107d1b306cdf8953edde0534562607fe8811b6c4d9a486298ad31de733b2"}, - {file = "Pillow-9.3.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b4012d06c846dc2b80651b120e2cdd787b013deb39c09f407727ba90015c684f"}, - {file = "Pillow-9.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77ec3e7be99629898c9a6d24a09de089fa5356ee408cdffffe62d67bb75fdd72"}, - {file = "Pillow-9.3.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:6c738585d7a9961d8c2821a1eb3dcb978d14e238be3d70f0a706f7fa9316946b"}, - {file = "Pillow-9.3.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:828989c45c245518065a110434246c44a56a8b2b2f6347d1409c787e6e4651ee"}, - {file = "Pillow-9.3.0-cp37-cp37m-win32.whl", hash = "sha256:82409ffe29d70fd733ff3c1025a602abb3e67405d41b9403b00b01debc4c9a29"}, - {file = "Pillow-9.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:41e0051336807468be450d52b8edd12ac60bebaa97fe10c8b660f116e50b30e4"}, - {file = "Pillow-9.3.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:b03ae6f1a1878233ac620c98f3459f79fd77c7e3c2b20d460284e1fb370557d4"}, - {file = "Pillow-9.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4390e9ce199fc1951fcfa65795f239a8a4944117b5935a9317fb320e7767b40f"}, - {file = "Pillow-9.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40e1ce476a7804b0fb74bcfa80b0a2206ea6a882938eaba917f7a0f004b42502"}, - {file = "Pillow-9.3.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0a06a052c5f37b4ed81c613a455a81f9a3a69429b4fd7bb913c3fa98abefc20"}, - {file = "Pillow-9.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03150abd92771742d4a8cd6f2fa6246d847dcd2e332a18d0c15cc75bf6703040"}, - {file = "Pillow-9.3.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:15c42fb9dea42465dfd902fb0ecf584b8848ceb28b41ee2b58f866411be33f07"}, - {file = "Pillow-9.3.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:51e0e543a33ed92db9f5ef69a0356e0b1a7a6b6a71b80df99f1d181ae5875636"}, - {file = "Pillow-9.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3dd6caf940756101205dffc5367babf288a30043d35f80936f9bfb37f8355b32"}, - {file = "Pillow-9.3.0-cp38-cp38-win32.whl", hash = "sha256:f1ff2ee69f10f13a9596480335f406dd1f70c3650349e2be67ca3139280cade0"}, - {file = "Pillow-9.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:276a5ca930c913f714e372b2591a22c4bd3b81a418c0f6635ba832daec1cbcfc"}, - {file = "Pillow-9.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:73bd195e43f3fadecfc50c682f5055ec32ee2c933243cafbfdec69ab1aa87cad"}, - {file = "Pillow-9.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1c7c8ae3864846fc95f4611c78129301e203aaa2af813b703c55d10cc1628535"}, - {file = "Pillow-9.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e0918e03aa0c72ea56edbb00d4d664294815aa11291a11504a377ea018330d3"}, - {file = "Pillow-9.3.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0915e734b33a474d76c28e07292f196cdf2a590a0d25bcc06e64e545f2d146c"}, - {file = "Pillow-9.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af0372acb5d3598f36ec0914deed2a63f6bcdb7b606da04dc19a88d31bf0c05b"}, - {file = "Pillow-9.3.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:ad58d27a5b0262c0c19b47d54c5802db9b34d38bbf886665b626aff83c74bacd"}, - {file = "Pillow-9.3.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:97aabc5c50312afa5e0a2b07c17d4ac5e865b250986f8afe2b02d772567a380c"}, - {file = "Pillow-9.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9aaa107275d8527e9d6e7670b64aabaaa36e5b6bd71a1015ddd21da0d4e06448"}, - {file = "Pillow-9.3.0-cp39-cp39-win32.whl", hash = "sha256:bac18ab8d2d1e6b4ce25e3424f709aceef668347db8637c2296bcf41acb7cf48"}, - {file = "Pillow-9.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:b472b5ea442148d1c3e2209f20f1e0bb0eb556538690fa70b5e1f79fa0ba8dc2"}, - {file = "Pillow-9.3.0-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:ab388aaa3f6ce52ac1cb8e122c4bd46657c15905904b3120a6248b5b8b0bc228"}, - {file = "Pillow-9.3.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbb8e7f2abee51cef77673be97760abff1674ed32847ce04b4af90f610144c7b"}, - {file = "Pillow-9.3.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca31dd6014cb8b0b2db1e46081b0ca7d936f856da3b39744aef499db5d84d02"}, - {file = "Pillow-9.3.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c7025dce65566eb6e89f56c9509d4f628fddcedb131d9465cacd3d8bac337e7e"}, - {file = "Pillow-9.3.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ebf2029c1f464c59b8bdbe5143c79fa2045a581ac53679733d3a91d400ff9efb"}, - {file = "Pillow-9.3.0-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:b59430236b8e58840a0dfb4099a0e8717ffb779c952426a69ae435ca1f57210c"}, - {file = "Pillow-9.3.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12ce4932caf2ddf3e41d17fc9c02d67126935a44b86df6a206cf0d7161548627"}, - {file = "Pillow-9.3.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae5331c23ce118c53b172fa64a4c037eb83c9165aba3a7ba9ddd3ec9fa64a699"}, - {file = "Pillow-9.3.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:0b07fffc13f474264c336298d1b4ce01d9c5a011415b79d4ee5527bb69ae6f65"}, - {file = "Pillow-9.3.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:073adb2ae23431d3b9bcbcff3fe698b62ed47211d0716b067385538a1b0f28b8"}, - {file = "Pillow-9.3.0.tar.gz", hash = "sha256:c935a22a557a560108d780f9a0fc426dd7459940dc54faa49d83249c8d3e760f"}, -] -pluggy = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] -pydantic = [ - {file = "pydantic-1.10.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb6ad4489af1bac6955d38ebcb95079a836af31e4c4f74aba1ca05bb9f6027bd"}, - {file = "pydantic-1.10.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a1f5a63a6dfe19d719b1b6e6106561869d2efaca6167f84f5ab9347887d78b98"}, - {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352aedb1d71b8b0736c6d56ad2bd34c6982720644b0624462059ab29bd6e5912"}, - {file = "pydantic-1.10.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19b3b9ccf97af2b7519c42032441a891a5e05c68368f40865a90eb88833c2559"}, - {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e9069e1b01525a96e6ff49e25876d90d5a563bc31c658289a8772ae186552236"}, - {file = "pydantic-1.10.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:355639d9afc76bcb9b0c3000ddcd08472ae75318a6eb67a15866b87e2efa168c"}, - {file = "pydantic-1.10.2-cp310-cp310-win_amd64.whl", hash = "sha256:ae544c47bec47a86bc7d350f965d8b15540e27e5aa4f55170ac6a75e5f73b644"}, - {file = "pydantic-1.10.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4c805731c33a8db4b6ace45ce440c4ef5336e712508b4d9e1aafa617dc9907f"}, - {file = "pydantic-1.10.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d49f3db871575e0426b12e2f32fdb25e579dea16486a26e5a0474af87cb1ab0a"}, - {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37c90345ec7dd2f1bcef82ce49b6235b40f282b94d3eec47e801baf864d15525"}, - {file = "pydantic-1.10.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b5ba54d026c2bd2cb769d3468885f23f43710f651688e91f5fb1edcf0ee9283"}, - {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05e00dbebbe810b33c7a7362f231893183bcc4251f3f2ff991c31d5c08240c42"}, - {file = "pydantic-1.10.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2d0567e60eb01bccda3a4df01df677adf6b437958d35c12a3ac3e0f078b0ee52"}, - {file = "pydantic-1.10.2-cp311-cp311-win_amd64.whl", hash = "sha256:c6f981882aea41e021f72779ce2a4e87267458cc4d39ea990729e21ef18f0f8c"}, - {file = "pydantic-1.10.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4aac8e7103bf598373208f6299fa9a5cfd1fc571f2d40bf1dd1955a63d6eeb5"}, - {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a7b66c3f499108b448f3f004801fcd7d7165fb4200acb03f1c2402da73ce4c"}, - {file = "pydantic-1.10.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bedf309630209e78582ffacda64a21f96f3ed2e51fbf3962d4d488e503420254"}, - {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9300fcbebf85f6339a02c6994b2eb3ff1b9c8c14f502058b5bf349d42447dcf5"}, - {file = "pydantic-1.10.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:216f3bcbf19c726b1cc22b099dd409aa371f55c08800bcea4c44c8f74b73478d"}, - {file = "pydantic-1.10.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dd3f9a40c16daf323cf913593083698caee97df2804aa36c4b3175d5ac1b92a2"}, - {file = "pydantic-1.10.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b97890e56a694486f772d36efd2ba31612739bc6f3caeee50e9e7e3ebd2fdd13"}, - {file = "pydantic-1.10.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9cabf4a7f05a776e7793e72793cd92cc865ea0e83a819f9ae4ecccb1b8aa6116"}, - {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06094d18dd5e6f2bbf93efa54991c3240964bb663b87729ac340eb5014310624"}, - {file = "pydantic-1.10.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc78cc83110d2f275ec1970e7a831f4e371ee92405332ebfe9860a715f8336e1"}, - {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ee433e274268a4b0c8fde7ad9d58ecba12b069a033ecc4645bb6303c062d2e9"}, - {file = "pydantic-1.10.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c2abc4393dea97a4ccbb4ec7d8658d4e22c4765b7b9b9445588f16c71ad9965"}, - {file = "pydantic-1.10.2-cp38-cp38-win_amd64.whl", hash = "sha256:0b959f4d8211fc964772b595ebb25f7652da3f22322c007b6fed26846a40685e"}, - {file = "pydantic-1.10.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c33602f93bfb67779f9c507e4d69451664524389546bacfe1bee13cae6dc7488"}, - {file = "pydantic-1.10.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5760e164b807a48a8f25f8aa1a6d857e6ce62e7ec83ea5d5c5a802eac81bad41"}, - {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eb843dcc411b6a2237a694f5e1d649fc66c6064d02b204a7e9d194dff81eb4b"}, - {file = "pydantic-1.10.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b8795290deaae348c4eba0cebb196e1c6b98bdbe7f50b2d0d9a4a99716342fe"}, - {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e0bedafe4bc165ad0a56ac0bd7695df25c50f76961da29c050712596cf092d6d"}, - {file = "pydantic-1.10.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e05aed07fa02231dbf03d0adb1be1d79cabb09025dd45aa094aa8b4e7b9dcda"}, - {file = "pydantic-1.10.2-cp39-cp39-win_amd64.whl", hash = "sha256:c1ba1afb396148bbc70e9eaa8c06c1716fdddabaf86e7027c5988bae2a829ab6"}, - {file = "pydantic-1.10.2-py3-none-any.whl", hash = "sha256:1b6ee725bd6e83ec78b1aa32c5b1fa67a3a65badddde3976bca5fe4568f27709"}, - {file = "pydantic-1.10.2.tar.gz", hash = "sha256:91b8e218852ef6007c2b98cd861601c6a09f1aa32bbbb74fab5b1c33d4a1e410"}, -] -Pygments = [ - {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, - {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, -] -pyparsing = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] -pyright = [ - {file = "pyright-1.1.285-py3-none-any.whl", hash = "sha256:8a6b60b3ff0d000c549621c367cdf0013abdaf24d09e6f0b4b95031b357cc4b1"}, - {file = "pyright-1.1.285.tar.gz", hash = "sha256:ecd28e8556352e2c7eb5f412c6841ec768d25e8a6136326d4a6a67d94370eba1"}, -] -pytest = [ - {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, - {file = "pytest-7.2.0.tar.gz", hash = "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59"}, -] -pytest-cov = [ - {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, - {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, -] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] -PyYAML = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] -rich = [ - {file = "rich-12.6.0-py3-none-any.whl", hash = "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e"}, - {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, -] -scipy = [ - {file = "scipy-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a15a1f3fc0abff33e792d6049161b7795909b40b97c6cc2934ed54384017ab76"}, - {file = "scipy-1.6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:e79570979ccdc3d165456dd62041d9556fb9733b86b4b6d818af7a0afc15f092"}, - {file = "scipy-1.6.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a423533c55fec61456dedee7b6ee7dce0bb6bfa395424ea374d25afa262be261"}, - {file = "scipy-1.6.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:33d6b7df40d197bdd3049d64e8e680227151673465e5d85723b3b8f6b15a6ced"}, - {file = "scipy-1.6.1-cp37-cp37m-win32.whl", hash = "sha256:6725e3fbb47da428794f243864f2297462e9ee448297c93ed1dcbc44335feb78"}, - {file = "scipy-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:5fa9c6530b1661f1370bcd332a1e62ca7881785cc0f80c0d559b636567fab63c"}, - {file = "scipy-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bd50daf727f7c195e26f27467c85ce653d41df4358a25b32434a50d8870fc519"}, - {file = "scipy-1.6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f46dd15335e8a320b0fb4685f58b7471702234cba8bb3442b69a3e1dc329c345"}, - {file = "scipy-1.6.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0e5b0ccf63155d90da576edd2768b66fb276446c371b73841e3503be1d63fb5d"}, - {file = "scipy-1.6.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2481efbb3740977e3c831edfd0bd9867be26387cacf24eb5e366a6a374d3d00d"}, - {file = "scipy-1.6.1-cp38-cp38-win32.whl", hash = "sha256:68cb4c424112cd4be886b4d979c5497fba190714085f46b8ae67a5e4416c32b4"}, - {file = "scipy-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:5f331eeed0297232d2e6eea51b54e8278ed8bb10b099f69c44e2558c090d06bf"}, - {file = "scipy-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0c8a51d33556bf70367452d4d601d1742c0e806cd0194785914daf19775f0e67"}, - {file = "scipy-1.6.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:83bf7c16245c15bc58ee76c5418e46ea1811edcc2e2b03041b804e46084ab627"}, - {file = "scipy-1.6.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:794e768cc5f779736593046c9714e0f3a5940bc6dcc1dba885ad64cbfb28e9f0"}, - {file = "scipy-1.6.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5da5471aed911fe7e52b86bf9ea32fb55ae93e2f0fac66c32e58897cfb02fa07"}, - {file = "scipy-1.6.1-cp39-cp39-win32.whl", hash = "sha256:8e403a337749ed40af60e537cc4d4c03febddcc56cd26e774c9b1b600a70d3e4"}, - {file = "scipy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a5193a098ae9f29af283dcf0041f762601faf2e595c0db1da929875b7570353f"}, - {file = "scipy-1.6.1.tar.gz", hash = "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11"}, -] -setuptools = [ - {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, - {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, -] -setuptools-scm = [ - {file = "setuptools_scm-6.4.2-py3-none-any.whl", hash = "sha256:acea13255093849de7ccb11af9e1fb8bde7067783450cee9ef7a93139bddf6d4"}, - {file = "setuptools_scm-6.4.2.tar.gz", hash = "sha256:6833ac65c6ed9711a4d5d2266f8024cfa07c533a0e55f4c12f6eff280a5a9e30"}, -] -shtab = [ - {file = "shtab-1.5.8-py2.py3-none-any.whl", hash = "sha256:1d326ea131edbba96e0489470a2c969da1976a586a2d9376bb4923a6fe8eaae0"}, - {file = "shtab-1.5.8.tar.gz", hash = "sha256:1f944e2e33c1554be69e6b26ef638ba3b516ac9449fdd2a40d197f9061c8bed8"}, -] -six = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] -tensorstore = [ - {file = "tensorstore-0.1.28-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:deb32f1e74ab7b836ecec02a759a558fc57522a8dea0db4f19a01a78e93abab5"}, - {file = "tensorstore-0.1.28-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bd8615308ad3e29111238dfdd3aca1f3fb79b55c3dac1c04c96c8a319985addf"}, - {file = "tensorstore-0.1.28-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee16297a9493ed0d6d1000f216e79d29a0b80e7e97646e8520afd44138165839"}, - {file = "tensorstore-0.1.28-cp310-cp310-win_amd64.whl", hash = "sha256:b6f7e0c7455d9e164e6143714519e7c96cc4166b61ed7d268a8ab84a77e0d593"}, - {file = "tensorstore-0.1.28-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:1c53e1e0e7996b8ba41854135d748c2bc8802b94a79122b56448aeb1f77efd9f"}, - {file = "tensorstore-0.1.28-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bd122a44db81369808c84b7739ad2d8453f89c548c5d61d62f2b1f3331dbe9f"}, - {file = "tensorstore-0.1.28-cp37-cp37m-win_amd64.whl", hash = "sha256:197159da3ee97c08d8769fa151ff167328108bf07a13886f688b21f64abfb7d6"}, - {file = "tensorstore-0.1.28-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:e480da4f95f7c95302c4b3e2a7fc32b8cfe1aa51742d72f84a335e49462a2465"}, - {file = "tensorstore-0.1.28-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7948ead0d5a55303ffa97a35659aa56a20c60bd89d9487643f51ac8d2e19b52c"}, - {file = "tensorstore-0.1.28-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:818aeb015b249069a686fb20f0c0b71663410ac6411cee88df18b8e05b567cbc"}, - {file = "tensorstore-0.1.28-cp38-cp38-win_amd64.whl", hash = "sha256:6719eaba4ec2692d890c3c67c389fcae073d40d9e8c27e2dabd27fc9fd745e5c"}, - {file = "tensorstore-0.1.28-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:d7c86f63a6d4d7a84e3201ba9746ca51015612d375d77b9302eb93e9b88bc7aa"}, - {file = "tensorstore-0.1.28-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dc82cbb77255fd2f7aa191f1e030f13276bf7272ea1027fcc6900537b30bfbcb"}, - {file = "tensorstore-0.1.28-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4de09e3fae30f8a9ab7648600c78e109ddd1d14af4280e055439fdd52a6dca9"}, - {file = "tensorstore-0.1.28-cp39-cp39-win_amd64.whl", hash = "sha256:f9270586401ee60ff79a4cd54c62ab5b06831d69b561df1fb9ebbeade4d4929c"}, - {file = "tensorstore-0.1.28.tar.gz", hash = "sha256:cd8d8185136632c58edcd7cf4c43d301bf8ad61a197c632cb495d7d33b7be04e"}, -] -tomli = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] -toolz = [ - {file = "toolz-0.12.0-py3-none-any.whl", hash = "sha256:2059bd4148deb1884bb0eb770a3cde70e7f954cfbbdc2285f1f2de01fd21eb6f"}, - {file = "toolz-0.12.0.tar.gz", hash = "sha256:88c570861c440ee3f2f6037c4654613228ff40c93a6c25e0eba70d17282c6194"}, -] -torch = [ - {file = "torch-1.13.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:fd12043868a34a8da7d490bf6db66991108b00ffbeecb034228bfcbbd4197143"}, - {file = "torch-1.13.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:d9fe785d375f2e26a5d5eba5de91f89e6a3be5d11efb497e76705fdf93fa3c2e"}, - {file = "torch-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:98124598cdff4c287dbf50f53fb455f0c1e3a88022b39648102957f3445e9b76"}, - {file = "torch-1.13.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:393a6273c832e047581063fb74335ff50b4c566217019cc6ace318cd79eb0566"}, - {file = "torch-1.13.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:0122806b111b949d21fa1a5f9764d1fd2fcc4a47cb7f8ff914204fd4fc752ed5"}, - {file = "torch-1.13.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:22128502fd8f5b25ac1cd849ecb64a418382ae81dd4ce2b5cebaa09ab15b0d9b"}, - {file = "torch-1.13.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:76024be052b659ac1304ab8475ab03ea0a12124c3e7626282c9c86798ac7bc11"}, - {file = "torch-1.13.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:ea8dda84d796094eb8709df0fcd6b56dc20b58fdd6bc4e8d7109930dafc8e419"}, - {file = "torch-1.13.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2ee7b81e9c457252bddd7d3da66fb1f619a5d12c24d7074de91c4ddafb832c93"}, - {file = "torch-1.13.1-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:0d9b8061048cfb78e675b9d2ea8503bfe30db43d583599ae8626b1263a0c1380"}, - {file = "torch-1.13.1-cp37-none-macosx_11_0_arm64.whl", hash = "sha256:f402ca80b66e9fbd661ed4287d7553f7f3899d9ab54bf5c67faada1555abde28"}, - {file = "torch-1.13.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:727dbf00e2cf858052364c0e2a496684b9cb5aa01dc8a8bc8bbb7c54502bdcdd"}, - {file = "torch-1.13.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:df8434b0695e9ceb8cc70650afc1310d8ba949e6db2a0525ddd9c3b2b181e5fe"}, - {file = "torch-1.13.1-cp38-cp38-win_amd64.whl", hash = "sha256:5e1e722a41f52a3f26f0c4fcec227e02c6c42f7c094f32e49d4beef7d1e213ea"}, - {file = "torch-1.13.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:33e67eea526e0bbb9151263e65417a9ef2d8fa53cbe628e87310060c9dcfa312"}, - {file = "torch-1.13.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:eeeb204d30fd40af6a2d80879b46a7efbe3cf43cdbeb8838dd4f3d126cc90b2b"}, - {file = "torch-1.13.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:50ff5e76d70074f6653d191fe4f6a42fdbe0cf942fbe2a3af0b75eaa414ac038"}, - {file = "torch-1.13.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:2c3581a3fd81eb1f0f22997cddffea569fea53bafa372b2c0471db373b26aafc"}, - {file = "torch-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:0aa46f0ac95050c604bcf9ef71da9f1172e5037fdf2ebe051962d47b123848e7"}, - {file = "torch-1.13.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:6930791efa8757cb6974af73d4996b6b50c592882a324b8fb0589c6a9ba2ddaf"}, - {file = "torch-1.13.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:e0df902a7c7dd6c795698532ee5970ce898672625635d885eade9976e5a04949"}, -] -typed-ast = [ +files = [ {file = "typed_ast-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4"}, {file = "typed_ast-1.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62"}, {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac"}, @@ -1526,15 +1482,48 @@ typed-ast = [ {file = "typed_ast-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1"}, {file = "typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"}, ] -typing-extensions = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, + +[[package]] +name = "typing-extensions" +version = "4.6.2" +description = "Backported and Experimental Type Hints for Python 3.7+" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.6.2-py3-none-any.whl", hash = "sha256:3a8b36f13dd5fdc5d1b16fe317f5668545de77fa0b8e02006381fd49d731ab98"}, + {file = "typing_extensions-4.6.2.tar.gz", hash = "sha256:06006244c70ac8ee83fa8282cb188f697b8db25bc8b4df07be1873c43897060c"}, ] -wheel = [ - {file = "wheel-0.38.4-py3-none-any.whl", hash = "sha256:b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8"}, - {file = "wheel-0.38.4.tar.gz", hash = "sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac"}, + +[[package]] +name = "wheel" +version = "0.40.0" +description = "A built-package format for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "wheel-0.40.0-py3-none-any.whl", hash = "sha256:d236b20e7cb522daf2390fa84c55eea81c5c30190f90f29ae2ca1ad8355bf247"}, + {file = "wheel-0.40.0.tar.gz", hash = "sha256:cd1196f3faee2b31968d626e1731c94f99cbdb67cf5a46e4f5656cbee7738873"}, ] -zipp = [ - {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, - {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, + +[package.extras] +test = ["pytest (>=6.0.0)"] + +[[package]] +name = "zipp" +version = "3.15.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.7" +files = [ + {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, + {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, ] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.7" +content-hash = "48a6bbab3cc4aaac5dea9aaf813de0b442ba24f7d81261dc416193bb9fb33453" diff --git a/pyproject.toml b/pyproject.toml index 9dbdcd04a..dbc7a6965 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tyro" -version = "0.5.2" +version = "0.5.3" description = "Strongly typed, zero-effort CLI interfaces" authors = ["brentyi "] include = ["./tyro/**/*"] diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 65989f237..c24dc0e9b 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -62,6 +62,13 @@ get_type_hints, ) +# There are cases where typing.Literal doesn't match typing_extensions.Literal: +# https://github.com/python/typing_extensions/pull/148 +try: + from typing import Literal as LiteralAlternate +except ImportError: + LiteralAlternate = Literal # type: ignore + from . import _strings from ._typing import TypeForm from .conf import _markers @@ -325,7 +332,7 @@ def _instantiator_from_container_type( _instantiator_from_tuple: (tuple,), _instantiator_from_dict: (dict, collections.abc.Mapping), _instantiator_from_union: (Union,), - _instantiator_from_literal: (Literal,), + _instantiator_from_literal: (Literal, LiteralAlternate), }.items(): if type_origin in matched_origins: return make(typ, type_from_typevar, markers) From ad0d4c9144e4fe98f011bcf7ff49353eb24eb3af Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 4 Jun 2023 18:29:54 -0700 Subject: [PATCH 295/491] Update in-the-wild examples, tweak docs --- README.md | 11 ++++++++--- docs/source/examples/02_nesting/01_nesting.rst | 2 -- .../examples/04_additional/02_dictionaries.rst | 2 +- docs/source/index.md | 15 --------------- examples/02_nesting/01_nesting.py | 2 -- tyro/__init__.py | 7 ++++--- 6 files changed, 13 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 5962a2d5b..05e1f6e0c 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,11 @@ See [documentation](https://brentyi.github.io/tyro) for examples. distributed training of large language models in JAX. - [kevinzakka/obj2mjcf](https://github.com/kevinzakka/obj2mjcf) is an interface for processing composite Wavefront OBJ files for Mujoco. -- [brentyi/tensorf-jax](https://github.com/brentyi/tensorf-jax/) is an - unofficial implementation of - [Tensorial Radiance Fields](https://apchenstu.github.io/TensoRF/) in JAX. +- [blurgyy/jaxngp](https://github.com/blurgyy/jaxngp) is a CUDA-accelerated + implementation of [instant-ngp](https://nvlabs.github.io/instant-ngp/), + implemented in JAX. +- [NVIDIAGameWorks/kaolin-wisp](https://github.com/NVIDIAGameWorks/kaolin-wisp) + is a PyTorch library for working with neural fields. +- [openrlbenchmark/openrlbenchmark](https://github.com/openrlbenchmark/openrlbenchmark) + is a comprehensive collection of tracked experiments for reinforcement + learning. diff --git a/docs/source/examples/02_nesting/01_nesting.rst b/docs/source/examples/02_nesting/01_nesting.rst index e5daf3a97..363e4d9e2 100644 --- a/docs/source/examples/02_nesting/01_nesting.rst +++ b/docs/source/examples/02_nesting/01_nesting.rst @@ -71,8 +71,6 @@ objects. This helps with modularity and grouping in larger projects. print(f"{out_dir=}, {restore_checkpoint=}, {checkpoint_interval=}") print() print(f"{config=}") - print() - print(tyro.to_yaml(config)) if __name__ == "__main__": diff --git a/docs/source/examples/04_additional/02_dictionaries.rst b/docs/source/examples/04_additional/02_dictionaries.rst index 99add94e9..a9cbee7f2 100644 --- a/docs/source/examples/04_additional/02_dictionaries.rst +++ b/docs/source/examples/04_additional/02_dictionaries.rst @@ -14,7 +14,7 @@ Dictionary inputs can be specified using either a standard ``Dict[K, V]`` annota :linenos: - from typing import Dict, Mapping, Tuple, TypedDict + from typing import Dict, Tuple, TypedDict import tyro diff --git a/docs/source/index.md b/docs/source/index.md index 6898be2cc..79248c018 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -46,21 +46,6 @@ To get started, we recommend browsing the examples to the left. By extending [shtab](https://github.com/iterative/shtab), `tyro` automatically generates tab completion scripts for bash, zsh, and tcsh. -### In the wild - -`tyro` is still a new library, but being stress tested in several projects! - -- [nerfstudio-project/nerfstudio](https://github.com/nerfstudio-project/nerfstudio/) - provides a set of tools for end-to-end training, testing, and rendering of - neural radiance fields. -- [Sea-Snell/JAXSeq](https://github.com/Sea-Snell/JAXSeq/) is a library for - distributed training of large language models in JAX. -- [kevinzakka/obj2mjcf](https://github.com/kevinzakka/obj2mjcf) is an interface - for processing composite Wavefront OBJ files for Mujoco. -- [brentyi/tensorf-jax](https://github.com/brentyi/tensorf-jax/) is an - unofficial implementation of - [Tensorial Radiance Fields](https://apchenstu.github.io/TensoRF/) in JAX. - .. toctree:: diff --git a/examples/02_nesting/01_nesting.py b/examples/02_nesting/01_nesting.py index dd176fec3..b6e673056 100644 --- a/examples/02_nesting/01_nesting.py +++ b/examples/02_nesting/01_nesting.py @@ -66,8 +66,6 @@ def train( print(f"{out_dir=}, {restore_checkpoint=}, {checkpoint_interval=}") print() print(f"{config=}") - print() - print(tyro.to_yaml(config)) if __name__ == "__main__": diff --git a/tyro/__init__.py b/tyro/__init__.py index 6f2252440..477413871 100644 --- a/tyro/__init__.py +++ b/tyro/__init__.py @@ -2,6 +2,7 @@ from ._cli import cli from ._fields import MISSING_PUBLIC as MISSING from ._instantiators import UnsupportedTypeAnnotationError +from typing import TYPE_CHECKING __all__ = [ "conf", @@ -11,6 +12,6 @@ "UnsupportedTypeAnnotationError", ] -# Deprecated interface. We use a star import to prevent these from showing up in -# autocomplete engines, etc. -from ._deprecated import * # noqa +# Deprecated interface. +if not TYPE_CHECKING: + from ._deprecated import * # noqa From c379edb9afeceda992665e74e9c2b70942aef3dc Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 5 Jun 2023 01:27:51 -0700 Subject: [PATCH 296/491] Minor refactor for populating parser specifications --- tyro/_parsers.py | 213 +++++++++++++++++++++++------------------- tyro/_unsafe_cache.py | 5 +- 2 files changed, 120 insertions(+), 98 deletions(-) diff --git a/tyro/_parsers.py b/tyro/_parsers.py index b56f4d932..dab1ef38c 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -103,104 +103,54 @@ def from_callable_or_type( subparsers_from_prefix = {} for field in field_list: - if isinstance(field.typ, TypeVar): - raise _instantiators.UnsupportedTypeAnnotationError( - f"Field {field.name} has an unbound TypeVar: {field.typ}." - ) - - if _markers.Fixed not in field.markers: - # (1) Handle Unions over callables; these result in subparsers. - subparsers_attempt = SubparsersSpecification.from_field( - field, - type_from_typevar=type_from_typevar, - parent_classes=parent_classes, - prefix=_strings.make_field_name([prefix, field.name]), - ) - if subparsers_attempt is not None: - if subparsers_attempt.required: - has_required_args = True - if ( - not subparsers_attempt.required - and _markers.AvoidSubcommands in field.markers - ): - # Don't make a subparser. - field = dataclasses.replace(field, typ=type(field.default)) - else: - subparsers_from_prefix[ - subparsers_attempt.prefix - ] = subparsers_attempt - subparsers = add_subparsers_to_leaves( - subparsers, subparsers_attempt - ) - continue - - # (2) Handle nested callables. - if _fields.is_nested_type(field.typ, field.default): - field = dataclasses.replace( - field, - typ=_resolver.narrow_type( - field.typ, - field.default, - ), - ) - nested_parser = ParserSpecification.from_callable_or_type( - ( - # Recursively apply marker types. - field.typ - if len(field.markers) == 0 - else Annotated.__class_getitem__( # type: ignore - (field.typ,) + tuple(field.markers) - ) - ), - description=None, - parent_classes=parent_classes, - default_instance=field.default, - prefix=_strings.make_field_name([prefix, field.name]), - subcommand_prefix=subcommand_prefix, - ) - if nested_parser.has_required_args: - has_required_args = True - args.extend(nested_parser.args) - - # Include nested subparsers. - if nested_parser.subparsers is not None: - subparsers_from_prefix.update( - nested_parser.subparsers_from_prefix - ) - subparsers = add_subparsers_to_leaves( - subparsers, nested_parser.subparsers - ) - - # Include nested strings. - for ( - k, - v, - ) in nested_parser.helptext_from_nested_class_field_name.items(): - helptext_from_nested_class_field_name[ - _strings.make_field_name([field.name, k]) - ] = v - - if field.helptext is not None: - helptext_from_nested_class_field_name[ - _strings.make_field_name([field.name]) - ] = field.helptext - else: - helptext_from_nested_class_field_name[ - _strings.make_field_name([field.name]) - ] = _docstrings.get_callable_description(field.typ) - continue - - # (3) Handle primitive or fixed types. These produce a single argument! - arg = _arguments.ArgumentDefinition( - dest_prefix=prefix, - name_prefix=prefix, - subcommand_prefix=subcommand_prefix, - field=field, + field_out = handle_field( + field, type_from_typevar=type_from_typevar, + parent_classes=parent_classes, + prefix=prefix, + subcommand_prefix=subcommand_prefix, ) - args.append(arg) - if arg.lowered.required: - has_required_args = True + if isinstance(field_out, _arguments.ArgumentDefinition): + # Handle single arguments. + args.append(field_out) + if field_out.lowered.required: + has_required_args = True + elif isinstance(field_out, SubparsersSpecification): + # Handle subparsers. + subparsers_from_prefix[field_out.prefix] = field_out + subparsers = add_subparsers_to_leaves(subparsers, field_out) + elif isinstance(field_out, ParserSpecification): + # Handle nested parsers. + nested_parser = field_out + + if nested_parser.has_required_args: + has_required_args = True + args.extend(nested_parser.args) + + # Include nested subparsers. + if nested_parser.subparsers is not None: + subparsers_from_prefix.update(nested_parser.subparsers_from_prefix) + subparsers = add_subparsers_to_leaves( + subparsers, nested_parser.subparsers + ) + + # Include nested strings. + for ( + k, + v, + ) in nested_parser.helptext_from_nested_class_field_name.items(): + helptext_from_nested_class_field_name[ + _strings.make_field_name([field.name, k]) + ] = v + + if field.helptext is not None: + helptext_from_nested_class_field_name[ + _strings.make_field_name([field.name]) + ] = field.helptext + else: + helptext_from_nested_class_field_name[ + _strings.make_field_name([field.name]) + ] = _docstrings.get_callable_description(nested_parser.f) return ParserSpecification( f=f, @@ -306,6 +256,77 @@ def format_group_name(prefix: str) -> str: arg.add_argument(group_from_prefix[""]) +def handle_field( + field: _fields.FieldDefinition, + type_from_typevar: Dict[TypeVar, TypeForm[Any]], + parent_classes: Set[Type[Any]], + prefix: str, + subcommand_prefix: str, +) -> Union[ + _arguments.ArgumentDefinition, + ParserSpecification, + SubparsersSpecification, +]: + """Determine what to do with a single field definition.""" + + if isinstance(field.typ, TypeVar): + raise _instantiators.UnsupportedTypeAnnotationError( + f"Field {field.name} has an unbound TypeVar: {field.typ}." + ) + + if _markers.Fixed not in field.markers: + # (1) Handle Unions over callables; these result in subparsers. + subparsers_attempt = SubparsersSpecification.from_field( + field, + type_from_typevar=type_from_typevar, + parent_classes=parent_classes, + prefix=_strings.make_field_name([prefix, field.name]), + ) + if subparsers_attempt is not None: + if ( + not subparsers_attempt.required + and _markers.AvoidSubcommands in field.markers + ): + # Don't make a subparser. + field = dataclasses.replace(field, typ=type(field.default)) + else: + return subparsers_attempt + + # (2) Handle nested callables. + if _fields.is_nested_type(field.typ, field.default): + field = dataclasses.replace( + field, + typ=_resolver.narrow_type( + field.typ, + field.default, + ), + ) + return ParserSpecification.from_callable_or_type( + ( + # Recursively apply marker types. + field.typ + if len(field.markers) == 0 + else Annotated.__class_getitem__( # type: ignore + (field.typ,) + tuple(field.markers) + ) + ), + description=None, + parent_classes=parent_classes, + default_instance=field.default, + prefix=_strings.make_field_name([prefix, field.name]), + subcommand_prefix=subcommand_prefix, + ) + + # (3) Handle primitive or fixed types. These produce a single argument! + return _arguments.ArgumentDefinition( + dest_prefix=prefix, + name_prefix=prefix, + subcommand_prefix=subcommand_prefix, + field=field, + type_from_typevar=type_from_typevar, + ) + + @dataclasses.dataclass(frozen=True) class SubparsersSpecification: """Structure for defining subparsers. Each subparser is a parser with a name.""" diff --git a/tyro/_unsafe_cache.py b/tyro/_unsafe_cache.py index b47a3ac5f..8b3fc37c6 100644 --- a/tyro/_unsafe_cache.py +++ b/tyro/_unsafe_cache.py @@ -13,8 +13,9 @@ def clear_cache() -> None: def unsafe_cache(maxsize: int) -> Callable[[CallableType], CallableType]: - """Cache decorator that relies object IDs when arguments are unhashable. Assumes - immutability.""" + """Cache decorator that relies object IDs when arguments are unhashable. Makes the + very strong assumption of not only immutability, but that unhashable types don't go + out of scope.""" _cache_list.append({}) local_cache = _cache_list[-1] From 7dee00d10cd9d56015363e497dd3ec799226fbc9 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 29 Jun 2023 01:05:49 -0700 Subject: [PATCH 297/491] Simplify base configs example --- .../03_config_systems/01_base_configs.rst | 46 +++++------------- examples/03_config_systems/01_base_configs.py | 48 +++++-------------- 2 files changed, 23 insertions(+), 71 deletions(-) diff --git a/docs/source/examples/03_config_systems/01_base_configs.rst b/docs/source/examples/03_config_systems/01_base_configs.rst index 783ee3ac6..7e7073f94 100644 --- a/docs/source/examples/03_config_systems/01_base_configs.rst +++ b/docs/source/examples/03_config_systems/01_base_configs.rst @@ -9,9 +9,6 @@ We can integrate ``tyro.cli()`` into common configuration patterns: here, we sel one of multiple possible base configurations, create a subcommand for each one, and then use the CLI to either override (existing) or fill in (missing) values. -This example is verbose; to shorten, consider using -:func:`tyro.extras.subcommand_type_from_defaults()`. - .. code-block:: python @@ -62,11 +59,9 @@ This example is verbose; to shorten, consider using # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. - SmallConfig = Annotated[ - ExperimentConfig, - tyro.conf.subcommand( - name="small", - default=ExperimentConfig( + Configs = tyro.extras.subcommand_type_from_defaults( + { + "small": ExperimentConfig( dataset="mnist", optimizer=AdamOptimizer(), batch_size=2048, @@ -76,15 +71,7 @@ This example is verbose; to shorten, consider using seed=0, activation=nn.ReLU, ), - description="Train a smaller model.", - prefix_name=False, - ), - ] - BigConfig = Annotated[ - ExperimentConfig, - tyro.conf.subcommand( - name="big", - default=ExperimentConfig( + "big": ExperimentConfig( dataset="imagenet-50", optimizer=AdamOptimizer(), batch_size=32, @@ -94,22 +81,11 @@ This example is verbose; to shorten, consider using seed=0, activation=nn.GELU, ), - description="Train a bigger model.", - prefix_name=False, - ), - ] - - - @tyro.conf.configure(tyro.conf.ConsolidateSubcommandArgs) - def main( - config: Union[SmallConfig, BigConfig], - restore_checkpoint: bool = False, - ) -> None: - print(config) - + } + ) if __name__ == "__main__": - config = tyro.cli(main) + config = tyro.cli(Configs) print(config) ------------ @@ -132,9 +108,9 @@ This example is verbose; to shorten, consider using .. raw:: html - python 03_config_systems/01_base_configs.py small --config.seed 94720 + python 03_config_systems/01_base_configs.py small --seed 94720 -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py small --config.seed 94720 +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py small --seed 94720 ------------ @@ -148,6 +124,6 @@ This example is verbose; to shorten, consider using .. raw:: html - python 03_config_systems/01_base_configs.py big --config.seed 94720 + python 03_config_systems/01_base_configs.py big --seed 94720 -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py big --config.seed 94720 +.. program-output:: python ../../examples/03_config_systems/01_base_configs.py big --seed 94720 diff --git a/examples/03_config_systems/01_base_configs.py b/examples/03_config_systems/01_base_configs.py index b1507a16c..60f220913 100644 --- a/examples/03_config_systems/01_base_configs.py +++ b/examples/03_config_systems/01_base_configs.py @@ -4,16 +4,13 @@ one of multiple possible base configurations, create a subcommand for each one, and then use the CLI to either override (existing) or fill in (missing) values. -This example is verbose; to shorten, consider using -:func:`tyro.extras.subcommand_type_from_defaults()`. - Usage: -`python ./10_base_configs.py --help` -`python ./10_base_configs.py small --help` -`python ./10_base_configs.py small --config.seed 94720` -`python ./10_base_configs.py big --help` -`python ./10_base_configs.py big --config.seed 94720` +`python ./01_base_configs.py --help` +`python ./01_base_configs.py small --help` +`python ./01_base_configs.py small --seed 94720` +`python ./01_base_configs.py big --help` +`python ./01_base_configs.py big --seed 94720` """ from dataclasses import dataclass @@ -60,11 +57,9 @@ class ExperimentConfig: # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. -SmallConfig = Annotated[ - ExperimentConfig, - tyro.conf.subcommand( - name="small", - default=ExperimentConfig( +Configs = tyro.extras.subcommand_type_from_defaults( + { + "small": ExperimentConfig( dataset="mnist", optimizer=AdamOptimizer(), batch_size=2048, @@ -74,15 +69,7 @@ class ExperimentConfig: seed=0, activation=nn.ReLU, ), - description="Train a smaller model.", - prefix_name=False, - ), -] -BigConfig = Annotated[ - ExperimentConfig, - tyro.conf.subcommand( - name="big", - default=ExperimentConfig( + "big": ExperimentConfig( dataset="imagenet-50", optimizer=AdamOptimizer(), batch_size=32, @@ -92,20 +79,9 @@ class ExperimentConfig: seed=0, activation=nn.GELU, ), - description="Train a bigger model.", - prefix_name=False, - ), -] - - -@tyro.conf.configure(tyro.conf.ConsolidateSubcommandArgs) -def main( - config: Union[SmallConfig, BigConfig], - restore_checkpoint: bool = False, -) -> None: - print(config) - + } +) if __name__ == "__main__": - config = tyro.cli(main) + config = tyro.cli(Configs) print(config) From 85b6b033ddf706b6536adb73dede520b448b9e37 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 3 Jul 2023 01:52:26 -0700 Subject: [PATCH 298/491] Python 3.11, Pydantic 2.0, switch away from poetry --- .github/workflows/build.yml | 8 +- .github/workflows/coverage.yml | 6 +- .github/workflows/docs.yml | 8 +- .github/workflows/mypy.yml | 6 +- .github/workflows/publish.yml | 8 +- .../03_config_systems/01_base_configs.rst | 3 +- docs/source/installation.md | 8 +- examples/03_config_systems/01_base_configs.py | 3 +- poetry.lock | 1529 ----------------- pyproject.toml | 88 +- tests/test_conf.py | 6 +- tests/test_generics_and_serialization.py | 4 +- tests/test_nested.py | 6 +- tyro/_cli.py | 4 +- tyro/_fields.py | 50 +- 15 files changed, 127 insertions(+), 1610 deletions(-) delete mode 100644 poetry.lock diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cfb9ae844..300bf6daf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 @@ -21,8 +21,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - curl -sSL https://install.python-poetry.org | python3 - - poetry install + pip install --upgrade pip + pip install ".[dev]" - name: Test with pytest run: | - poetry run pytest + pytest diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index be421f157..d07af0bc8 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,11 +17,11 @@ jobs: python-version: "3.10" - name: Install dependencies run: | - curl -sSL https://install.python-poetry.org | python3 - - poetry install + pip install --upgrade pip + pip install -e ".[dev]" - name: Generate coverage report run: | - poetry run pytest --cov=tyro --cov-report=xml + pytest --cov=tyro --cov-report=xml - name: Upload to Codecov uses: codecov/codecov-action@v3 with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 35401b80d..14238bcac 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -15,10 +15,10 @@ jobs: # Build documentation - name: Building documentation run: | - curl -sSL https://install.python-poetry.org | python3 - - poetry install - poetry run pip install -r docs/requirements.txt - poetry run sphinx-build docs/source docs/build -b dirhtml + pip install --upgrade pip + pip install -e . + pip install -r docs/requirements.txt + sphinx-build docs/source docs/build -b dirhtml # Deploy - name: Deploy to GitHub Pages diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index e317f125b..b8efb59d5 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -17,8 +17,8 @@ jobs: python-version: "3.8" - name: Install dependencies run: | - curl -sSL https://install.python-poetry.org | python3 - - poetry install + pip install --upgrade pip + pip install -e ".[dev]" - name: Test with mypy run: | - poetry run mypy --install-types --non-interactive . + mypy --install-types --non-interactive . diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ac50fb86d..e75680dee 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,8 +20,9 @@ jobs: python-version: '3.8' - name: Install dependencies run: | - curl -sSL https://install.python-poetry.org | python3 - - poetry install + pip install --upgrade pip + pip install -e ".[dev]" + pip install build twine - name: Strip unsupported tags in README run: | sed -i '//,//d' README.md @@ -30,4 +31,5 @@ jobs: PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - poetry publish --build --username $PYPI_USERNAME --password $PYPI_PASSWORD + python -m build + twine upload --username $PYPI_USERNAME --password $PYPI_PASSWORD dist/* diff --git a/docs/source/examples/03_config_systems/01_base_configs.rst b/docs/source/examples/03_config_systems/01_base_configs.rst index 7e7073f94..30f5f4b3f 100644 --- a/docs/source/examples/03_config_systems/01_base_configs.rst +++ b/docs/source/examples/03_config_systems/01_base_configs.rst @@ -16,10 +16,9 @@ use the CLI to either override (existing) or fill in (missing) values. from dataclasses import dataclass - from typing import Callable, Literal, Tuple, Union + from typing import Callable, Literal, Tuple from torch import nn - from typing_extensions import Annotated import tyro diff --git a/docs/source/installation.md b/docs/source/installation.md index ad8b0e4b3..180bcb113 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -12,17 +12,17 @@ pip install tyro ## Development If you're interested in development, the recommended way to install `tyro` is -via [poetry](https://github.com/python-poetry/poetry). +via `pip`. ```bash # Clone repository and install. git clone git@github.com:brentyi/tyro.git cd tyro -poetry install +python -m pip install -e ".[dev]" # Run tests. -poetry run pytest +pytest # Check types. -poetry run mypy --install-types . +mypy --install-types . ``` diff --git a/examples/03_config_systems/01_base_configs.py b/examples/03_config_systems/01_base_configs.py index 60f220913..f62559304 100644 --- a/examples/03_config_systems/01_base_configs.py +++ b/examples/03_config_systems/01_base_configs.py @@ -14,10 +14,9 @@ """ from dataclasses import dataclass -from typing import Callable, Literal, Tuple, Union +from typing import Callable, Literal, Tuple from torch import nn -from typing_extensions import Annotated import tyro diff --git a/poetry.lock b/poetry.lock deleted file mode 100644 index 11ef9b2ab..000000000 --- a/poetry.lock +++ /dev/null @@ -1,1529 +0,0 @@ -# This file is automatically @generated by Poetry 1.5.0 and should not be changed by hand. - -[[package]] -name = "absl-py" -version = "1.4.0" -description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -optional = false -python-versions = ">=3.6" -files = [ - {file = "absl-py-1.4.0.tar.gz", hash = "sha256:d2c244d01048ba476e7c080bd2c6df5e141d211de80223460d5b3b8a2a58433d"}, - {file = "absl_py-1.4.0-py3-none-any.whl", hash = "sha256:0d3fe606adfa4f7db64792dd4c7aee4ee0c38ab75dfd353b7a83ed3e957fcb47"}, -] - -[[package]] -name = "antlr4-python3-runtime" -version = "4.9.3" -description = "ANTLR 4.9.3 runtime for Python 3.7" -optional = false -python-versions = "*" -files = [ - {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, -] - -[[package]] -name = "attrs" -version = "21.4.0" -description = "Classes Without Boilerplate" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, - {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, -] - -[package.extras] -dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "sphinx", "sphinx-notfound-page", "zope.interface"] -docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] -tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "zope.interface"] -tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six"] - -[[package]] -name = "backports-cached-property" -version = "1.0.2" -description = "cached_property() - computed once per instance, cached as attribute" -optional = false -python-versions = ">=3.6.0" -files = [ - {file = "backports.cached-property-1.0.2.tar.gz", hash = "sha256:9306f9eed6ec55fd156ace6bc1094e2c86fae5fb2bf07b6a9c00745c656e75dd"}, - {file = "backports.cached_property-1.0.2-py3-none-any.whl", hash = "sha256:baeb28e1cd619a3c9ab8941431fe34e8490861fb998c6c4590693d50171db0cc"}, -] - -[[package]] -name = "cached-property" -version = "1.5.2" -description = "A decorator for caching properties in classes." -optional = false -python-versions = "*" -files = [ - {file = "cached-property-1.5.2.tar.gz", hash = "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130"}, - {file = "cached_property-1.5.2-py2.py3-none-any.whl", hash = "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0"}, -] - -[[package]] -name = "chex" -version = "0.1.5" -description = "Chex: Testing made fun, in JAX!" -optional = false -python-versions = ">=3.7" -files = [ - {file = "chex-0.1.5-py3-none-any.whl", hash = "sha256:b3321184850d5fc29b2eca63087cdbdd83a1b3e4f33c1314ff8b3b8bd67abbca"}, - {file = "chex-0.1.5.tar.gz", hash = "sha256:686858320f8f220c82a6c7eeb54dcdcaa4f3d7f66690dacd13a24baa1ee8299e"}, -] - -[package.dependencies] -absl-py = ">=0.9.0" -dm-tree = ">=0.1.5" -jax = ">=0.1.55" -jaxlib = ">=0.1.37" -numpy = ">=1.18.0" -toolz = ">=0.9.0" - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "coverage" -version = "6.5.0" -description = "Code coverage measurement for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, - {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, - {file = "coverage-6.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4"}, - {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83516205e254a0cb77d2d7bb3632ee019d93d9f4005de31dca0a8c3667d5bc04"}, - {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0"}, - {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:97117225cdd992a9c2a5515db1f66b59db634f59d0679ca1fa3fe8da32749cae"}, - {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1170fa54185845505fbfa672f1c1ab175446c887cce8212c44149581cf2d466"}, - {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:11b990d520ea75e7ee8dcab5bc908072aaada194a794db9f6d7d5cfd19661e5a"}, - {file = "coverage-6.5.0-cp310-cp310-win32.whl", hash = "sha256:5dbec3b9095749390c09ab7c89d314727f18800060d8d24e87f01fb9cfb40b32"}, - {file = "coverage-6.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:59f53f1dc5b656cafb1badd0feb428c1e7bc19b867479ff72f7a9dd9b479f10e"}, - {file = "coverage-6.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795"}, - {file = "coverage-6.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75"}, - {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33a7da4376d5977fbf0a8ed91c4dffaaa8dbf0ddbf4c8eea500a2486d8bc4d7b"}, - {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91"}, - {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a6b7d95969b8845250586f269e81e5dfdd8ff828ddeb8567a4a2eaa7313460c4"}, - {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1ef221513e6f68b69ee9e159506d583d31aa3567e0ae84eaad9d6ec1107dddaa"}, - {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cca4435eebea7962a52bdb216dec27215d0df64cf27fc1dd538415f5d2b9da6b"}, - {file = "coverage-6.5.0-cp311-cp311-win32.whl", hash = "sha256:98e8a10b7a314f454d9eff4216a9a94d143a7ee65018dd12442e898ee2310578"}, - {file = "coverage-6.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:bc8ef5e043a2af066fa8cbfc6e708d58017024dc4345a1f9757b329a249f041b"}, - {file = "coverage-6.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4433b90fae13f86fafff0b326453dd42fc9a639a0d9e4eec4d366436d1a41b6d"}, - {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4f05d88d9a80ad3cac6244d36dd89a3c00abc16371769f1340101d3cb899fc3"}, - {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:94e2565443291bd778421856bc975d351738963071e9b8839ca1fc08b42d4bef"}, - {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:027018943386e7b942fa832372ebc120155fd970837489896099f5cfa2890f79"}, - {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:255758a1e3b61db372ec2736c8e2a1fdfaf563977eedbdf131de003ca5779b7d"}, - {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:851cf4ff24062c6aec510a454b2584f6e998cada52d4cb58c5e233d07172e50c"}, - {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:12adf310e4aafddc58afdb04d686795f33f4d7a6fa67a7a9d4ce7d6ae24d949f"}, - {file = "coverage-6.5.0-cp37-cp37m-win32.whl", hash = "sha256:b5604380f3415ba69de87a289a2b56687faa4fe04dbee0754bfcae433489316b"}, - {file = "coverage-6.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4a8dbc1f0fbb2ae3de73eb0bdbb914180c7abfbf258e90b311dcd4f585d44bd2"}, - {file = "coverage-6.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c"}, - {file = "coverage-6.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba"}, - {file = "coverage-6.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e"}, - {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:20c8ac5386253717e5ccc827caad43ed66fea0efe255727b1053a8154d952398"}, - {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b"}, - {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dbdb91cd8c048c2b09eb17713b0c12a54fbd587d79adcebad543bc0cd9a3410b"}, - {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:de3001a203182842a4630e7b8d1a2c7c07ec1b45d3084a83d5d227a3806f530f"}, - {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e07f4a4a9b41583d6eabec04f8b68076ab3cd44c20bd29332c6572dda36f372e"}, - {file = "coverage-6.5.0-cp38-cp38-win32.whl", hash = "sha256:6d4817234349a80dbf03640cec6109cd90cba068330703fa65ddf56b60223a6d"}, - {file = "coverage-6.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:7ccf362abd726b0410bf8911c31fbf97f09f8f1061f8c1cf03dfc4b6372848f6"}, - {file = "coverage-6.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745"}, - {file = "coverage-6.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc"}, - {file = "coverage-6.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe"}, - {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:265de0fa6778d07de30bcf4d9dc471c3dc4314a23a3c6603d356a3c9abc2dfcf"}, - {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5"}, - {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7b6be138d61e458e18d8e6ddcddd36dd96215edfe5f1168de0b1b32635839b62"}, - {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:42eafe6778551cf006a7c43153af1211c3aaab658d4d66fa5fcc021613d02518"}, - {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:723e8130d4ecc8f56e9a611e73b31219595baa3bb252d539206f7bbbab6ffc1f"}, - {file = "coverage-6.5.0-cp39-cp39-win32.whl", hash = "sha256:d9ecf0829c6a62b9b573c7bb6d4dcd6ba8b6f80be9ba4fc7ed50bf4ac9aecd72"}, - {file = "coverage-6.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc2af30ed0d5ae0b1abdb4ebdce598eafd5b35397d4d75deb341a614d333d987"}, - {file = "coverage-6.5.0-pp36.pp37.pp38-none-any.whl", hash = "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a"}, - {file = "coverage-6.5.0.tar.gz", hash = "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84"}, -] - -[package.dependencies] -tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} - -[package.extras] -toml = ["tomli"] - -[[package]] -name = "cycler" -version = "0.11.0" -description = "Composable style cycles" -optional = false -python-versions = ">=3.6" -files = [ - {file = "cycler-0.11.0-py3-none-any.whl", hash = "sha256:3a27e95f763a428a739d2add979fa7494c912a32c17c4c38c4d5f082cad165a3"}, - {file = "cycler-0.11.0.tar.gz", hash = "sha256:9c87405839a19696e837b3b818fed3f5f69f16f1eec1a1ad77e043dcea9c772f"}, -] - -[[package]] -name = "dm-tree" -version = "0.1.8" -description = "Tree is a library for working with nested data structures." -optional = false -python-versions = "*" -files = [ - {file = "dm-tree-0.1.8.tar.gz", hash = "sha256:0fcaabbb14e7980377439e7140bd05552739ca5e515ecb3119f234acee4b9430"}, - {file = "dm_tree-0.1.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:35cc164a79336bfcfafb47e5f297898359123bbd3330c1967f0c4994f9cf9f60"}, - {file = "dm_tree-0.1.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39070ba268c0491af9fe7a58644d99e8b4f2cde6e5884ba3380bddc84ed43d5f"}, - {file = "dm_tree-0.1.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2869228d9c619074de501a3c10dc7f07c75422f8fab36ecdcb859b6f1b1ec3ef"}, - {file = "dm_tree-0.1.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d20f2faa3672b52e5013f4077117bfb99c4cfc0b445d3bde1584c34032b57436"}, - {file = "dm_tree-0.1.8-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5483dca4d7eb1a0d65fe86d3b6a53ae717face83c1f17e0887b1a4a64ae5c410"}, - {file = "dm_tree-0.1.8-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1d7c26e431fc93cc7e0cba867eb000db6a05f6f2b25af11ac4e9dada88fc5bca"}, - {file = "dm_tree-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4d714371bb08839e4e5e29024fc95832d9affe129825ef38836b143028bd144"}, - {file = "dm_tree-0.1.8-cp310-cp310-win_amd64.whl", hash = "sha256:d40fa4106ca6edc66760246a08f500ec0c85ef55c762fb4a363f6ee739ba02ee"}, - {file = "dm_tree-0.1.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad16ceba90a56ec47cf45b21856d14962ac314787975ef786efb5e6e9ca75ec7"}, - {file = "dm_tree-0.1.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:803bfc53b4659f447ac694dbd04235f94a73ef7c1fd1e0df7c84ac41e0bc963b"}, - {file = "dm_tree-0.1.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:378cc8ad93c5fe3590f405a309980721f021c790ca1bdf9b15bb1d59daec57f5"}, - {file = "dm_tree-0.1.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1607ce49aa42f010d1e5e616d92ce899d66835d4d8bea49679582435285515de"}, - {file = "dm_tree-0.1.8-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:343a4a4ebaa127451ff971254a4be4084eb4bdc0b2513c32b46f6f728fd03f9e"}, - {file = "dm_tree-0.1.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa42a605d099ee7d41ba2b5fb75e21423951fd26e5d50583a00471238fb3021d"}, - {file = "dm_tree-0.1.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83b7764de0d855338abefc6e3ee9fe40d301668310aa3baea3f778ff051f4393"}, - {file = "dm_tree-0.1.8-cp311-cp311-win_amd64.whl", hash = "sha256:a5d819c38c03f0bb5b3b3703c60e4b170355a0fc6b5819325bf3d4ceb3ae7e80"}, - {file = "dm_tree-0.1.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8c60a7eadab64c2278861f56bca320b2720f163dca9d7558103c3b77f2416571"}, - {file = "dm_tree-0.1.8-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af4b3d372f2477dcd89a6e717e4a575ca35ccc20cc4454a8a4b6f8838a00672d"}, - {file = "dm_tree-0.1.8-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de287fabc464b8734be251e46e06aa9aa1001f34198da2b6ce07bd197172b9cb"}, - {file = "dm_tree-0.1.8-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:054b461f8176f4bce7a21f7b1870f873a1ced3bdbe1282c816c550bb43c71fa6"}, - {file = "dm_tree-0.1.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f7915660f59c09068e428613c480150180df1060561fd0d1470684ae7007bd1"}, - {file = "dm_tree-0.1.8-cp37-cp37m-win_amd64.whl", hash = "sha256:b9f89a454e98806b44fe9d40ec9eee61f848388f7e79ac2371a55679bd5a3ac6"}, - {file = "dm_tree-0.1.8-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0e9620ccf06393eb6b613b5e366469304622d4ea96ae6540b28a33840e6c89cf"}, - {file = "dm_tree-0.1.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b095ba4f8ca1ba19350fd53cf1f8f3eb0bd406aa28af64a6dfc86707b32a810a"}, - {file = "dm_tree-0.1.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b9bd9b9ccb59409d33d51d84b7668010c04c2af7d4a371632874c1ca356cff3d"}, - {file = "dm_tree-0.1.8-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d3172394079a86c3a759179c65f64c48d1a42b89495fcf38976d11cc3bb952c"}, - {file = "dm_tree-0.1.8-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1612fcaecd79023dbc6a6ae48d51a80beb5c385d6f3f6d71688e57bc8d07de8"}, - {file = "dm_tree-0.1.8-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c5c8c12e3fda754ef6af94161bacdaeda816d941995fac415d6855c6c386af68"}, - {file = "dm_tree-0.1.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:694c3654cfd2a81552c08ec66bb5c4a3d48fa292b9a181880fb081c36c5b9134"}, - {file = "dm_tree-0.1.8-cp38-cp38-win_amd64.whl", hash = "sha256:bb2d109f42190225112da899b9f3d46d0d5f26aef501c61e43529fe9322530b5"}, - {file = "dm_tree-0.1.8-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d16e1f2a073604cfcc09f7131ae8d534674f43c3aef4c25742eae295bc60d04f"}, - {file = "dm_tree-0.1.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:250b692fb75f45f02e2f58fbef9ab338904ef334b90557565621fa251df267cf"}, - {file = "dm_tree-0.1.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81fce77f22a302d7a5968aebdf4efafef4def7ce96528719a354e6990dcd49c7"}, - {file = "dm_tree-0.1.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7ac31b9aecccb2c6e1ab29706f6ded3eba0c2c69c770322c9c685929c3d6afb"}, - {file = "dm_tree-0.1.8-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fe962015b2fe1282892b28ebe962faed53c7f98d942da9a4625cbf27baef913"}, - {file = "dm_tree-0.1.8-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c52cbf4f8b3dbd0beaedf44f69fa85eec5e9dede612e08035e06ada6ec9426"}, - {file = "dm_tree-0.1.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:181c35521d480d0365f39300542cb6cd7fd2b77351bb43d7acfda15aef63b317"}, - {file = "dm_tree-0.1.8-cp39-cp39-win_amd64.whl", hash = "sha256:8ed3564abed97c806db122c2d3e1a2b64c74a63debe9903aad795167cc301368"}, -] - -[[package]] -name = "docstring-parser" -version = "0.14.1" -description = "Parse Python docstrings in reST, Google and Numpydoc format" -optional = false -python-versions = ">=3.6,<4.0" -files = [ - {file = "docstring_parser-0.14.1-py3-none-any.whl", hash = "sha256:14ac6ec1f1ba6905c4d8cb90fd0bc55394f5678183752c90e44812bf28d7a515"}, - {file = "docstring_parser-0.14.1.tar.gz", hash = "sha256:2c77522e31b7c88b1ab457a1f3c9ae38947ad719732260ba77ee8a3deb58622a"}, -] - -[[package]] -name = "etils" -version = "0.9.0" -description = "Collection of common python utils" -optional = false -python-versions = ">=3.7" -files = [ - {file = "etils-0.9.0-py3-none-any.whl", hash = "sha256:635d6f7d1c519eb194304228543a4c5c7df0e6b58243302473e34c18cf720588"}, - {file = "etils-0.9.0.tar.gz", hash = "sha256:489103e9e499a566765c60458ee15d185cf0065f2060a4d16a68f8f46962ed0d"}, -] - -[package.extras] -all = ["etils[array-types]", "etils[eapp]", "etils[ecolab]", "etils[edc]", "etils[enp]", "etils[epath]", "etils[epy]", "etils[etqdm]", "etils[etree-dm]", "etils[etree-jax]", "etils[etree-tf]", "etils[etree]"] -array-types = ["etils[enp]"] -dev = ["chex", "pylint (>=2.6.0)", "pytest", "pytest-subtests", "pytest-xdist", "yapf"] -eapp = ["absl-py", "simple_parsing"] -ecolab = ["etils[enp]", "etils[epy]", "jupyter", "mediapy", "numpy"] -edc = ["etils[epy]", "typing_extensions"] -enp = ["etils[epy]", "numpy"] -epath = ["etils[epy]", "importlib_resources", "typing_extensions", "zipp"] -epy = ["typing_extensions"] -etqdm = ["absl-py", "etils[epy]", "tqdm"] -etree = ["etils[array-types]", "etils[enp]", "etils[epy]", "etils[etqdm]"] -etree-dm = ["dm-tree", "etils[etree]"] -etree-jax = ["etils[etree]", "jax[cpu]"] -etree-tf = ["etils[etree]", "tf-nightly"] -lazy-imports = ["etils[ecolab]"] - -[[package]] -name = "exceptiongroup" -version = "1.1.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, - {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, -] - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "flax" -version = "0.6.4" -description = "Flax: A neural network library for JAX designed for flexibility" -optional = false -python-versions = "*" -files = [ - {file = "flax-0.6.4-py3-none-any.whl", hash = "sha256:fe5010525202241fdc960920033d2e4c0b35f06090c1ad9e280b1f4415ae308f"}, - {file = "flax-0.6.4.tar.gz", hash = "sha256:d06465a3e6636c3c23c29f651a13f5367d06c41373b441dc8ec1bfaa4db06a48"}, -] - -[package.dependencies] -jax = ">=0.3.16" -matplotlib = "*" -msgpack = "*" -numpy = ">=1.12" -optax = "*" -orbax = "*" -PyYAML = ">=5.4.1" -rich = ">=11.1" -tensorstore = "*" -typing-extensions = ">=4.1.1" - -[package.extras] -testing = ["atari-py (==0.2.5)", "clu", "gym (==0.18.3)", "jaxlib", "jraph (>=0.0.6dev0)", "ml-collections", "mypy", "opencv-python", "pytest", "pytest-cov", "pytest-custom-exit-code", "pytest-xdist (==1.34.0)", "pytype", "sentencepiece", "tensorflow", "tensorflow-datasets", "tensorflow-text (>=2.4.0)", "torch"] - -[[package]] -name = "fonttools" -version = "4.38.0" -description = "Tools to manipulate font files" -optional = false -python-versions = ">=3.7" -files = [ - {file = "fonttools-4.38.0-py3-none-any.whl", hash = "sha256:820466f43c8be8c3009aef8b87e785014133508f0de64ec469e4efb643ae54fb"}, - {file = "fonttools-4.38.0.zip", hash = "sha256:2bb244009f9bf3fa100fc3ead6aeb99febe5985fa20afbfbaa2f8946c2fbdaf1"}, -] - -[package.extras] -all = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "fs (>=2.2.0,<3)", "lxml (>=4.0,<5)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres", "scipy", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=14.0.0)", "xattr", "zopfli (>=0.1.4)"] -graphite = ["lz4 (>=1.7.4.2)"] -interpolatable = ["munkres", "scipy"] -lxml = ["lxml (>=4.0,<5)"] -pathops = ["skia-pathops (>=0.5.0)"] -plot = ["matplotlib"] -repacker = ["uharfbuzz (>=0.23.0)"] -symfont = ["sympy"] -type1 = ["xattr"] -ufo = ["fs (>=2.2.0,<3)"] -unicode = ["unicodedata2 (>=14.0.0)"] -woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"] - -[[package]] -name = "frozendict" -version = "2.3.8" -description = "A simple immutable dictionary" -optional = false -python-versions = ">=3.6" -files = [ - {file = "frozendict-2.3.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d188d062084fba0e4bf32719ff7380b26c050b932ff164043ce82ab90587c52b"}, - {file = "frozendict-2.3.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f2a4e818ac457f6354401dcb631527af25e5a20fcfc81e6b5054b45fc245caca"}, - {file = "frozendict-2.3.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a506d807858fa961aaa5b48dab6154fdc6bd045bbe9310788bbff141bb42d13"}, - {file = "frozendict-2.3.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:750632cc890d8ee9484fe6d31b261159144b6efacc08e1317fe46accd1410373"}, - {file = "frozendict-2.3.8-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7ee5fe2658a8ac9a57f748acaf563f6a47f80b8308cbf0a04fac0ba057d41f75"}, - {file = "frozendict-2.3.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23c4bb46e6b8246e1e7e49b5593c2bc09221db0d8f31f7c092be8dfb42b9e620"}, - {file = "frozendict-2.3.8-cp310-cp310-win_amd64.whl", hash = "sha256:c31abc8acea309b132dde441856829f6003a3d242da8b54bce4c0f2a3c8c63f0"}, - {file = "frozendict-2.3.8-cp310-cp310-win_arm64.whl", hash = "sha256:9ea5520e85447ff8d4681e181941e482662817ccba921b7cb3f87922056d892a"}, - {file = "frozendict-2.3.8-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f83fed36497af9562ead5e9fb8443224ba2781786bd3b92b1087cb7d0ff20135"}, - {file = "frozendict-2.3.8-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e27c5c1d29d0eda7979253ec88abc239da1313b38f39f4b16984db3b3e482300"}, - {file = "frozendict-2.3.8-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4c785de7f1a13f15963945f400656b18f057c2fc76c089dacf127a2bb188c03"}, - {file = "frozendict-2.3.8-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8cf35ddd25513428ec152614def9696afb93ae5ec0eb54fa6aa6206eda77ac4c"}, - {file = "frozendict-2.3.8-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:ffc684773de7c88724788fa9787d0016fd75830412d58acbd9ed1a04762c675b"}, - {file = "frozendict-2.3.8-cp36-cp36m-win_amd64.whl", hash = "sha256:4c258aab9c8488338634f2ec670ef049dbf0ab0e7a2fa9bc2c7b5009cb614801"}, - {file = "frozendict-2.3.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:47fc26468407fdeb428cfc89495b7921419e670355c21b383765482fdf6c5c14"}, - {file = "frozendict-2.3.8-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ea638228692db2bf94bce40ea4b25f4077588497b516bd16576575560094bd9"}, - {file = "frozendict-2.3.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a75bf87e76c4386caecdbdd02a99e53ad43a6b5c38fb3d5a634a9fc9ce41462"}, - {file = "frozendict-2.3.8-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ed5a6c5c7a0f57269577c2a338a6002949aea21a23b7b7d06da7e7dced8b605b"}, - {file = "frozendict-2.3.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d086440328a465dea9bef2dbad7548d75d1a0a0d21f43a08c03e1ec79ac5240e"}, - {file = "frozendict-2.3.8-cp37-cp37m-win_amd64.whl", hash = "sha256:0bc4767e2f83db5b701c787e22380296977368b0c57e485ca71b2eedfa11c4a3"}, - {file = "frozendict-2.3.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:638cf363d3cbca31a341503cf2219eac52a5f5140449676fae3d9644cd3c5487"}, - {file = "frozendict-2.3.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2b2fd8ce36277919b36e3c834d2389f3cd7ac068ae730c312671dd4439a5dd65"}, - {file = "frozendict-2.3.8-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3957d52f1906b0c85f641a1911d214255873f6408ab4e5ad657cc27a247fb145"}, - {file = "frozendict-2.3.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72cfe08ab8ae524e54848fa90b22d02c1b1ecfb3064438696bcaa4b953f18772"}, - {file = "frozendict-2.3.8-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4742e76c4111bd09198d3ab66cef94be8506212311338f9182d6ef5f5cb60493"}, - {file = "frozendict-2.3.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:313ed8d9ba6bac35d7635cd9580ee5721a0fb016f4d2d20f0efa05dbecbdb1be"}, - {file = "frozendict-2.3.8-cp38-cp38-win_amd64.whl", hash = "sha256:d3c6ce943946c2a61501c8cf116fff0892d11dd579877eb36e2aea2c27fddfef"}, - {file = "frozendict-2.3.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0f573dc4861dd7ec9e055c8cceaf45355e894e749f621f199aab7b311ac4bdb"}, - {file = "frozendict-2.3.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2b3435e5f1ca5ae68a5e95e64b09d6d5c645cadd6b87569a0b3019dd248c8d00"}, - {file = "frozendict-2.3.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:145afd033ebfade28416093335261b8ec1af5cccc593482309e7add062ec8668"}, - {file = "frozendict-2.3.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da98427de26b5a2865727947480cbb53860089c4d195baa29c539da811cea617"}, - {file = "frozendict-2.3.8-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5e82befa7c385a668d569cebbebbdf49cee6fea4083f08e869a1b08cfb640a9f"}, - {file = "frozendict-2.3.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:80abe81d36e889ceec665e06ec764a7638000fa3e7be09786ac4d3ddc64b76db"}, - {file = "frozendict-2.3.8-cp39-cp39-win_amd64.whl", hash = "sha256:8ccc94ac781710db44e142e1a11ff9b31d02c032c01c6868d51fcbef73086225"}, - {file = "frozendict-2.3.8-cp39-cp39-win_arm64.whl", hash = "sha256:e72dbc1bcc2203cef38d205f692396f5505921a5680f66aa9a7e8bb71fd38f28"}, - {file = "frozendict-2.3.8-py311-none-any.whl", hash = "sha256:ba41a7ed019bd03b62d63ed3f8dea35b8243d1936f7c9ed4b5298ca45a01928e"}, - {file = "frozendict-2.3.8.tar.gz", hash = "sha256:5526559eca8f1780a4ee5146896f59afc31435313560208dd394a3a5e537d3ff"}, -] - -[[package]] -name = "importlib-metadata" -version = "6.6.0" -description = "Read metadata from Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "importlib_metadata-6.6.0-py3-none-any.whl", hash = "sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed"}, - {file = "importlib_metadata-6.6.0.tar.gz", hash = "sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705"}, -] - -[package.dependencies] -typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} -zipp = ">=0.5" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] - -[[package]] -name = "importlib-resources" -version = "5.12.0" -description = "Read resources from Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, - {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, -] - -[package.dependencies] -zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[[package]] -name = "iniconfig" -version = "2.0.0" -description = "brain-dead simple config-ini parsing" -optional = false -python-versions = ">=3.7" -files = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, -] - -[[package]] -name = "jax" -version = "0.3.25" -description = "Differentiate, compile, and transform Numpy code." -optional = false -python-versions = ">=3.7" -files = [ - {file = "jax-0.3.25.tar.gz", hash = "sha256:18bea69321cb95ea5ea913adfe5e2c1d453cade9d4cfd0dc814ecba9fc0cb6e3"}, -] - -[package.dependencies] -numpy = ">=1.20" -opt_einsum = "*" -scipy = ">=1.5" -typing_extensions = "*" - -[package.extras] -australis = ["protobuf (>=3.13,<4)"] -ci = ["jaxlib (==0.3.24)"] -cpu = ["jaxlib (==0.3.25)"] -cuda = ["jaxlib (==0.3.25+cuda11.cudnn82)"] -cuda11-cudnn805 = ["jaxlib (==0.3.25+cuda11.cudnn805)"] -cuda11-cudnn82 = ["jaxlib (==0.3.25+cuda11.cudnn82)"] -minimum-jaxlib = ["jaxlib (==0.3.22)"] -tpu = ["jaxlib (==0.3.25)", "libtpu-nightly (==0.1.dev20221109)", "requests"] - -[[package]] -name = "jaxlib" -version = "0.3.25" -description = "XLA library for JAX" -optional = false -python-versions = ">=3.7" -files = [ - {file = "jaxlib-0.3.25-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:09508f7000c0fa958fba29267338e8de75b31d7ea29bd79719a568c38f0f8d31"}, - {file = "jaxlib-0.3.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c75c8efd3702687968820446e3fb9ff997f8a2a07ab92e33b80e2f12eab3d9a"}, - {file = "jaxlib-0.3.25-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:6e2f4e51041b8371aa3976b5a3a9cdcdccb1bd7b040c9b1345cbf24bd28a8d19"}, - {file = "jaxlib-0.3.25-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:f2d517635fd77e2729c0ab7863be0d290927d01b2abb2f5dc955c821f8b0d53e"}, - {file = "jaxlib-0.3.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:13446a8382aa9ed944c16af636ca111d0afbbead91eed5cc2dc71195045e71b3"}, - {file = "jaxlib-0.3.25-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:71866aeaafbc9a12b59dcbe443353772ef235b40c53f8bd7403d39311822c276"}, - {file = "jaxlib-0.3.25-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:1e59ba58c9e93c1e1cef243f2609ec0b0c0a81160c20b9555aecdea32ccd6a78"}, - {file = "jaxlib-0.3.25-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:5295354ed5db111e6f3e88cdfa4010d11c33dd926ac61735b9096b4e0746aa7b"}, - {file = "jaxlib-0.3.25-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:9f3116389ee834b3cdeb30001b085f4b55d7741366034f041c1d377154aa5afa"}, - {file = "jaxlib-0.3.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:78b29c72d0680829db9377ed9be326875849258a60b8173b4a388b34ad18bc78"}, - {file = "jaxlib-0.3.25-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:2e008e0a6c10aa7e949555e98dc0471e0d550d5d7c109771e38a971b49480538"}, - {file = "jaxlib-0.3.25-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:fec043cdd55f3257d02e9d8880b33860eacadcae1bd5e26f43fdd08ada23614d"}, - {file = "jaxlib-0.3.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a50193ba0cbf879021c9d73d7bcfa7eafb9138895d057b774c301aac3701f9a5"}, - {file = "jaxlib-0.3.25-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:1f1448f102a9d05186f579b6931fa0c607783ecc915fdfaa482c19538affa180"}, -] - -[package.dependencies] -numpy = ">=1.20" -scipy = ">=1.5" - -[[package]] -name = "kiwisolver" -version = "1.4.4" -description = "A fast implementation of the Cassowary constraint solver" -optional = false -python-versions = ">=3.7" -files = [ - {file = "kiwisolver-1.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2f5e60fabb7343a836360c4f0919b8cd0d6dbf08ad2ca6b9cf90bf0c76a3c4f6"}, - {file = "kiwisolver-1.4.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:10ee06759482c78bdb864f4109886dff7b8a56529bc1609d4f1112b93fe6423c"}, - {file = "kiwisolver-1.4.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c79ebe8f3676a4c6630fd3f777f3cfecf9289666c84e775a67d1d358578dc2e3"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:abbe9fa13da955feb8202e215c4018f4bb57469b1b78c7a4c5c7b93001699938"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7577c1987baa3adc4b3c62c33bd1118c3ef5c8ddef36f0f2c950ae0b199e100d"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8ad8285b01b0d4695102546b342b493b3ccc6781fc28c8c6a1bb63e95d22f09"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ed58b8acf29798b036d347791141767ccf65eee7f26bde03a71c944449e53de"}, - {file = "kiwisolver-1.4.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a68b62a02953b9841730db7797422f983935aeefceb1679f0fc85cbfbd311c32"}, - {file = "kiwisolver-1.4.4-cp310-cp310-win32.whl", hash = "sha256:e92a513161077b53447160b9bd8f522edfbed4bd9759e4c18ab05d7ef7e49408"}, - {file = "kiwisolver-1.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:3fe20f63c9ecee44560d0e7f116b3a747a5d7203376abeea292ab3152334d004"}, - {file = "kiwisolver-1.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e0ea21f66820452a3f5d1655f8704a60d66ba1191359b96541eaf457710a5fc6"}, - {file = "kiwisolver-1.4.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bc9db8a3efb3e403e4ecc6cd9489ea2bac94244f80c78e27c31dcc00d2790ac2"}, - {file = "kiwisolver-1.4.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d5b61785a9ce44e5a4b880272baa7cf6c8f48a5180c3e81c59553ba0cb0821ca"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2dbb44c3f7e6c4d3487b31037b1bdbf424d97687c1747ce4ff2895795c9bf69"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6295ecd49304dcf3bfbfa45d9a081c96509e95f4b9d0eb7ee4ec0530c4a96514"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4bd472dbe5e136f96a4b18f295d159d7f26fd399136f5b17b08c4e5f498cd494"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bf7d9fce9bcc4752ca4a1b80aabd38f6d19009ea5cbda0e0856983cf6d0023f5"}, - {file = "kiwisolver-1.4.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d6601aed50c74e0ef02f4204da1816147a6d3fbdc8b3872d263338a9052c51"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:877272cf6b4b7e94c9614f9b10140e198d2186363728ed0f701c6eee1baec1da"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:db608a6757adabb32f1cfe6066e39b3706d8c3aa69bbc353a5b61edad36a5cb4"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:5853eb494c71e267912275e5586fe281444eb5e722de4e131cddf9d442615626"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:f0a1dbdb5ecbef0d34eb77e56fcb3e95bbd7e50835d9782a45df81cc46949750"}, - {file = "kiwisolver-1.4.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:283dffbf061a4ec60391d51e6155e372a1f7a4f5b15d59c8505339454f8989e4"}, - {file = "kiwisolver-1.4.4-cp311-cp311-win32.whl", hash = "sha256:d06adcfa62a4431d404c31216f0f8ac97397d799cd53800e9d3efc2fbb3cf14e"}, - {file = "kiwisolver-1.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:e7da3fec7408813a7cebc9e4ec55afed2d0fd65c4754bc376bf03498d4e92686"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:62ac9cc684da4cf1778d07a89bf5f81b35834cb96ca523d3a7fb32509380cbf6"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41dae968a94b1ef1897cb322b39360a0812661dba7c682aa45098eb8e193dbdf"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02f79693ec433cb4b5f51694e8477ae83b3205768a6fb48ffba60549080e295b"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0611a0a2a518464c05ddd5a3a1a0e856ccc10e67079bb17f265ad19ab3c7597"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:db5283d90da4174865d520e7366801a93777201e91e79bacbac6e6927cbceede"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1041feb4cda8708ce73bb4dcb9ce1ccf49d553bf87c3954bdfa46f0c3f77252c"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-win32.whl", hash = "sha256:a553dadda40fef6bfa1456dc4be49b113aa92c2a9a9e8711e955618cd69622e3"}, - {file = "kiwisolver-1.4.4-cp37-cp37m-win_amd64.whl", hash = "sha256:03baab2d6b4a54ddbb43bba1a3a2d1627e82d205c5cf8f4c924dc49284b87166"}, - {file = "kiwisolver-1.4.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:841293b17ad704d70c578f1f0013c890e219952169ce8a24ebc063eecf775454"}, - {file = "kiwisolver-1.4.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f4f270de01dd3e129a72efad823da90cc4d6aafb64c410c9033aba70db9f1ff0"}, - {file = "kiwisolver-1.4.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f9f39e2f049db33a908319cf46624a569b36983c7c78318e9726a4cb8923b26c"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c97528e64cb9ebeff9701e7938653a9951922f2a38bd847787d4a8e498cc83ae"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d1573129aa0fd901076e2bfb4275a35f5b7aa60fbfb984499d661ec950320b0"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad881edc7ccb9d65b0224f4e4d05a1e85cf62d73aab798943df6d48ab0cd79a1"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b428ef021242344340460fa4c9185d0b1f66fbdbfecc6c63eff4b7c29fad429d"}, - {file = "kiwisolver-1.4.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2e407cb4bd5a13984a6c2c0fe1845e4e41e96f183e5e5cd4d77a857d9693494c"}, - {file = "kiwisolver-1.4.4-cp38-cp38-win32.whl", hash = "sha256:75facbe9606748f43428fc91a43edb46c7ff68889b91fa31f53b58894503a191"}, - {file = "kiwisolver-1.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:5bce61af018b0cb2055e0e72e7d65290d822d3feee430b7b8203d8a855e78766"}, - {file = "kiwisolver-1.4.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8c808594c88a025d4e322d5bb549282c93c8e1ba71b790f539567932722d7bd8"}, - {file = "kiwisolver-1.4.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0a71d85ecdd570ded8ac3d1c0f480842f49a40beb423bb8014539a9f32a5897"}, - {file = "kiwisolver-1.4.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b533558eae785e33e8c148a8d9921692a9fe5aa516efbdff8606e7d87b9d5824"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:efda5fc8cc1c61e4f639b8067d118e742b812c930f708e6667a5ce0d13499e29"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7c43e1e1206cd421cd92e6b3280d4385d41d7166b3ed577ac20444b6995a445f"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc8d3bd6c72b2dd9decf16ce70e20abcb3274ba01b4e1c96031e0c4067d1e7cd"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ea39b0ccc4f5d803e3337dd46bcce60b702be4d86fd0b3d7531ef10fd99a1ac"}, - {file = "kiwisolver-1.4.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:968f44fdbf6dd757d12920d63b566eeb4d5b395fd2d00d29d7ef00a00582aac9"}, - {file = "kiwisolver-1.4.4-cp39-cp39-win32.whl", hash = "sha256:da7e547706e69e45d95e116e6939488d62174e033b763ab1496b4c29b76fabea"}, - {file = "kiwisolver-1.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:ba59c92039ec0a66103b1d5fe588fa546373587a7d68f5c96f743c3396afc04b"}, - {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:91672bacaa030f92fc2f43b620d7b337fd9a5af28b0d6ed3f77afc43c4a64b5a"}, - {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:787518a6789009c159453da4d6b683f468ef7a65bbde796bcea803ccf191058d"}, - {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da152d8cdcab0e56e4f45eb08b9aea6455845ec83172092f09b0e077ece2cf7a"}, - {file = "kiwisolver-1.4.4-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:ecb1fa0db7bf4cff9dac752abb19505a233c7f16684c5826d1f11ebd9472b871"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:28bc5b299f48150b5f822ce68624e445040595a4ac3d59251703779836eceff9"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:81e38381b782cc7e1e46c4e14cd997ee6040768101aefc8fa3c24a4cc58e98f8"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2a66fdfb34e05b705620dd567f5a03f239a088d5a3f321e7b6ac3239d22aa286"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872b8ca05c40d309ed13eb2e582cab0c5a05e81e987ab9c521bf05ad1d5cf5cb"}, - {file = "kiwisolver-1.4.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:70e7c2e7b750585569564e2e5ca9845acfaa5da56ac46df68414f29fea97be9f"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9f85003f5dfa867e86d53fac6f7e6f30c045673fa27b603c397753bebadc3008"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e307eb9bd99801f82789b44bb45e9f541961831c7311521b13a6c85afc09767"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1792d939ec70abe76f5054d3f36ed5656021dcad1322d1cc996d4e54165cef9"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6cb459eea32a4e2cf18ba5fcece2dbdf496384413bc1bae15583f19e567f3b2"}, - {file = "kiwisolver-1.4.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36dafec3d6d6088d34e2de6b85f9d8e2324eb734162fba59d2ba9ed7a2043d5b"}, - {file = "kiwisolver-1.4.4.tar.gz", hash = "sha256:d41997519fcba4a1e46eb4a2fe31bc12f0ff957b2b81bac28db24744f333e955"}, -] - -[package.dependencies] -typing-extensions = {version = "*", markers = "python_version < \"3.8\""} - -[[package]] -name = "markdown-it-py" -version = "2.2.0" -description = "Python port of markdown-it. Markdown parsing, done right!" -optional = false -python-versions = ">=3.7" -files = [ - {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"}, - {file = "markdown_it_py-2.2.0-py3-none-any.whl", hash = "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30"}, -] - -[package.dependencies] -mdurl = ">=0.1,<1.0" -typing_extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""} - -[package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark"] -code-style = ["pre-commit (>=3.0,<4.0)"] -compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] -linkify = ["linkify-it-py (>=1,<3)"] -plugins = ["mdit-py-plugins"] -profiling = ["gprof2dot"] -rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] - -[[package]] -name = "matplotlib" -version = "3.5.3" -description = "Python plotting package" -optional = false -python-versions = ">=3.7" -files = [ - {file = "matplotlib-3.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a206a1b762b39398efea838f528b3a6d60cdb26fe9d58b48265787e29cd1d693"}, - {file = "matplotlib-3.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cd45a6f3e93a780185f70f05cf2a383daed13c3489233faad83e81720f7ede24"}, - {file = "matplotlib-3.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d62880e1f60e5a30a2a8484432bcb3a5056969dc97258d7326ad465feb7ae069"}, - {file = "matplotlib-3.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ab29589cef03bc88acfa3a1490359000c18186fc30374d8aa77d33cc4a51a4a"}, - {file = "matplotlib-3.5.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2886cc009f40e2984c083687251821f305d811d38e3df8ded414265e4583f0c5"}, - {file = "matplotlib-3.5.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c995f7d9568f18b5db131ab124c64e51b6820a92d10246d4f2b3f3a66698a15b"}, - {file = "matplotlib-3.5.3-cp310-cp310-win32.whl", hash = "sha256:6bb93a0492d68461bd458eba878f52fdc8ac7bdb6c4acdfe43dba684787838c2"}, - {file = "matplotlib-3.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:2e6d184ebe291b9e8f7e78bbab7987d269c38ea3e062eace1fe7d898042ef804"}, - {file = "matplotlib-3.5.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6ea6aef5c4338e58d8d376068e28f80a24f54e69f09479d1c90b7172bad9f25b"}, - {file = "matplotlib-3.5.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:839d47b8ead7ad9669aaacdbc03f29656dc21f0d41a6fea2d473d856c39c8b1c"}, - {file = "matplotlib-3.5.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3b4fa56159dc3c7f9250df88f653f085068bcd32dcd38e479bba58909254af7f"}, - {file = "matplotlib-3.5.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:94ff86af56a3869a4ae26a9637a849effd7643858a1a04dd5ee50e9ab75069a7"}, - {file = "matplotlib-3.5.3-cp37-cp37m-win32.whl", hash = "sha256:35a8ad4dddebd51f94c5d24bec689ec0ec66173bf614374a1244c6241c1595e0"}, - {file = "matplotlib-3.5.3-cp37-cp37m-win_amd64.whl", hash = "sha256:43e9d3fa077bf0cc95ded13d331d2156f9973dce17c6f0c8b49ccd57af94dbd9"}, - {file = "matplotlib-3.5.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:22227c976ad4dc8c5a5057540421f0d8708c6560744ad2ad638d48e2984e1dbc"}, - {file = "matplotlib-3.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bf618a825deb6205f015df6dfe6167a5d9b351203b03fab82043ae1d30f16511"}, - {file = "matplotlib-3.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9befa5954cdbc085e37d974ff6053da269474177921dd61facdad8023c4aeb51"}, - {file = "matplotlib-3.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3840c280ebc87a48488a46f760ea1c0c0c83fcf7abbe2e6baf99d033fd35fd8"}, - {file = "matplotlib-3.5.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dacddf5bfcec60e3f26ec5c0ae3d0274853a258b6c3fc5ef2f06a8eb23e042be"}, - {file = "matplotlib-3.5.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b428076a55fb1c084c76cb93e68006f27d247169f056412607c5c88828d08f88"}, - {file = "matplotlib-3.5.3-cp38-cp38-win32.whl", hash = "sha256:874df7505ba820e0400e7091199decf3ff1fde0583652120c50cd60d5820ca9a"}, - {file = "matplotlib-3.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:b28de401d928890187c589036857a270a032961411934bdac4cf12dde3d43094"}, - {file = "matplotlib-3.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3211ba82b9f1518d346f6309df137b50c3dc4421b4ed4815d1d7eadc617f45a1"}, - {file = "matplotlib-3.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6fe807e8a22620b4cd95cfbc795ba310dc80151d43b037257250faf0bfcd82bc"}, - {file = "matplotlib-3.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5c096363b206a3caf43773abebdbb5a23ea13faef71d701b21a9c27fdcef72f4"}, - {file = "matplotlib-3.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bcdfcb0f976e1bac6721d7d457c17be23cf7501f977b6a38f9d38a3762841f7"}, - {file = "matplotlib-3.5.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1e64ac9be9da6bfff0a732e62116484b93b02a0b4d4b19934fb4f8e7ad26ad6a"}, - {file = "matplotlib-3.5.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:73dd93dc35c85dece610cca8358003bf0760d7986f70b223e2306b4ea6d1406b"}, - {file = "matplotlib-3.5.3-cp39-cp39-win32.whl", hash = "sha256:879c7e5fce4939c6aa04581dfe08d57eb6102a71f2e202e3314d5fbc072fd5a0"}, - {file = "matplotlib-3.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:ab8d26f07fe64f6f6736d635cce7bfd7f625320490ed5bfc347f2cdb4fae0e56"}, - {file = "matplotlib-3.5.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:99482b83ebf4eb6d5fc6813d7aacdefdd480f0d9c0b52dcf9f1cc3b2c4b3361a"}, - {file = "matplotlib-3.5.3-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f814504e459c68118bf2246a530ed953ebd18213dc20e3da524174d84ed010b2"}, - {file = "matplotlib-3.5.3-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57f1b4e69f438a99bb64d7f2c340db1b096b41ebaa515cf61ea72624279220ce"}, - {file = "matplotlib-3.5.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:d2484b350bf3d32cae43f85dcfc89b3ed7bd2bcd781ef351f93eb6fb2cc483f9"}, - {file = "matplotlib-3.5.3.tar.gz", hash = "sha256:339cac48b80ddbc8bfd05daae0a3a73414651a8596904c2a881cfd1edb65f26c"}, -] - -[package.dependencies] -cycler = ">=0.10" -fonttools = ">=4.22.0" -kiwisolver = ">=1.0.1" -numpy = ">=1.17" -packaging = ">=20.0" -pillow = ">=6.2.0" -pyparsing = ">=2.2.1" -python-dateutil = ">=2.7" - -[[package]] -name = "mdurl" -version = "0.1.2" -description = "Markdown URL utilities" -optional = false -python-versions = ">=3.7" -files = [ - {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, - {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, -] - -[[package]] -name = "msgpack" -version = "1.0.5" -description = "MessagePack serializer" -optional = false -python-versions = "*" -files = [ - {file = "msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9"}, - {file = "msgpack-1.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f8d8b3bf1ff2672567d6b5c725a1b347fe838b912772aa8ae2bf70338d5a198"}, - {file = "msgpack-1.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdc793c50be3f01106245a61b739328f7dccc2c648b501e237f0699fe1395b81"}, - {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cb47c21a8a65b165ce29f2bec852790cbc04936f502966768e4aae9fa763cb7"}, - {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e42b9594cc3bf4d838d67d6ed62b9e59e201862a25e9a157019e171fbe672dd3"}, - {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:55b56a24893105dc52c1253649b60f475f36b3aa0fc66115bffafb624d7cb30b"}, - {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1967f6129fc50a43bfe0951c35acbb729be89a55d849fab7686004da85103f1c"}, - {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:20a97bf595a232c3ee6d57ddaadd5453d174a52594bf9c21d10407e2a2d9b3bd"}, - {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d25dd59bbbbb996eacf7be6b4ad082ed7eacc4e8f3d2df1ba43822da9bfa122a"}, - {file = "msgpack-1.0.5-cp310-cp310-win32.whl", hash = "sha256:382b2c77589331f2cb80b67cc058c00f225e19827dbc818d700f61513ab47bea"}, - {file = "msgpack-1.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:4867aa2df9e2a5fa5f76d7d5565d25ec76e84c106b55509e78c1ede0f152659a"}, - {file = "msgpack-1.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9f5ae84c5c8a857ec44dc180a8b0cc08238e021f57abdf51a8182e915e6299f0"}, - {file = "msgpack-1.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e6ca5d5699bcd89ae605c150aee83b5321f2115695e741b99618f4856c50898"}, - {file = "msgpack-1.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5494ea30d517a3576749cad32fa27f7585c65f5f38309c88c6d137877fa28a5a"}, - {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ab2f3331cb1b54165976a9d976cb251a83183631c88076613c6c780f0d6e45a"}, - {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28592e20bbb1620848256ebc105fc420436af59515793ed27d5c77a217477705"}, - {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe5c63197c55bce6385d9aee16c4d0641684628f63ace85f73571e65ad1c1e8d"}, - {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ed40e926fa2f297e8a653c954b732f125ef97bdd4c889f243182299de27e2aa9"}, - {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b2de4c1c0538dcb7010902a2b97f4e00fc4ddf2c8cda9749af0e594d3b7fa3d7"}, - {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bf22a83f973b50f9d38e55c6aade04c41ddda19b00c4ebc558930d78eecc64ed"}, - {file = "msgpack-1.0.5-cp311-cp311-win32.whl", hash = "sha256:c396e2cc213d12ce017b686e0f53497f94f8ba2b24799c25d913d46c08ec422c"}, - {file = "msgpack-1.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:6c4c68d87497f66f96d50142a2b73b97972130d93677ce930718f68828b382e2"}, - {file = "msgpack-1.0.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a2b031c2e9b9af485d5e3c4520f4220d74f4d222a5b8dc8c1a3ab9448ca79c57"}, - {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f837b93669ce4336e24d08286c38761132bc7ab29782727f8557e1eb21b2080"}, - {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1d46dfe3832660f53b13b925d4e0fa1432b00f5f7210eb3ad3bb9a13c6204a6"}, - {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:366c9a7b9057e1547f4ad51d8facad8b406bab69c7d72c0eb6f529cf76d4b85f"}, - {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4c075728a1095efd0634a7dccb06204919a2f67d1893b6aa8e00497258bf926c"}, - {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:f933bbda5a3ee63b8834179096923b094b76f0c7a73c1cfe8f07ad608c58844b"}, - {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:36961b0568c36027c76e2ae3ca1132e35123dcec0706c4b7992683cc26c1320c"}, - {file = "msgpack-1.0.5-cp36-cp36m-win32.whl", hash = "sha256:b5ef2f015b95f912c2fcab19c36814963b5463f1fb9049846994b007962743e9"}, - {file = "msgpack-1.0.5-cp36-cp36m-win_amd64.whl", hash = "sha256:288e32b47e67f7b171f86b030e527e302c91bd3f40fd9033483f2cacc37f327a"}, - {file = "msgpack-1.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:137850656634abddfb88236008339fdaba3178f4751b28f270d2ebe77a563b6c"}, - {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c05a4a96585525916b109bb85f8cb6511db1c6f5b9d9cbcbc940dc6b4be944b"}, - {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56a62ec00b636583e5cb6ad313bbed36bb7ead5fa3a3e38938503142c72cba4f"}, - {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef8108f8dedf204bb7b42994abf93882da1159728a2d4c5e82012edd92c9da9f"}, - {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1835c84d65f46900920b3708f5ba829fb19b1096c1800ad60bae8418652a951d"}, - {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e57916ef1bd0fee4f21c4600e9d1da352d8816b52a599c46460e93a6e9f17086"}, - {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:17358523b85973e5f242ad74aa4712b7ee560715562554aa2134d96e7aa4cbbf"}, - {file = "msgpack-1.0.5-cp37-cp37m-win32.whl", hash = "sha256:cb5aaa8c17760909ec6cb15e744c3ebc2ca8918e727216e79607b7bbce9c8f77"}, - {file = "msgpack-1.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:ab31e908d8424d55601ad7075e471b7d0140d4d3dd3272daf39c5c19d936bd82"}, - {file = "msgpack-1.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b72d0698f86e8d9ddf9442bdedec15b71df3598199ba33322d9711a19f08145c"}, - {file = "msgpack-1.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:379026812e49258016dd84ad79ac8446922234d498058ae1d415f04b522d5b2d"}, - {file = "msgpack-1.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:332360ff25469c346a1c5e47cbe2a725517919892eda5cfaffe6046656f0b7bb"}, - {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:476a8fe8fae289fdf273d6d2a6cb6e35b5a58541693e8f9f019bfe990a51e4ba"}, - {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9985b214f33311df47e274eb788a5893a761d025e2b92c723ba4c63936b69b1"}, - {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48296af57cdb1d885843afd73c4656be5c76c0c6328db3440c9601a98f303d87"}, - {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:addab7e2e1fcc04bd08e4eb631c2a90960c340e40dfc4a5e24d2ff0d5a3b3edb"}, - {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:916723458c25dfb77ff07f4c66aed34e47503b2eb3188b3adbec8d8aa6e00f48"}, - {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:821c7e677cc6acf0fd3f7ac664c98803827ae6de594a9f99563e48c5a2f27eb0"}, - {file = "msgpack-1.0.5-cp38-cp38-win32.whl", hash = "sha256:1c0f7c47f0087ffda62961d425e4407961a7ffd2aa004c81b9c07d9269512f6e"}, - {file = "msgpack-1.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:bae7de2026cbfe3782c8b78b0db9cbfc5455e079f1937cb0ab8d133496ac55e1"}, - {file = "msgpack-1.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:20c784e66b613c7f16f632e7b5e8a1651aa5702463d61394671ba07b2fc9e025"}, - {file = "msgpack-1.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:266fa4202c0eb94d26822d9bfd7af25d1e2c088927fe8de9033d929dd5ba24c5"}, - {file = "msgpack-1.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18334484eafc2b1aa47a6d42427da7fa8f2ab3d60b674120bce7a895a0a85bdd"}, - {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57e1f3528bd95cc44684beda696f74d3aaa8a5e58c816214b9046512240ef437"}, - {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:586d0d636f9a628ddc6a17bfd45aa5b5efaf1606d2b60fa5d87b8986326e933f"}, - {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a740fa0e4087a734455f0fc3abf5e746004c9da72fbd541e9b113013c8dc3282"}, - {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3055b0455e45810820db1f29d900bf39466df96ddca11dfa6d074fa47054376d"}, - {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a61215eac016f391129a013c9e46f3ab308db5f5ec9f25811e811f96962599a8"}, - {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:362d9655cd369b08fda06b6657a303eb7172d5279997abe094512e919cf74b11"}, - {file = "msgpack-1.0.5-cp39-cp39-win32.whl", hash = "sha256:ac9dd47af78cae935901a9a500104e2dea2e253207c924cc95de149606dc43cc"}, - {file = "msgpack-1.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:06f5174b5f8ed0ed919da0e62cbd4ffde676a374aba4020034da05fab67b9164"}, - {file = "msgpack-1.0.5.tar.gz", hash = "sha256:c075544284eadc5cddc70f4757331d99dcbc16b2bbd4849d15f8aae4cf36d31c"}, -] - -[[package]] -name = "mypy" -version = "0.991" -description = "Optional static typing for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "mypy-0.991-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7d17e0a9707d0772f4a7b878f04b4fd11f6f5bcb9b3813975a9b13c9332153ab"}, - {file = "mypy-0.991-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0714258640194d75677e86c786e80ccf294972cc76885d3ebbb560f11db0003d"}, - {file = "mypy-0.991-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c8f3be99e8a8bd403caa8c03be619544bc2c77a7093685dcf308c6b109426c6"}, - {file = "mypy-0.991-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc9ec663ed6c8f15f4ae9d3c04c989b744436c16d26580eaa760ae9dd5d662eb"}, - {file = "mypy-0.991-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4307270436fd7694b41f913eb09210faff27ea4979ecbcd849e57d2da2f65305"}, - {file = "mypy-0.991-cp310-cp310-win_amd64.whl", hash = "sha256:901c2c269c616e6cb0998b33d4adbb4a6af0ac4ce5cd078afd7bc95830e62c1c"}, - {file = "mypy-0.991-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d13674f3fb73805ba0c45eb6c0c3053d218aa1f7abead6e446d474529aafc372"}, - {file = "mypy-0.991-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c8cd4fb70e8584ca1ed5805cbc7c017a3d1a29fb450621089ffed3e99d1857f"}, - {file = "mypy-0.991-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:209ee89fbb0deed518605edddd234af80506aec932ad28d73c08f1400ef80a33"}, - {file = "mypy-0.991-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37bd02ebf9d10e05b00d71302d2c2e6ca333e6c2a8584a98c00e038db8121f05"}, - {file = "mypy-0.991-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:26efb2fcc6b67e4d5a55561f39176821d2adf88f2745ddc72751b7890f3194ad"}, - {file = "mypy-0.991-cp311-cp311-win_amd64.whl", hash = "sha256:3a700330b567114b673cf8ee7388e949f843b356a73b5ab22dd7cff4742a5297"}, - {file = "mypy-0.991-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1f7d1a520373e2272b10796c3ff721ea1a0712288cafaa95931e66aa15798813"}, - {file = "mypy-0.991-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:641411733b127c3e0dab94c45af15fea99e4468f99ac88b39efb1ad677da5711"}, - {file = "mypy-0.991-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3d80e36b7d7a9259b740be6d8d906221789b0d836201af4234093cae89ced0cd"}, - {file = "mypy-0.991-cp37-cp37m-win_amd64.whl", hash = "sha256:e62ebaad93be3ad1a828a11e90f0e76f15449371ffeecca4a0a0b9adc99abcef"}, - {file = "mypy-0.991-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b86ce2c1866a748c0f6faca5232059f881cda6dda2a893b9a8373353cfe3715a"}, - {file = "mypy-0.991-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac6e503823143464538efda0e8e356d871557ef60ccd38f8824a4257acc18d93"}, - {file = "mypy-0.991-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0cca5adf694af539aeaa6ac633a7afe9bbd760df9d31be55ab780b77ab5ae8bf"}, - {file = "mypy-0.991-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a12c56bf73cdab116df96e4ff39610b92a348cc99a1307e1da3c3768bbb5b135"}, - {file = "mypy-0.991-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:652b651d42f155033a1967739788c436491b577b6a44e4c39fb340d0ee7f0d70"}, - {file = "mypy-0.991-cp38-cp38-win_amd64.whl", hash = "sha256:4175593dc25d9da12f7de8de873a33f9b2b8bdb4e827a7cae952e5b1a342e243"}, - {file = "mypy-0.991-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:98e781cd35c0acf33eb0295e8b9c55cdbef64fcb35f6d3aa2186f289bed6e80d"}, - {file = "mypy-0.991-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6d7464bac72a85cb3491c7e92b5b62f3dcccb8af26826257760a552a5e244aa5"}, - {file = "mypy-0.991-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c9166b3f81a10cdf9b49f2d594b21b31adadb3d5e9db9b834866c3258b695be3"}, - {file = "mypy-0.991-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8472f736a5bfb159a5e36740847808f6f5b659960115ff29c7cecec1741c648"}, - {file = "mypy-0.991-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5e80e758243b97b618cdf22004beb09e8a2de1af481382e4d84bc52152d1c476"}, - {file = "mypy-0.991-cp39-cp39-win_amd64.whl", hash = "sha256:74e259b5c19f70d35fcc1ad3d56499065c601dfe94ff67ae48b85596b9ec1461"}, - {file = "mypy-0.991-py3-none-any.whl", hash = "sha256:de32edc9b0a7e67c2775e574cb061a537660e51210fbf6006b0b36ea695ae9bb"}, - {file = "mypy-0.991.tar.gz", hash = "sha256:3c0165ba8f354a6d9881809ef29f1a9318a236a6d81c690094c5df32107bde06"}, -] - -[package.dependencies] -mypy-extensions = ">=0.4.3" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} -typing-extensions = ">=3.10" - -[package.extras] -dmypy = ["psutil (>=4.0)"] -install-types = ["pip"] -python2 = ["typed-ast (>=1.4.0,<2)"] -reports = ["lxml"] - -[[package]] -name = "mypy-extensions" -version = "1.0.0" -description = "Type system extensions for programs checked with the mypy type checker." -optional = false -python-versions = ">=3.5" -files = [ - {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, - {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, -] - -[[package]] -name = "nodeenv" -version = "1.8.0" -description = "Node.js virtual environment builder" -optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" -files = [ - {file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"}, - {file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"}, -] - -[package.dependencies] -setuptools = "*" - -[[package]] -name = "numpy" -version = "1.21.1" -description = "NumPy is the fundamental package for array computing with Python." -optional = false -python-versions = ">=3.7" -files = [ - {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e"}, - {file = "numpy-1.21.1-cp37-cp37m-win32.whl", hash = "sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172"}, - {file = "numpy-1.21.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8"}, - {file = "numpy-1.21.1-cp38-cp38-win32.whl", hash = "sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd"}, - {file = "numpy-1.21.1-cp38-cp38-win_amd64.whl", hash = "sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a"}, - {file = "numpy-1.21.1-cp39-cp39-win32.whl", hash = "sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2"}, - {file = "numpy-1.21.1-cp39-cp39-win_amd64.whl", hash = "sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33"}, - {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"}, - {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, -] - -[[package]] -name = "nvidia-cublas-cu11" -version = "11.10.3.66" -description = "CUBLAS native runtime libraries" -optional = false -python-versions = ">=3" -files = [ - {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl", hash = "sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded"}, - {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-win_amd64.whl", hash = "sha256:8ac17ba6ade3ed56ab898a036f9ae0756f1e81052a317bf98f8c6d18dc3ae49e"}, -] - -[package.dependencies] -setuptools = "*" -wheel = "*" - -[[package]] -name = "nvidia-cuda-nvrtc-cu11" -version = "11.7.99" -description = "NVRTC native runtime libraries" -optional = false -python-versions = ">=3" -files = [ - {file = "nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:9f1562822ea264b7e34ed5930567e89242d266448e936b85bc97a3370feabb03"}, - {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:f7d9610d9b7c331fa0da2d1b2858a4a8315e6d49765091d28711c8946e7425e7"}, - {file = "nvidia_cuda_nvrtc_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:f2effeb1309bdd1b3854fc9b17eaf997808f8b25968ce0c7070945c4265d64a3"}, -] - -[package.dependencies] -setuptools = "*" -wheel = "*" - -[[package]] -name = "nvidia-cuda-runtime-cu11" -version = "11.7.99" -description = "CUDA Runtime native Libraries" -optional = false -python-versions = ">=3" -files = [ - {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl", hash = "sha256:cc768314ae58d2641f07eac350f40f99dcb35719c4faff4bc458a7cd2b119e31"}, - {file = "nvidia_cuda_runtime_cu11-11.7.99-py3-none-win_amd64.whl", hash = "sha256:bc77fa59a7679310df9d5c70ab13c4e34c64ae2124dd1efd7e5474b71be125c7"}, -] - -[package.dependencies] -setuptools = "*" -wheel = "*" - -[[package]] -name = "nvidia-cudnn-cu11" -version = "8.5.0.96" -description = "cuDNN runtime libraries" -optional = false -python-versions = ">=3" -files = [ - {file = "nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl", hash = "sha256:402f40adfc6f418f9dae9ab402e773cfed9beae52333f6d86ae3107a1b9527e7"}, - {file = "nvidia_cudnn_cu11-8.5.0.96-py3-none-manylinux1_x86_64.whl", hash = "sha256:71f8111eb830879ff2836db3cccf03bbd735df9b0d17cd93761732ac50a8a108"}, -] - -[package.dependencies] -setuptools = "*" -wheel = "*" - -[[package]] -name = "omegaconf" -version = "2.3.0" -description = "A flexible configuration library" -optional = false -python-versions = ">=3.6" -files = [ - {file = "omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b"}, - {file = "omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7"}, -] - -[package.dependencies] -antlr4-python3-runtime = "==4.9.*" -PyYAML = ">=5.1.0" - -[[package]] -name = "opt-einsum" -version = "3.3.0" -description = "Optimizing numpys einsum function" -optional = false -python-versions = ">=3.5" -files = [ - {file = "opt_einsum-3.3.0-py3-none-any.whl", hash = "sha256:2455e59e3947d3c275477df7f5205b30635e266fe6dc300e3d9f9646bfcea147"}, - {file = "opt_einsum-3.3.0.tar.gz", hash = "sha256:59f6475f77bbc37dcf7cd748519c0ec60722e91e63ca114e68821c0c54a46549"}, -] - -[package.dependencies] -numpy = ">=1.7" - -[package.extras] -docs = ["numpydoc", "sphinx (==1.2.3)", "sphinx-rtd-theme", "sphinxcontrib-napoleon"] -tests = ["pytest", "pytest-cov", "pytest-pep8"] - -[[package]] -name = "optax" -version = "0.1.4" -description = "A gradient processing and optimisation library in JAX." -optional = false -python-versions = ">=3.7" -files = [ - {file = "optax-0.1.4-py3-none-any.whl", hash = "sha256:12fcf33bd682f9a162a3deb097f864130c3224d76771af2ba09410de80399a9b"}, - {file = "optax-0.1.4.tar.gz", hash = "sha256:fb7a0550d57a6636164a3de25986a8a19be8ff6431fcdf1225b4e05175810f22"}, -] - -[package.dependencies] -absl-py = ">=0.7.1" -chex = ">=0.1.5" -jax = ">=0.1.55" -jaxlib = ">=0.1.37" -numpy = ">=1.18.0" -typing-extensions = ">=3.10.0" - -[[package]] -name = "orbax" -version = "0.1.0" -description = "Orbax" -optional = false -python-versions = ">=3.7" -files = [ - {file = "orbax-0.1.0-py3-none-any.whl", hash = "sha256:df2b5534cb379e1c72a8829bbb62a2c7ebc0e06567ad98e4b06e6aced5d231b1"}, - {file = "orbax-0.1.0.tar.gz", hash = "sha256:317fae101b504dfea9204d667fd79ad7039395b123356995e13a01332e996595"}, -] - -[package.dependencies] -absl-py = "*" -cached_property = "*" -etils = "*" -flax = "*" -importlib_resources = "*" -jax = "*" -jaxlib = "*" -numpy = "*" -pytest = "*" -pyyaml = "*" -tensorstore = ">=0.1.20" - -[package.extras] -dev = ["pytest-xdist"] - -[[package]] -name = "packaging" -version = "23.1" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, -] - -[[package]] -name = "pillow" -version = "9.5.0" -description = "Python Imaging Library (Fork)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "Pillow-9.5.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:ace6ca218308447b9077c14ea4ef381ba0b67ee78d64046b3f19cf4e1139ad16"}, - {file = "Pillow-9.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d3d403753c9d5adc04d4694d35cf0391f0f3d57c8e0030aac09d7678fa8030aa"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ba1b81ee69573fe7124881762bb4cd2e4b6ed9dd28c9c60a632902fe8db8b38"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe7e1c262d3392afcf5071df9afa574544f28eac825284596ac6db56e6d11062"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f36397bf3f7d7c6a3abdea815ecf6fd14e7fcd4418ab24bae01008d8d8ca15e"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:252a03f1bdddce077eff2354c3861bf437c892fb1832f75ce813ee94347aa9b5"}, - {file = "Pillow-9.5.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:85ec677246533e27770b0de5cf0f9d6e4ec0c212a1f89dfc941b64b21226009d"}, - {file = "Pillow-9.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b416f03d37d27290cb93597335a2f85ed446731200705b22bb927405320de903"}, - {file = "Pillow-9.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1781a624c229cb35a2ac31cc4a77e28cafc8900733a864870c49bfeedacd106a"}, - {file = "Pillow-9.5.0-cp310-cp310-win32.whl", hash = "sha256:8507eda3cd0608a1f94f58c64817e83ec12fa93a9436938b191b80d9e4c0fc44"}, - {file = "Pillow-9.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:d3c6b54e304c60c4181da1c9dadf83e4a54fd266a99c70ba646a9baa626819eb"}, - {file = "Pillow-9.5.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:7ec6f6ce99dab90b52da21cf0dc519e21095e332ff3b399a357c187b1a5eee32"}, - {file = "Pillow-9.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:560737e70cb9c6255d6dcba3de6578a9e2ec4b573659943a5e7e4af13f298f5c"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96e88745a55b88a7c64fa49bceff363a1a27d9a64e04019c2281049444a571e3"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d9c206c29b46cfd343ea7cdfe1232443072bbb270d6a46f59c259460db76779a"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cfcc2c53c06f2ccb8976fb5c71d448bdd0a07d26d8e07e321c103416444c7ad1"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:a0f9bb6c80e6efcde93ffc51256d5cfb2155ff8f78292f074f60f9e70b942d99"}, - {file = "Pillow-9.5.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:8d935f924bbab8f0a9a28404422da8af4904e36d5c33fc6f677e4c4485515625"}, - {file = "Pillow-9.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fed1e1cf6a42577953abbe8e6cf2fe2f566daebde7c34724ec8803c4c0cda579"}, - {file = "Pillow-9.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c1170d6b195555644f0616fd6ed929dfcf6333b8675fcca044ae5ab110ded296"}, - {file = "Pillow-9.5.0-cp311-cp311-win32.whl", hash = "sha256:54f7102ad31a3de5666827526e248c3530b3a33539dbda27c6843d19d72644ec"}, - {file = "Pillow-9.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:cfa4561277f677ecf651e2b22dc43e8f5368b74a25a8f7d1d4a3a243e573f2d4"}, - {file = "Pillow-9.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:965e4a05ef364e7b973dd17fc765f42233415974d773e82144c9bbaaaea5d089"}, - {file = "Pillow-9.5.0-cp312-cp312-win32.whl", hash = "sha256:22baf0c3cf0c7f26e82d6e1adf118027afb325e703922c8dfc1d5d0156bb2eeb"}, - {file = "Pillow-9.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:432b975c009cf649420615388561c0ce7cc31ce9b2e374db659ee4f7d57a1f8b"}, - {file = "Pillow-9.5.0-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:5d4ebf8e1db4441a55c509c4baa7a0587a0210f7cd25fcfe74dbbce7a4bd1906"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:375f6e5ee9620a271acb6820b3d1e94ffa8e741c0601db4c0c4d3cb0a9c224bf"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:99eb6cafb6ba90e436684e08dad8be1637efb71c4f2180ee6b8f940739406e78"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dfaaf10b6172697b9bceb9a3bd7b951819d1ca339a5ef294d1f1ac6d7f63270"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:763782b2e03e45e2c77d7779875f4432e25121ef002a41829d8868700d119392"}, - {file = "Pillow-9.5.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:35f6e77122a0c0762268216315bf239cf52b88865bba522999dc38f1c52b9b47"}, - {file = "Pillow-9.5.0-cp37-cp37m-win32.whl", hash = "sha256:aca1c196f407ec7cf04dcbb15d19a43c507a81f7ffc45b690899d6a76ac9fda7"}, - {file = "Pillow-9.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322724c0032af6692456cd6ed554bb85f8149214d97398bb80613b04e33769f6"}, - {file = "Pillow-9.5.0-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:a0aa9417994d91301056f3d0038af1199eb7adc86e646a36b9e050b06f526597"}, - {file = "Pillow-9.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f8286396b351785801a976b1e85ea88e937712ee2c3ac653710a4a57a8da5d9c"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c830a02caeb789633863b466b9de10c015bded434deb3ec87c768e53752ad22a"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fbd359831c1657d69bb81f0db962905ee05e5e9451913b18b831febfe0519082"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8fc330c3370a81bbf3f88557097d1ea26cd8b019d6433aa59f71195f5ddebbf"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:7002d0797a3e4193c7cdee3198d7c14f92c0836d6b4a3f3046a64bd1ce8df2bf"}, - {file = "Pillow-9.5.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:229e2c79c00e85989a34b5981a2b67aa079fd08c903f0aaead522a1d68d79e51"}, - {file = "Pillow-9.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9adf58f5d64e474bed00d69bcd86ec4bcaa4123bfa70a65ce72e424bfb88ed96"}, - {file = "Pillow-9.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:662da1f3f89a302cc22faa9f14a262c2e3951f9dbc9617609a47521c69dd9f8f"}, - {file = "Pillow-9.5.0-cp38-cp38-win32.whl", hash = "sha256:6608ff3bf781eee0cd14d0901a2b9cc3d3834516532e3bd673a0a204dc8615fc"}, - {file = "Pillow-9.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:e49eb4e95ff6fd7c0c402508894b1ef0e01b99a44320ba7d8ecbabefddcc5569"}, - {file = "Pillow-9.5.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:482877592e927fd263028c105b36272398e3e1be3269efda09f6ba21fd83ec66"}, - {file = "Pillow-9.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3ded42b9ad70e5f1754fb7c2e2d6465a9c842e41d178f262e08b8c85ed8a1d8e"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c446d2245ba29820d405315083d55299a796695d747efceb5717a8b450324115"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aca1152d93dcc27dc55395604dcfc55bed5f25ef4c98716a928bacba90d33a3"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:608488bdcbdb4ba7837461442b90ea6f3079397ddc968c31265c1e056964f1ef"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:60037a8db8750e474af7ffc9faa9b5859e6c6d0a50e55c45576bf28be7419705"}, - {file = "Pillow-9.5.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:07999f5834bdc404c442146942a2ecadd1cb6292f5229f4ed3b31e0a108746b1"}, - {file = "Pillow-9.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a127ae76092974abfbfa38ca2d12cbeddcdeac0fb71f9627cc1135bedaf9d51a"}, - {file = "Pillow-9.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:489f8389261e5ed43ac8ff7b453162af39c3e8abd730af8363587ba64bb2e865"}, - {file = "Pillow-9.5.0-cp39-cp39-win32.whl", hash = "sha256:9b1af95c3a967bf1da94f253e56b6286b50af23392a886720f563c547e48e964"}, - {file = "Pillow-9.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:77165c4a5e7d5a284f10a6efaa39a0ae8ba839da344f20b111d62cc932fa4e5d"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:833b86a98e0ede388fa29363159c9b1a294b0905b5128baf01db683672f230f5"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aaf305d6d40bd9632198c766fb64f0c1a83ca5b667f16c1e79e1661ab5060140"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0852ddb76d85f127c135b6dd1f0bb88dbb9ee990d2cd9aa9e28526c93e794fba"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:91ec6fe47b5eb5a9968c79ad9ed78c342b1f97a091677ba0e012701add857829"}, - {file = "Pillow-9.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cb841572862f629b99725ebaec3287fc6d275be9b14443ea746c1dd325053cbd"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:c380b27d041209b849ed246b111b7c166ba36d7933ec6e41175fd15ab9eb1572"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c9af5a3b406a50e313467e3565fc99929717f780164fe6fbb7704edba0cebbe"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5671583eab84af046a397d6d0ba25343c00cd50bce03787948e0fff01d4fd9b1"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:84a6f19ce086c1bf894644b43cd129702f781ba5751ca8572f08aa40ef0ab7b7"}, - {file = "Pillow-9.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1e7723bd90ef94eda669a3c2c19d549874dd5badaeefabefd26053304abe5799"}, - {file = "Pillow-9.5.0.tar.gz", hash = "sha256:bf548479d336726d7a0eceb6e767e179fbde37833ae42794602631a070d630f1"}, -] - -[package.extras] -docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] -tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] - -[[package]] -name = "pluggy" -version = "1.0.0" -description = "plugin and hook calling mechanisms for python" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] - -[package.dependencies] -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "pydantic" -version = "1.10.8" -description = "Data validation and settings management using python type hints" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pydantic-1.10.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1243d28e9b05003a89d72e7915fdb26ffd1d39bdd39b00b7dbe4afae4b557f9d"}, - {file = "pydantic-1.10.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0ab53b609c11dfc0c060d94335993cc2b95b2150e25583bec37a49b2d6c6c3f"}, - {file = "pydantic-1.10.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9613fadad06b4f3bc5db2653ce2f22e0de84a7c6c293909b48f6ed37b83c61f"}, - {file = "pydantic-1.10.8-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df7800cb1984d8f6e249351139667a8c50a379009271ee6236138a22a0c0f319"}, - {file = "pydantic-1.10.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0c6fafa0965b539d7aab0a673a046466d23b86e4b0e8019d25fd53f4df62c277"}, - {file = "pydantic-1.10.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e82d4566fcd527eae8b244fa952d99f2ca3172b7e97add0b43e2d97ee77f81ab"}, - {file = "pydantic-1.10.8-cp310-cp310-win_amd64.whl", hash = "sha256:ab523c31e22943713d80d8d342d23b6f6ac4b792a1e54064a8d0cf78fd64e800"}, - {file = "pydantic-1.10.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:666bdf6066bf6dbc107b30d034615d2627e2121506c555f73f90b54a463d1f33"}, - {file = "pydantic-1.10.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:35db5301b82e8661fa9c505c800d0990bc14e9f36f98932bb1d248c0ac5cada5"}, - {file = "pydantic-1.10.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f90c1e29f447557e9e26afb1c4dbf8768a10cc676e3781b6a577841ade126b85"}, - {file = "pydantic-1.10.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93e766b4a8226e0708ef243e843105bf124e21331694367f95f4e3b4a92bbb3f"}, - {file = "pydantic-1.10.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88f195f582851e8db960b4a94c3e3ad25692c1c1539e2552f3df7a9e972ef60e"}, - {file = "pydantic-1.10.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:34d327c81e68a1ecb52fe9c8d50c8a9b3e90d3c8ad991bfc8f953fb477d42fb4"}, - {file = "pydantic-1.10.8-cp311-cp311-win_amd64.whl", hash = "sha256:d532bf00f381bd6bc62cabc7d1372096b75a33bc197a312b03f5838b4fb84edd"}, - {file = "pydantic-1.10.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7d5b8641c24886d764a74ec541d2fc2c7fb19f6da2a4001e6d580ba4a38f7878"}, - {file = "pydantic-1.10.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b1f6cb446470b7ddf86c2e57cd119a24959af2b01e552f60705910663af09a4"}, - {file = "pydantic-1.10.8-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c33b60054b2136aef8cf190cd4c52a3daa20b2263917c49adad20eaf381e823b"}, - {file = "pydantic-1.10.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1952526ba40b220b912cdc43c1c32bcf4a58e3f192fa313ee665916b26befb68"}, - {file = "pydantic-1.10.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bb14388ec45a7a0dc429e87def6396f9e73c8c77818c927b6a60706603d5f2ea"}, - {file = "pydantic-1.10.8-cp37-cp37m-win_amd64.whl", hash = "sha256:16f8c3e33af1e9bb16c7a91fc7d5fa9fe27298e9f299cff6cb744d89d573d62c"}, - {file = "pydantic-1.10.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1ced8375969673929809d7f36ad322934c35de4af3b5e5b09ec967c21f9f7887"}, - {file = "pydantic-1.10.8-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:93e6bcfccbd831894a6a434b0aeb1947f9e70b7468f274154d03d71fabb1d7c6"}, - {file = "pydantic-1.10.8-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:191ba419b605f897ede9892f6c56fb182f40a15d309ef0142212200a10af4c18"}, - {file = "pydantic-1.10.8-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:052d8654cb65174d6f9490cc9b9a200083a82cf5c3c5d3985db765757eb3b375"}, - {file = "pydantic-1.10.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ceb6a23bf1ba4b837d0cfe378329ad3f351b5897c8d4914ce95b85fba96da5a1"}, - {file = "pydantic-1.10.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f2e754d5566f050954727c77f094e01793bcb5725b663bf628fa6743a5a9108"}, - {file = "pydantic-1.10.8-cp38-cp38-win_amd64.whl", hash = "sha256:6a82d6cda82258efca32b40040228ecf43a548671cb174a1e81477195ed3ed56"}, - {file = "pydantic-1.10.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e59417ba8a17265e632af99cc5f35ec309de5980c440c255ab1ca3ae96a3e0e"}, - {file = "pydantic-1.10.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84d80219c3f8d4cad44575e18404099c76851bc924ce5ab1c4c8bb5e2a2227d0"}, - {file = "pydantic-1.10.8-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e4148e635994d57d834be1182a44bdb07dd867fa3c2d1b37002000646cc5459"}, - {file = "pydantic-1.10.8-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12f7b0bf8553e310e530e9f3a2f5734c68699f42218bf3568ef49cd9b0e44df4"}, - {file = "pydantic-1.10.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:42aa0c4b5c3025483240a25b09f3c09a189481ddda2ea3a831a9d25f444e03c1"}, - {file = "pydantic-1.10.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17aef11cc1b997f9d574b91909fed40761e13fac438d72b81f902226a69dac01"}, - {file = "pydantic-1.10.8-cp39-cp39-win_amd64.whl", hash = "sha256:66a703d1983c675a6e0fed8953b0971c44dba48a929a2000a493c3772eb61a5a"}, - {file = "pydantic-1.10.8-py3-none-any.whl", hash = "sha256:7456eb22ed9aaa24ff3e7b4757da20d9e5ce2a81018c1b3ebd81a0b88a18f3b2"}, - {file = "pydantic-1.10.8.tar.gz", hash = "sha256:1410275520dfa70effadf4c21811d755e7ef9bb1f1d077a21958153a92c8d9ca"}, -] - -[package.dependencies] -typing-extensions = ">=4.2.0" - -[package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] - -[[package]] -name = "pygments" -version = "2.15.1" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.7" -files = [ - {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, - {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, -] - -[package.extras] -plugins = ["importlib-metadata"] - -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -optional = false -python-versions = ">=3.6.8" -files = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - -[[package]] -name = "pyright" -version = "1.1.310" -description = "Command line wrapper for pyright" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pyright-1.1.310-py3-none-any.whl", hash = "sha256:55995ac76bf56cb7a44193b7b1ffafc573abab1f1dbc9f62d327f2a1768b3bda"}, - {file = "pyright-1.1.310.tar.gz", hash = "sha256:9e95335a678db2717eaa0c867d61f9399e916289f4a9f47d993e0df74e7d7391"}, -] - -[package.dependencies] -nodeenv = ">=1.6.0" -typing-extensions = {version = ">=3.7", markers = "python_version < \"3.8\""} - -[package.extras] -all = ["twine (>=3.4.1)"] -dev = ["twine (>=3.4.1)"] - -[[package]] -name = "pytest" -version = "7.3.1" -description = "pytest: simple powerful testing with Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, - {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} - -[package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] - -[[package]] -name = "pytest-cov" -version = "3.0.0" -description = "Pytest plugin for measuring coverage." -optional = false -python-versions = ">=3.6" -files = [ - {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, - {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, -] - -[package.dependencies] -coverage = {version = ">=5.2.1", extras = ["toml"]} -pytest = ">=4.6" - -[package.extras] -testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.6" -files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] - -[[package]] -name = "rich" -version = "13.3.5" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "rich-13.3.5-py3-none-any.whl", hash = "sha256:69cdf53799e63f38b95b9bf9c875f8c90e78dd62b2f00c13a911c7a3b9fa4704"}, - {file = "rich-13.3.5.tar.gz", hash = "sha256:2d11b9b8dd03868f09b4fffadc84a6a8cda574e40dc90821bd845720ebb8e89c"}, -] - -[package.dependencies] -markdown-it-py = ">=2.2.0,<3.0.0" -pygments = ">=2.13.0,<3.0.0" -typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} - -[package.extras] -jupyter = ["ipywidgets (>=7.5.1,<9)"] - -[[package]] -name = "scipy" -version = "1.6.1" -description = "SciPy: Scientific Library for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "scipy-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a15a1f3fc0abff33e792d6049161b7795909b40b97c6cc2934ed54384017ab76"}, - {file = "scipy-1.6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:e79570979ccdc3d165456dd62041d9556fb9733b86b4b6d818af7a0afc15f092"}, - {file = "scipy-1.6.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a423533c55fec61456dedee7b6ee7dce0bb6bfa395424ea374d25afa262be261"}, - {file = "scipy-1.6.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:33d6b7df40d197bdd3049d64e8e680227151673465e5d85723b3b8f6b15a6ced"}, - {file = "scipy-1.6.1-cp37-cp37m-win32.whl", hash = "sha256:6725e3fbb47da428794f243864f2297462e9ee448297c93ed1dcbc44335feb78"}, - {file = "scipy-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:5fa9c6530b1661f1370bcd332a1e62ca7881785cc0f80c0d559b636567fab63c"}, - {file = "scipy-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bd50daf727f7c195e26f27467c85ce653d41df4358a25b32434a50d8870fc519"}, - {file = "scipy-1.6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f46dd15335e8a320b0fb4685f58b7471702234cba8bb3442b69a3e1dc329c345"}, - {file = "scipy-1.6.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0e5b0ccf63155d90da576edd2768b66fb276446c371b73841e3503be1d63fb5d"}, - {file = "scipy-1.6.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2481efbb3740977e3c831edfd0bd9867be26387cacf24eb5e366a6a374d3d00d"}, - {file = "scipy-1.6.1-cp38-cp38-win32.whl", hash = "sha256:68cb4c424112cd4be886b4d979c5497fba190714085f46b8ae67a5e4416c32b4"}, - {file = "scipy-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:5f331eeed0297232d2e6eea51b54e8278ed8bb10b099f69c44e2558c090d06bf"}, - {file = "scipy-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0c8a51d33556bf70367452d4d601d1742c0e806cd0194785914daf19775f0e67"}, - {file = "scipy-1.6.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:83bf7c16245c15bc58ee76c5418e46ea1811edcc2e2b03041b804e46084ab627"}, - {file = "scipy-1.6.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:794e768cc5f779736593046c9714e0f3a5940bc6dcc1dba885ad64cbfb28e9f0"}, - {file = "scipy-1.6.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5da5471aed911fe7e52b86bf9ea32fb55ae93e2f0fac66c32e58897cfb02fa07"}, - {file = "scipy-1.6.1-cp39-cp39-win32.whl", hash = "sha256:8e403a337749ed40af60e537cc4d4c03febddcc56cd26e774c9b1b600a70d3e4"}, - {file = "scipy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a5193a098ae9f29af283dcf0041f762601faf2e595c0db1da929875b7570353f"}, - {file = "scipy-1.6.1.tar.gz", hash = "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11"}, -] - -[package.dependencies] -numpy = ">=1.16.5" - -[[package]] -name = "setuptools" -version = "67.8.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "setuptools-67.8.0-py3-none-any.whl", hash = "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f"}, - {file = "setuptools-67.8.0.tar.gz", hash = "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102"}, -] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - -[[package]] -name = "shtab" -version = "1.6.1" -description = "Automagic shell tab completion for Python CLI applications" -optional = false -python-versions = ">=3.7" -files = [ - {file = "shtab-1.6.1-py3-none-any.whl", hash = "sha256:db2c41d81a61c7ecefd1dad8212bdc4b667e7449b8172aa6445f557f901810c4"}, - {file = "shtab-1.6.1.tar.gz", hash = "sha256:decc78082c3ffb518c1dfd3a8da99653a2d47e58e3104197bce8ae6507dad78b"}, -] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] - -[[package]] -name = "tensorstore" -version = "0.1.28" -description = "Read and write large, multi-dimensional arrays" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tensorstore-0.1.28-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:deb32f1e74ab7b836ecec02a759a558fc57522a8dea0db4f19a01a78e93abab5"}, - {file = "tensorstore-0.1.28-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bd8615308ad3e29111238dfdd3aca1f3fb79b55c3dac1c04c96c8a319985addf"}, - {file = "tensorstore-0.1.28-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee16297a9493ed0d6d1000f216e79d29a0b80e7e97646e8520afd44138165839"}, - {file = "tensorstore-0.1.28-cp310-cp310-win_amd64.whl", hash = "sha256:b6f7e0c7455d9e164e6143714519e7c96cc4166b61ed7d268a8ab84a77e0d593"}, - {file = "tensorstore-0.1.28-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:1c53e1e0e7996b8ba41854135d748c2bc8802b94a79122b56448aeb1f77efd9f"}, - {file = "tensorstore-0.1.28-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bd122a44db81369808c84b7739ad2d8453f89c548c5d61d62f2b1f3331dbe9f"}, - {file = "tensorstore-0.1.28-cp37-cp37m-win_amd64.whl", hash = "sha256:197159da3ee97c08d8769fa151ff167328108bf07a13886f688b21f64abfb7d6"}, - {file = "tensorstore-0.1.28-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:e480da4f95f7c95302c4b3e2a7fc32b8cfe1aa51742d72f84a335e49462a2465"}, - {file = "tensorstore-0.1.28-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7948ead0d5a55303ffa97a35659aa56a20c60bd89d9487643f51ac8d2e19b52c"}, - {file = "tensorstore-0.1.28-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:818aeb015b249069a686fb20f0c0b71663410ac6411cee88df18b8e05b567cbc"}, - {file = "tensorstore-0.1.28-cp38-cp38-win_amd64.whl", hash = "sha256:6719eaba4ec2692d890c3c67c389fcae073d40d9e8c27e2dabd27fc9fd745e5c"}, - {file = "tensorstore-0.1.28-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:d7c86f63a6d4d7a84e3201ba9746ca51015612d375d77b9302eb93e9b88bc7aa"}, - {file = "tensorstore-0.1.28-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dc82cbb77255fd2f7aa191f1e030f13276bf7272ea1027fcc6900537b30bfbcb"}, - {file = "tensorstore-0.1.28-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4de09e3fae30f8a9ab7648600c78e109ddd1d14af4280e055439fdd52a6dca9"}, - {file = "tensorstore-0.1.28-cp39-cp39-win_amd64.whl", hash = "sha256:f9270586401ee60ff79a4cd54c62ab5b06831d69b561df1fb9ebbeade4d4929c"}, - {file = "tensorstore-0.1.28.tar.gz", hash = "sha256:cd8d8185136632c58edcd7cf4c43d301bf8ad61a197c632cb495d7d33b7be04e"}, -] - -[package.dependencies] -numpy = ">=1.16.0" - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] - -[[package]] -name = "toolz" -version = "0.12.0" -description = "List processing tools and functional utilities" -optional = false -python-versions = ">=3.5" -files = [ - {file = "toolz-0.12.0-py3-none-any.whl", hash = "sha256:2059bd4148deb1884bb0eb770a3cde70e7f954cfbbdc2285f1f2de01fd21eb6f"}, - {file = "toolz-0.12.0.tar.gz", hash = "sha256:88c570861c440ee3f2f6037c4654613228ff40c93a6c25e0eba70d17282c6194"}, -] - -[[package]] -name = "torch" -version = "1.13.1" -description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "torch-1.13.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:fd12043868a34a8da7d490bf6db66991108b00ffbeecb034228bfcbbd4197143"}, - {file = "torch-1.13.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:d9fe785d375f2e26a5d5eba5de91f89e6a3be5d11efb497e76705fdf93fa3c2e"}, - {file = "torch-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:98124598cdff4c287dbf50f53fb455f0c1e3a88022b39648102957f3445e9b76"}, - {file = "torch-1.13.1-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:393a6273c832e047581063fb74335ff50b4c566217019cc6ace318cd79eb0566"}, - {file = "torch-1.13.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:0122806b111b949d21fa1a5f9764d1fd2fcc4a47cb7f8ff914204fd4fc752ed5"}, - {file = "torch-1.13.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:22128502fd8f5b25ac1cd849ecb64a418382ae81dd4ce2b5cebaa09ab15b0d9b"}, - {file = "torch-1.13.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:76024be052b659ac1304ab8475ab03ea0a12124c3e7626282c9c86798ac7bc11"}, - {file = "torch-1.13.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:ea8dda84d796094eb8709df0fcd6b56dc20b58fdd6bc4e8d7109930dafc8e419"}, - {file = "torch-1.13.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2ee7b81e9c457252bddd7d3da66fb1f619a5d12c24d7074de91c4ddafb832c93"}, - {file = "torch-1.13.1-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:0d9b8061048cfb78e675b9d2ea8503bfe30db43d583599ae8626b1263a0c1380"}, - {file = "torch-1.13.1-cp37-none-macosx_11_0_arm64.whl", hash = "sha256:f402ca80b66e9fbd661ed4287d7553f7f3899d9ab54bf5c67faada1555abde28"}, - {file = "torch-1.13.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:727dbf00e2cf858052364c0e2a496684b9cb5aa01dc8a8bc8bbb7c54502bdcdd"}, - {file = "torch-1.13.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:df8434b0695e9ceb8cc70650afc1310d8ba949e6db2a0525ddd9c3b2b181e5fe"}, - {file = "torch-1.13.1-cp38-cp38-win_amd64.whl", hash = "sha256:5e1e722a41f52a3f26f0c4fcec227e02c6c42f7c094f32e49d4beef7d1e213ea"}, - {file = "torch-1.13.1-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:33e67eea526e0bbb9151263e65417a9ef2d8fa53cbe628e87310060c9dcfa312"}, - {file = "torch-1.13.1-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:eeeb204d30fd40af6a2d80879b46a7efbe3cf43cdbeb8838dd4f3d126cc90b2b"}, - {file = "torch-1.13.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:50ff5e76d70074f6653d191fe4f6a42fdbe0cf942fbe2a3af0b75eaa414ac038"}, - {file = "torch-1.13.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:2c3581a3fd81eb1f0f22997cddffea569fea53bafa372b2c0471db373b26aafc"}, - {file = "torch-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:0aa46f0ac95050c604bcf9ef71da9f1172e5037fdf2ebe051962d47b123848e7"}, - {file = "torch-1.13.1-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:6930791efa8757cb6974af73d4996b6b50c592882a324b8fb0589c6a9ba2ddaf"}, - {file = "torch-1.13.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:e0df902a7c7dd6c795698532ee5970ce898672625635d885eade9976e5a04949"}, -] - -[package.dependencies] -nvidia-cublas-cu11 = {version = "11.10.3.66", markers = "platform_system == \"Linux\""} -nvidia-cuda-nvrtc-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\""} -nvidia-cuda-runtime-cu11 = {version = "11.7.99", markers = "platform_system == \"Linux\""} -nvidia-cudnn-cu11 = {version = "8.5.0.96", markers = "platform_system == \"Linux\""} -typing-extensions = "*" - -[package.extras] -opt-einsum = ["opt-einsum (>=3.3)"] - -[[package]] -name = "typed-ast" -version = "1.5.4" -description = "a fork of Python 2 and 3 ast modules with type comment support" -optional = false -python-versions = ">=3.6" -files = [ - {file = "typed_ast-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4"}, - {file = "typed_ast-1.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62"}, - {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac"}, - {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe"}, - {file = "typed_ast-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72"}, - {file = "typed_ast-1.5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec"}, - {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47"}, - {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6"}, - {file = "typed_ast-1.5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1"}, - {file = "typed_ast-1.5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6"}, - {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66"}, - {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c"}, - {file = "typed_ast-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2"}, - {file = "typed_ast-1.5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d"}, - {file = "typed_ast-1.5.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f"}, - {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc"}, - {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6"}, - {file = "typed_ast-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e"}, - {file = "typed_ast-1.5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35"}, - {file = "typed_ast-1.5.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97"}, - {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3"}, - {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72"}, - {file = "typed_ast-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1"}, - {file = "typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"}, -] - -[[package]] -name = "typing-extensions" -version = "4.6.2" -description = "Backported and Experimental Type Hints for Python 3.7+" -optional = false -python-versions = ">=3.7" -files = [ - {file = "typing_extensions-4.6.2-py3-none-any.whl", hash = "sha256:3a8b36f13dd5fdc5d1b16fe317f5668545de77fa0b8e02006381fd49d731ab98"}, - {file = "typing_extensions-4.6.2.tar.gz", hash = "sha256:06006244c70ac8ee83fa8282cb188f697b8db25bc8b4df07be1873c43897060c"}, -] - -[[package]] -name = "wheel" -version = "0.40.0" -description = "A built-package format for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "wheel-0.40.0-py3-none-any.whl", hash = "sha256:d236b20e7cb522daf2390fa84c55eea81c5c30190f90f29ae2ca1ad8355bf247"}, - {file = "wheel-0.40.0.tar.gz", hash = "sha256:cd1196f3faee2b31968d626e1731c94f99cbdb67cf5a46e4f5656cbee7738873"}, -] - -[package.extras] -test = ["pytest (>=6.0.0)"] - -[[package]] -name = "zipp" -version = "3.15.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.7" -files = [ - {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, - {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, -] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[metadata] -lock-version = "2.0" -python-versions = "^3.7" -content-hash = "48a6bbab3cc4aaac5dea9aaf813de0b442ba24f7d81261dc416193bb9fb33453" diff --git a/pyproject.toml b/pyproject.toml index dbc7a6965..42361bacf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,41 +1,63 @@ -[tool.poetry] +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] name = "tyro" +authors = [ + {name = "brentyi", email = "brentyi@berkeley.edu"}, +] version = "0.5.3" description = "Strongly typed, zero-effort CLI interfaces" -authors = ["brentyi "] -include = ["./tyro/**/*"] readme = "README.md" -repository = "https://github.com/brentyi/tyro" -homepage = "https://github.com/brentyi/tyro" -documentation = "https://brentyi.github.io/tyro/" - -[tool.poetry.dependencies] -python = "^3.7" -docstring-parser = "^0.14.1" -typing-extensions = "^4.3.0" -PyYAML = "^6.0" -"backports.cached-property" = { version = "^1.0.2", python = "~3.7" } -colorama = {version = "^0.4.0", platform = "win32"} -frozendict = "^2.3.4" -rich = ">=11.1.0" -shtab = "^1.5.6" - -[tool.poetry.group.dev.dependencies] -pytest = "^7.1.2" -pytest-cov = "^3.0.0" -omegaconf = "^2.2.2" -attrs = "^21.4.0" -torch = "^1.10.0" -pyright = "^1.1.264" -mypy = "^0.991" -numpy = ">=1.20.0" -flax = "^0.6.0" -pydantic = "^1.10.2" -coverage = {extras = ["toml"], version = "^6.5.0"} +license = { text="MIT" } +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent" +] +dependencies = [ + "docstring-parser>=0.14.1", + "typing-extensions>=4.3.0", + "PyYAML>=6.0", + "backports.cached-property>=1.0.2; python_version<'3.8'", + "colorama>=0.4.0; platform_system=='Windows'", + "frozendict>=2.3.4", + "rich>=11.1.0", + "shtab>=1.5.6" +] -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" +[project.optional-dependencies] +dev = [ + "pytest>=7.1.2", + "pytest-cov>=3.0.0", + "omegaconf>=2.2.2", + "attrs>=21.4.0", + "torch>=1.10.0", + "pyright>=1.1.264", + "mypy>=0.991", + "numpy>=1.20.0", + "flax>=0.6.0", + "pydantic>=1.10.2", + "coverage[toml]>=6.5.0" +] + +[project.urls] +"GitHub" = "https://github.com/brentyi/tyro" + +[tool.setuptools.packages.find] +exclude = [ + "tests*" +] + +[tool.setuptools.package-data] +tyro = ["py.typed"] [tool.isort] profile = "black" diff --git a/tests/test_conf.py b/tests/test_conf.py index fdf685c63..2e93d9a9a 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -150,7 +150,7 @@ class DefaultInstanceSubparser: def test_subparser_in_nested_with_metadata() -> None: - @dataclasses.dataclass + @dataclasses.dataclass(frozen=True) class A: a: int @@ -202,7 +202,7 @@ class Parent: def test_subparser_in_nested_with_metadata_generic() -> None: - @dataclasses.dataclass + @dataclasses.dataclass(frozen=True) class A: a: int @@ -258,7 +258,7 @@ class Parent: def test_subparser_in_nested_with_metadata_generic_alt() -> None: - @dataclasses.dataclass + @dataclasses.dataclass(frozen=True) class A: a: int diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 7fc3e4a1a..5af75ee2f 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -373,7 +373,7 @@ def test_pculbertson() -> None: # https://github.com/brentyi/tyro/issues/7 from typing import Union - @dataclasses.dataclass + @dataclasses.dataclass(frozen=True) class TypeA: data: int @@ -392,7 +392,7 @@ class Wrapper: def test_annotated() -> None: # https://github.com/brentyi/tyro/issues/7 - @dataclasses.dataclass + @dataclasses.dataclass(frozen=True) class TypeA: data: int diff --git a/tests/test_nested.py b/tests/test_nested.py index 584e14119..f87a6d81e 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -60,7 +60,7 @@ class Nested: def test_nested_default() -> None: - @dataclasses.dataclass + @dataclasses.dataclass(frozen=True) class B: y: int = 1 @@ -869,7 +869,7 @@ class Wrapper: def test_nested_in_subparser_override_with_default() -> None: - @dataclasses.dataclass + @dataclasses.dataclass(frozen=True) class Mnist: binary: bool = False """Set to load binary version of MNIST dataset.""" @@ -888,7 +888,7 @@ class ImageNet: } ) - @dataclasses.dataclass + @dataclasses.dataclass(frozen=True) class DatasetContainer: dataset: Selector = Mnist() # type: ignore diff --git a/tyro/_cli.py b/tyro/_cli.py index af2aa1e06..a5358293c 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -288,7 +288,9 @@ def _cli_impl( dummy_field = cast( dataclasses.Field, dataclasses.field( - default=default if default is not None else dataclasses.MISSING + default_factory=lambda: default + if default is not None + else dataclasses.MISSING ), ) f = dataclasses.make_dataclass( diff --git a/tyro/_fields.py b/tyro/_fields.py index 198a02c9b..bb6c16be6 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -459,21 +459,43 @@ def _field_list_from_pydantic( # Handle pydantic models. field_list = [] - for pd_field in cls.__fields__.values(): # type: ignore - helptext = pd_field.field_info.description - if helptext is None: - helptext = _docstrings.get_field_docstring(cls, pd_field.name) - - field_list.append( - FieldDefinition.make( - name=pd_field.name, - typ=pd_field.outer_type_, - default=( - MISSING_NONPROP if pd_field.required else pd_field.get_default() - ), - helptext=helptext, + pydantic_version = float(getattr(pydantic, "__version__", "1.0")) + if pydantic_version < 2.0: # pragma: no cover + # Pydantic 1.xx. + for pd_field in cls.__fields__.values(): # type: ignore + helptext = pd_field.field_info.description + if helptext is None: + helptext = _docstrings.get_field_docstring(cls, pd_field.name) + + field_list.append( + FieldDefinition.make( + name=pd_field.name, + typ=pd_field.outer_type_, + default=( + MISSING_NONPROP if pd_field.required else pd_field.get_default() + ), + helptext=helptext, + ) + ) + else: + # Pydantic 2.xx. + for name, pd_field in cls.model_fields.items(): # type: ignore + helptext = pd_field.description + if helptext is None: + helptext = _docstrings.get_field_docstring(cls, name) + + field_list.append( + FieldDefinition.make( + name=name, + typ=pd_field.annotation, + default=( + MISSING_NONPROP + if pd_field.is_required() + else pd_field.get_default(call_default_factory=True) + ), + helptext=helptext, + ) ) - ) return field_list From 1626cee88d1f30edddb404abbe94767820a7b581 Mon Sep 17 00:00:00 2001 From: Shenhan Qian Date: Tue, 4 Jul 2023 09:11:32 +0200 Subject: [PATCH 299/491] Support setting Tuple, Dict, and Union to empty in the command line. (#57) * Support setting Tuple and Union to empty in the command line. * Account for nargs change in tests * Fix formatting, type error * Update metavars for variable-length args * Run black --------- Co-authored-by: Brent Yi --- tests/test_collections.py | 47 ++++++++++------------------ tests/test_dcargs.py | 12 +++++++ tests/test_dict_namedtuple.py | 3 +- tests/test_helptext.py | 11 ++++--- tests/test_positional_ignore_py37.py | 3 +- tyro/__init__.py | 3 +- tyro/_calling.py | 11 +++++++ tyro/_instantiators.py | 16 +++++----- tyro/_strings.py | 4 +-- 9 files changed, 60 insertions(+), 50 deletions(-) diff --git a/tests/test_collections.py b/tests/test_collections.py index b63d09968..a20e73c22 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -64,10 +64,9 @@ class A: assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) assert tyro.cli(A, args=[]) == A(x=(1, 2)) + assert tyro.cli(A, args=["--x"]) == A(x=()) with pytest.raises(SystemExit): tyro.cli(A, args=["--x", "1", "2", "3", "4"]) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) def test_positional_tuple_with_literal_and_default() -> None: @@ -116,8 +115,7 @@ class A: x: Tuple[int, ...] assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=["--x"]) == A(x=()) with pytest.raises(SystemExit): tyro.cli(A, args=[]) @@ -130,8 +128,7 @@ class A: assert tyro.cli(A, args=["--x", "True", "True", "False"]) == A( x=(True, True, False) ) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=["--x"]) == A(x=()) with pytest.raises(SystemExit): tyro.cli(A, args=[]) @@ -142,8 +139,7 @@ class A: x: Optional[Tuple[int, ...]] = None assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=["--x"]) == A(x=()) assert tyro.cli(A, args=[]) == A(x=None) @@ -153,8 +149,7 @@ class A: x: Sequence[int] assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) with pytest.raises(SystemExit): tyro.cli(A, args=[]) @@ -165,8 +160,7 @@ class A: x: List[int] assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) with pytest.raises(SystemExit): tyro.cli(A, args=[]) @@ -177,10 +171,9 @@ class A: x: List[Literal[1, 2, 3]] assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) with pytest.raises(SystemExit): tyro.cli(A, args=["--x", "1", "2", "3", "4"]) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): tyro.cli(A, args=[]) @@ -198,10 +191,9 @@ class A: assert tyro.cli(A, args=["--x", "RED", "RED", "BLUE"]) == A( x=[Color.RED, Color.RED, Color.BLUE] ) + assert tyro.cli(A, args=["--x"]) == A(x=[]) with pytest.raises(SystemExit): tyro.cli(A, args=["--x", "RED", "RED", "YELLOW"]) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) with pytest.raises(SystemExit): tyro.cli(A, args=[]) @@ -232,8 +224,7 @@ class A: assert tyro.cli(A, args=["--x", "True", "False", "True"]) == A( x=[True, False, True] ) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) with pytest.raises(SystemExit): tyro.cli(A, args=[]) @@ -244,8 +235,7 @@ class A: x: Set[int] assert tyro.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=["--x"]) == A(set()) with pytest.raises(SystemExit): tyro.cli(A, args=[]) @@ -256,8 +246,7 @@ class A: x: FrozenSet[int] assert tyro.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x=frozenset({1, 2, 3})) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=["--x"]) == A(x=frozenset()) with pytest.raises(SystemExit): tyro.cli(A, args=[]) @@ -270,8 +259,7 @@ class A: assert tyro.cli(A, args=["--x", "1", "2", "3", "3"]) == A( x=collections.deque([1, 2, 3, 3]) ) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=["--x"]) == A(collections.deque()) with pytest.raises(SystemExit): tyro.cli(A, args=[]) @@ -283,8 +271,7 @@ class A: assert tyro.cli(A, args=[]) == A(x={0, 1, 2}) assert tyro.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=["--x"]) == A(x=set()) def test_optional_sequences() -> None: @@ -293,8 +280,8 @@ class A: x: Optional[Sequence[int]] = None assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + assert tyro.cli(A, args=["--x", "None"]) == A(x=None) assert tyro.cli(A, args=[]) == A(x=None) @@ -304,8 +291,8 @@ class A: x: Optional[List[int]] = None assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) - with pytest.raises(SystemExit): - tyro.cli(A, args=["--x"]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + assert tyro.cli(A, args=["--x", "None"]) == A(x=None) assert tyro.cli(A, args=[]) == A(x=None) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 3f6cc9d12..3d21d512b 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -676,3 +676,15 @@ def main(*args: int, **kwargs: float) -> Tuple[Tuple[int, ...], Dict[str, float] assert tyro.cli( main, args="--args 1 2 3 --kwargs learning_rate 1e-4 beta1 0.99".split(" ") ) == ((1, 2, 3), {"learning_rate": 1e-4, "beta1": 0.99}) + + +def test_empty_container() -> None: + @dataclasses.dataclass + class A: + x: Tuple[int, ...] = (1, 2, 3) + y: Union[int, str, List[bool]] = dataclasses.field( + default_factory=lambda: [False, False, True] + ) + + assert tyro.cli(A, args="--x".split(" ")).x == () + assert tyro.cli(A, args="--y".split(" ")).y == [] diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index 9f3efb1b8..6aa61bd16 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -24,12 +24,11 @@ def main(params: Dict[str, int]) -> Dict[str, int]: "hey": 5, "hello": 2, } + assert tyro.cli(main, args="--params".split(" ")) == {} with pytest.raises(SystemExit): tyro.cli(main, args="--params hey 5 hello hey".split(" ")) with pytest.raises(SystemExit): tyro.cli(main, args="--params hey 5 hello".split(" ")) - with pytest.raises(SystemExit): - tyro.cli(main, args="--params".split(" ")) def test_dict_with_default() -> None: diff --git a/tests/test_helptext.py b/tests/test_helptext.py index a308c435a..1e8055c85 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -359,7 +359,7 @@ class GenericTupleHelptext(Generic[T]): x: List[T] helptext = get_helptext(GenericTupleHelptext[int]) - assert "--x INT [INT ...]" in helptext + assert "--x [INT [INT ...]]" in helptext def test_literal_helptext() -> None: @@ -445,7 +445,7 @@ class OptionalHelptext: assert cast(str, cast(str, OptionalHelptext.__doc__)[:20]) in helptext assert "2% milk" in helptext assert "--x {None}|INT" in helptext - assert "--y {None}|INT [{None}|INT ...]" in helptext + assert "--y [{None}|INT [{None}|INT ...]]" in helptext assert "[--z {None}|INT]" in helptext @@ -469,7 +469,7 @@ def main( # The comma formatting is unfortunate, but matches argparse's default behavior. helptext = get_helptext(main) - assert "--x {0,1,2,3,hey,there,hello}|{INT [INT ...]}" in helptext + assert "--x {0,1,2,3,hey,there,hello}|{[INT [INT ...]]}" in helptext def test_metavar_2() -> None: @@ -519,7 +519,7 @@ def main( pass helptext = get_helptext(main) - assert "[--x {INT INT}|{STR STR} [{INT INT}|{STR STR} ...]]" in helptext + assert "[--x [{INT INT}|{STR STR} [{INT INT}|{STR STR} ...]]]" in helptext def test_metavar_6() -> None: @@ -528,7 +528,8 @@ def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> d helptext = get_helptext(main) assert ( - "--x {INT INT}|{STR STR} INT INT [{INT INT}|{STR STR} INT INT ...]" in helptext + "--x [{INT INT}|{STR STR} INT INT [{INT INT}|{STR STR} INT INT ...]]" + in helptext ) diff --git a/tests/test_positional_ignore_py37.py b/tests/test_positional_ignore_py37.py index 91c87731c..15f8f7911 100644 --- a/tests/test_positional_ignore_py37.py +++ b/tests/test_positional_ignore_py37.py @@ -96,8 +96,7 @@ def main(a: Optional[List[int]], /) -> Optional[List[int]]: assert tyro.cli(main, args=["None"]) is None assert tyro.cli(main, args=["1", "2"]) == [1, 2] - with pytest.raises(SystemExit): - tyro.cli(main, args=[]) + assert tyro.cli(main, args=[]) == [] with pytest.raises(SystemExit): tyro.cli(main, args=["hm"]) diff --git a/tyro/__init__.py b/tyro/__init__.py index 477413871..bd6f1e4ab 100644 --- a/tyro/__init__.py +++ b/tyro/__init__.py @@ -1,8 +1,9 @@ +from typing import TYPE_CHECKING + from . import conf, extras from ._cli import cli from ._fields import MISSING_PUBLIC as MISSING from ._instantiators import UnsupportedTypeAnnotationError -from typing import TYPE_CHECKING __all__ = [ "conf", diff --git a/tyro/_calling.py b/tyro/_calling.py index d26e99429..52d2c30a4 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -72,6 +72,17 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: if value in _fields.MISSING_SINGLETONS: value = arg.field.default + + # Consider a function with a positional sequence argument: + # + # def f(x: tuple[int, ...], /) + # + # If we run this script with no arguments, we should interpret this + # as empty input for x. But the argparse default will be a MISSING + # value, and the field default will be inspect.Parameter.empty. + if value in _fields.MISSING_SINGLETONS: + assert field.is_positional() and arg.lowered.nargs in ("?", "*") + value = [] else: if arg.lowered.nargs == "?": # Special case for optional positional arguments: this is the diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index c24dc0e9b..9fc35b6a2 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -89,7 +89,7 @@ class InstantiatorMetadata: # Unlike in vanilla argparse, we never set nargs to None. To make things simpler, we # instead use nargs=1. - nargs: Optional[Union[int, Literal["+"]]] + nargs: Optional[Union[int, Literal["*"]]] # Unlike in vanilla argparse, our metavar is always a string. We handle # sequences, multiple arguments, etc, manually. metavar: str @@ -290,7 +290,7 @@ def _instantiator_from_type_inner( """Thin wrapper over instantiator_from_type, with some extra asserts for catching errors.""" out = instantiator_from_type(typ, type_from_typevar, markers) - if out[1].nargs == "+": + if out[1].nargs == "*": # We currently only use allow_sequences=False for options in Literal types, # which are evaluated using `type()`. It should not be possible to hit this # condition from polling a runtime type. @@ -444,7 +444,7 @@ def _instantiator_from_union( # right. instantiators = [] metas = [] - nargs: Optional[Union[int, Literal["+"]]] = 1 + nargs: Optional[Union[int, Literal["*"]]] = 1 first = True for t in options: a, b = _instantiator_from_type_inner( @@ -461,7 +461,7 @@ def _instantiator_from_union( first = False elif nargs != b.nargs: # Just be as general as possible if we see inconsistencies. - nargs = "+" + nargs = "*" metavar: str metavar = _join_union_metavars(map(lambda x: cast(str, x.metavar), metas)) @@ -480,7 +480,7 @@ def union_instantiator(strings: List[str]) -> Any: continue # Try passing input into instantiator. - if len(strings) == metadata.nargs or (metadata.nargs == "+"): + if len(strings) == metadata.nargs or (metadata.nargs == "*"): try: return instantiator(strings) # type: ignore except ValueError as e: @@ -533,7 +533,7 @@ def append_dict_instantiator(strings: List[List[str]]) -> Any: return out return append_dict_instantiator, InstantiatorMetadata( - nargs=key_nargs + val_nargs if isinstance(val_nargs, int) else "+", + nargs=key_nargs + val_nargs if isinstance(val_nargs, int) else "*", metavar=pair_metavar, choices=None, action="append", @@ -579,7 +579,7 @@ def dict_instantiator(strings: List[str]) -> Any: return out return dict_instantiator, InstantiatorMetadata( - nargs="+", + nargs="*", metavar=_strings.multi_metavar_from_single(pair_metavar), choices=None, action=None, @@ -650,7 +650,7 @@ def sequence_instantiator(strings: List[str]) -> Any: return container_type(out) return sequence_instantiator, InstantiatorMetadata( - nargs="+", + nargs="*", metavar=_strings.multi_metavar_from_single(inner_meta.metavar), choices=inner_meta.choices, action=None, diff --git a/tyro/_strings.py b/tyro/_strings.py index 620106e36..4f8dd0782 100644 --- a/tyro/_strings.py +++ b/tyro/_strings.py @@ -129,9 +129,9 @@ def strip_ansi_sequences(x: str): def multi_metavar_from_single(single: str) -> str: if len(strip_ansi_sequences(single)) >= 32: # Shorten long metavars - return f"{single} [...]" + return f"[{single} [...]]" else: - return f"{single} [{single} ...]" + return f"[{single} [{single} ...]]" def remove_single_line_breaks(helptext: str) -> str: From 54c22dba106557d6a0166f988fce7917fa6173bd Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Jul 2023 16:04:00 -0700 Subject: [PATCH 300/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 42361bacf..3dd4b9870 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.3" +version = "0.5.4" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } From 827f1d66b1500f658b3dcb5b44ab4f1b75a1ea60 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Jul 2023 16:11:59 -0700 Subject: [PATCH 301/491] Switch export style --- tyro/__init__.py | 19 +++++++------------ tyro/_fields.py | 2 +- tyro/extras/__init__.py | 22 ++++++++-------------- 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/tyro/__init__.py b/tyro/__init__.py index bd6f1e4ab..947254dc3 100644 --- a/tyro/__init__.py +++ b/tyro/__init__.py @@ -1,17 +1,12 @@ from typing import TYPE_CHECKING -from . import conf, extras -from ._cli import cli -from ._fields import MISSING_PUBLIC as MISSING -from ._instantiators import UnsupportedTypeAnnotationError - -__all__ = [ - "conf", - "extras", - "cli", - "MISSING", - "UnsupportedTypeAnnotationError", -] +from . import conf as conf +from . import extras as extras +from ._cli import cli as cli +from ._fields import MISSING as MISSING +from ._instantiators import ( + UnsupportedTypeAnnotationError as UnsupportedTypeAnnotationError, +) # Deprecated interface. if not TYPE_CHECKING: diff --git a/tyro/_fields.py b/tyro/_fields.py index bb6c16be6..a3d2eed1a 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -144,7 +144,7 @@ class ExcludeFromCallType(_singleton.Singleton): EXCLUDE_FROM_CALL = ExcludeFromCallType() # Note that our "public" missing API will always be the propagating missing sentinel. -MISSING_PUBLIC: Any = MISSING_PROP +MISSING: Any = MISSING_PROP """Sentinel value to mark fields as missing. Can be used to mark fields passed in as a `default_instance` for `tyro.cli()` as required.""" diff --git a/tyro/extras/__init__.py b/tyro/extras/__init__.py index 75b6c4bde..d6e6b9819 100644 --- a/tyro/extras/__init__.py +++ b/tyro/extras/__init__.py @@ -2,17 +2,11 @@ Compared to the core interface, APIs here are more likely to be changed or deprecated. """ -from .._argparse_formatter import set_accent_color -from .._cli import get_parser -from ._base_configs import subcommand_type_from_defaults -from ._choices_type import literal_type_from_choices -from ._serialization import from_yaml, to_yaml - -__all__ = [ - "set_accent_color", - "subcommand_type_from_defaults", - "get_parser", - "literal_type_from_choices", - "from_yaml", - "to_yaml", -] +from .._argparse_formatter import set_accent_color as set_accent_color +from .._cli import get_parser as get_parser +from ._base_configs import ( + subcommand_type_from_defaults as subcommand_type_from_defaults, +) +from ._choices_type import literal_type_from_choices as literal_type_from_choices +from ._serialization import from_yaml as from_yaml +from ._serialization import to_yaml as to_yaml From 2977c45c1c62cb57da800d759418104e1ec8c26f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Jul 2023 16:14:29 -0700 Subject: [PATCH 302/491] Fix version check for pydantic!=2.0 --- tyro/_fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tyro/_fields.py b/tyro/_fields.py index a3d2eed1a..25642ea3e 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -459,8 +459,8 @@ def _field_list_from_pydantic( # Handle pydantic models. field_list = [] - pydantic_version = float(getattr(pydantic, "__version__", "1.0")) - if pydantic_version < 2.0: # pragma: no cover + pydantic_version = int(getattr(pydantic, "__version__", "1.0.0").partition(".")[0]) + if pydantic_version < 2: # pragma: no cover # Pydantic 1.xx. for pd_field in cls.__fields__.values(): # type: ignore helptext = pd_field.field_info.description From 4d630c32976a9070752fc7bba14ea28f5d1749a2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 5 Jul 2023 16:24:26 -0700 Subject: [PATCH 303/491] Add `hydra-zen` link --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 05e1f6e0c..b01a9269e 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ See [documentation](https://brentyi.github.io/tyro) for examples. implementation of [instant-ngp](https://nvlabs.github.io/instant-ngp/), implemented in JAX. - [NVIDIAGameWorks/kaolin-wisp](https://github.com/NVIDIAGameWorks/kaolin-wisp) - is a PyTorch library for working with neural fields. + combines `tyro` with [`hydra-zen`](https://github.com/mit-ll-responsible-ai/hydra-zen) + for neural fields in PyTorch. - [openrlbenchmark/openrlbenchmark](https://github.com/openrlbenchmark/openrlbenchmark) - is a comprehensive collection of tracked experiments for reinforcement - learning. + is a collection of tracked experiments for reinforcement learning. From 2264caba850e38813ba38476581b2c9220523372 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 20 Jul 2023 13:39:06 -0700 Subject: [PATCH 304/491] Fix dependencies for docs build --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 14238bcac..b59a66cf6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,7 +16,7 @@ jobs: - name: Building documentation run: | pip install --upgrade pip - pip install -e . + pip install -e ".[dev]" pip install -r docs/requirements.txt sphinx-build docs/source docs/build -b dirhtml From fdd45e4c6353817d463057d3f51036c9c6c26a28 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 27 Jul 2023 12:19:35 -0700 Subject: [PATCH 305/491] Fix edge cases when generating subcommands from Optional[] types --- tests/test_nested.py | 61 ++++++++++++++++++++++++++++++++++++++++++++ tyro/_calling.py | 37 ++++++++++++--------------- tyro/_parsers.py | 53 +++++++++++++++++--------------------- tyro/_strings.py | 4 +-- 4 files changed, 102 insertions(+), 53 deletions(-) diff --git a/tests/test_nested.py b/tests/test_nested.py index f87a6d81e..38a1c2ba4 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -169,6 +169,67 @@ class OptionalNested: ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) +def test_optional_nested_multiple() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/60""" + + @dataclasses.dataclass(frozen=True) + class OutputHeadSettings: + number_of_outputs: int = 1 + + @dataclasses.dataclass(frozen=True) + class OptimizerSettings: + name: str = "adam" + + @dataclasses.dataclass(frozen=True) + class ModelSettings: + output_head_settings: Optional[OutputHeadSettings] = None + optimizer_settings: Optional[OptimizerSettings] = None + + assert tyro.cli( + ModelSettings, + args="output-head-settings:None optimizer-settings:None".split(" "), + ) == ModelSettings(None, None) + + with pytest.raises(SystemExit): + # Order cannot be flipped, unfortunately. + tyro.cli( + ModelSettings, + args="optimizer-settings:None output-head-settings:None".split(" "), + ) + + assert tyro.cli( + ModelSettings, + args="output-head-settings:output-head-settings optimizer-settings:None".split( + " " + ), + ) == ModelSettings(OutputHeadSettings(1), None) + + assert tyro.cli( + ModelSettings, + args="output-head-settings:output-head-settings --output-head-settings.number-of-outputs 5 optimizer-settings:None".split( + " " + ), + ) == ModelSettings(OutputHeadSettings(5), None) + + assert tyro.cli( + tyro.conf.OmitSubcommandPrefixes[ + tyro.conf.ConsolidateSubcommandArgs[ModelSettings] + ], + args="output-head-settings:output-head-settings optimizer-settings:None --number-of-outputs 5".split( + " " + ), + ) == ModelSettings(OutputHeadSettings(5), None) + + assert tyro.cli( + tyro.conf.OmitSubcommandPrefixes[ + tyro.conf.ConsolidateSubcommandArgs[ModelSettings] + ], + args="output-head-settings:output-head-settings optimizer-settings:optimizer-settings --name sgd --number-of-outputs 5".split( + " " + ), + ) == ModelSettings(OutputHeadSettings(5), OptimizerSettings("sgd")) + + def test_subparser() -> None: @dataclasses.dataclass class HTTPServer: diff --git a/tyro/_calling.py b/tyro/_calling.py index 52d2c30a4..5931b5bf9 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -143,15 +143,6 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: or subparser_def.default_instance is not None ) value = subparser_def.default_instance - elif ( - subparser_def.can_be_none - and subparser_name - == _strings.subparser_name_from_type(prefixed_field_name, None) - ): - # do either - # Optional[Union[A, B, ...]] or Union[A, B, None], or have a - # default/default_factory set. - value = None else: options = map( lambda x: _resolver.apply_type_from_typevar(x, type_from_typevar), @@ -166,18 +157,22 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: chosen_f = option break assert chosen_f is not None - value, consumed_keywords_child = call_from_args( - chosen_f, - subparser_def.parser_from_name[subparser_name], - ( - field.default - if type(field.default) is chosen_f - else _fields.MISSING_NONPROP - ), - value_from_prefixed_field_name, - field_name_prefix=prefixed_field_name, - ) - consumed_keywords |= consumed_keywords_child + + if chosen_f is type(None): + value = None + else: + value, consumed_keywords_child = call_from_args( + chosen_f, + subparser_def.parser_from_name[subparser_name], + ( + field.default + if type(field.default) is chosen_f + else _fields.MISSING_NONPROP + ), + value_from_prefixed_field_name, + field_name_prefix=prefixed_field_name, + ) + consumed_keywords |= consumed_keywords_child if value is _fields.EXCLUDE_FROM_CALL: continue diff --git a/tyro/_parsers.py b/tyro/_parsers.py index dab1ef38c..3a9b5aa5d 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -337,7 +337,6 @@ class SubparsersSpecification: prefix: str required: bool default_instance: Any - can_be_none: bool # If underlying type is Optional[Something]. @staticmethod def from_field( @@ -352,15 +351,20 @@ def from_field( return None # We don't use sets here to retain order of subcommands. + options: List[Union[type, Callable]] options = [ _resolver.apply_type_from_typevar(typ, type_from_typevar) for typ in get_args(typ) ] - options_no_none = [o for o in options if o is not type(None)] # noqa + options = [ + # Cast seems unnecessary but needed in mypy... (1.4.1) + cast(Callable, none_proxy) if o is type(None) else o + for o in options + ] if not all( [ - _fields.is_nested_type(o, _fields.MISSING_NONPROP) - for o in options_no_none + _fields.is_nested_type(cast(type, o), _fields.MISSING_NONPROP) + for o in options ] ): return None @@ -370,8 +374,10 @@ def from_field( str, _confstruct._SubcommandConfiguration ] = {} subcommand_type_from_name: Dict[str, type] = {} - for option in options_no_none: - subcommand_name = _strings.subparser_name_from_type(prefix, option) + for option in options: + subcommand_name = _strings.subparser_name_from_type( + prefix, type(None) if option is none_proxy else cast(type, option) + ) option, found_subcommand_configs = _resolver.unwrap_annotated( option, _confstruct._SubcommandConfiguration ) @@ -384,7 +390,7 @@ def from_field( subcommand_config_from_name[subcommand_name] = found_subcommand_configs[ 0 ] - subcommand_type_from_name[subcommand_name] = option + subcommand_type_from_name[subcommand_name] = cast(type, option) # If a field default is provided, try to find a matching subcommand name. if field.default is None or field.default in _fields.MISSING_SINGLETONS: @@ -401,14 +407,16 @@ def from_field( f"`{prefix}` was provided a default value of type" f" {type(field.default)} but no matching subcommand was found. A" " type may be missing in the Union type declaration for" - f" `{prefix}`, which currently expects {options_no_none}." + f" `{prefix}`, which currently expects {options}." ) return None # Add subcommands for each option. parser_from_name: Dict[str, ParserSpecification] = {} - for option in options_no_none: - subcommand_name = _strings.subparser_name_from_type(prefix, option) + for option in options: + subcommand_name = _strings.subparser_name_from_type( + prefix, type(None) if option is none_proxy else cast(type, option) + ) option, _ = _resolver.unwrap_annotated(option) # Get a subcommand config: either pulled from the type annotations or the @@ -495,25 +503,13 @@ def from_field( prefix=prefix, required=required, default_instance=field.default, - can_be_none=options != options_no_none, ) def apply( self, parent_parser: argparse.ArgumentParser ) -> Tuple[argparse.ArgumentParser, ...]: title = "subcommands" - metavar = ( - "{" - + ",".join( - ( - (_strings.subparser_name_from_type(self.prefix, None),) - if self.can_be_none - else () - ) - + tuple(self.parser_from_name.keys()) - ) - + "}" - ) + metavar = "{" + ",".join(self.parser_from_name.keys()) + "}" if not self.required: title = "optional " + title metavar = f"[{metavar}]" @@ -527,13 +523,6 @@ def apply( metavar=metavar, ) - if self.can_be_none: - subparser = argparse_subparsers.add_parser( - name=_strings.subparser_name_from_type(self.prefix, None), - formatter_class=_argparse_formatter.TyroArgparseHelpFormatter, - help="", - ) - subparser_tree_leaves: List[argparse.ArgumentParser] = [] for name, subparser_def in self.parser_from_name.items(): helptext = subparser_def.description.replace("%", "%%") @@ -568,3 +557,7 @@ def add_subparsers_to_leaves( parser_from_name=new_parsers_from_name, required=root.required or leaf.required, ) + + +def none_proxy() -> None: + return None diff --git a/tyro/_strings.py b/tyro/_strings.py index 4f8dd0782..0abfce80a 100644 --- a/tyro/_strings.py +++ b/tyro/_strings.py @@ -107,9 +107,9 @@ def get_name(cls: Type) -> str: ) -def subparser_name_from_type(prefix: str, cls: Union[Type, None]) -> str: +def subparser_name_from_type(prefix: str, cls: Type) -> str: suffix, use_prefix = ( - _subparser_name_from_type(cls) if cls is not None else ("None", True) + _subparser_name_from_type(cls) if cls is not type(None) else ("None", True) ) if len(prefix) == 0 or not use_prefix: return suffix From 7fd53ac4b35307f2d48083d52e8786fa830f6d04 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 27 Jul 2023 12:22:45 -0700 Subject: [PATCH 306/491] Fix dummy dataclass warnings --- tyro/_cli.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tyro/_cli.py b/tyro/_cli.py index a5358293c..6d0137998 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -287,16 +287,13 @@ def _cli_impl( if not _fields.is_nested_type(cast(type, f), default_instance_internal): dummy_field = cast( dataclasses.Field, - dataclasses.field( - default_factory=lambda: default - if default is not None - else dataclasses.MISSING - ), + dataclasses.field(), ) f = dataclasses.make_dataclass( cls_name="", fields=[(_strings.dummy_field_name, cast(type, f), dummy_field)], ) + default_instance_internal = f(default_instance_internal) dummy_wrapped = True else: dummy_wrapped = False From ec2b5a8e2427b5e404e70098497711853d8f16ea Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 27 Jul 2023 12:24:19 -0700 Subject: [PATCH 307/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3dd4b9870..c4738c8b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.4" +version = "0.5.5" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } From 81a6ec4d7902973e9aa7cc89d03235d74cb4f29a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 27 Jul 2023 13:24:32 -0700 Subject: [PATCH 308/491] Fix 3.7 install --- pyproject.toml | 1 + tyro/_cli.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c4738c8b9..07a1b61f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ dev = [ "pyright>=1.1.264", "mypy>=0.991", "numpy>=1.20.0", + "orbax==0.1.6", # Needed for flax install in Python 3.7 as of 7/27/2023. "flax>=0.6.0", "pydantic>=1.10.2", "coverage[toml]>=6.5.0" diff --git a/tyro/_cli.py b/tyro/_cli.py index 6d0137998..adb034e17 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -292,6 +292,7 @@ def _cli_impl( f = dataclasses.make_dataclass( cls_name="", fields=[(_strings.dummy_field_name, cast(type, f), dummy_field)], + frozen=True, ) default_instance_internal = f(default_instance_internal) dummy_wrapped = True From a45e4d7dfc0ef9d25213e20f11c37d637f42079a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 27 Jul 2023 13:25:53 -0700 Subject: [PATCH 309/491] Fix mypy --- tyro/_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tyro/_cli.py b/tyro/_cli.py index adb034e17..1837bd912 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -294,7 +294,7 @@ def _cli_impl( fields=[(_strings.dummy_field_name, cast(type, f), dummy_field)], frozen=True, ) - default_instance_internal = f(default_instance_internal) + default_instance_internal = f(default_instance_internal) # type: ignore dummy_wrapped = True else: dummy_wrapped = False From cf6e178382c635dc7b8eff4d1ac01f0b06024efc Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 27 Jul 2023 13:29:27 -0700 Subject: [PATCH 310/491] Pin flax --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 07a1b61f0..6b57bd5e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,8 +43,7 @@ dev = [ "pyright>=1.1.264", "mypy>=0.991", "numpy>=1.20.0", - "orbax==0.1.6", # Needed for flax install in Python 3.7 as of 7/27/2023. - "flax>=0.6.0", + "flax==0.6.0", # Pinning needed for Python 3.7 compatibility. "pydantic>=1.10.2", "coverage[toml]>=6.5.0" ] From 512dc4849650cdfbbe6447463062af1a127c4b60 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 27 Jul 2023 19:57:17 -0700 Subject: [PATCH 311/491] Fix helptext formatting edge cases --- tyro/_arguments.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 0d503d8f4..348bcd0de 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -398,9 +398,6 @@ def _rule_generate_helptext( primary_help = _strings.make_field_name([arg.name_prefix, arg.field.name]) if primary_help is not None and primary_help != "": - # Note that the percent symbol needs some extra handling in argparse. - # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string - primary_help = primary_help.replace("%", "%%") help_parts.append(_rich_tag_if_enabled(primary_help, "helptext")) default = lowered.default @@ -444,7 +441,9 @@ def _rule_generate_helptext( else: help_parts.append(_rich_tag_if_enabled("(required)", "helptext_required")) - return dataclasses.replace(lowered, help=" ".join(help_parts)) + # Note that the percent symbol needs some extra handling in argparse. + # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string + return dataclasses.replace(lowered, help=" ".join(help_parts).replace("%", "%%")) def _rule_set_name_or_flag_and_dest( From 5c144d1186c7bcb263e50deb8c61e911c57bde4d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 27 Jul 2023 22:28:44 -0700 Subject: [PATCH 312/491] Rework test run conditions --- pyproject.toml | 4 +++- tests/conftest.py | 12 +++++------- ...st_flax_ignore_py310.py => test_flax_min_py38.py} | 0 ...itvar_ignore_py37.py => test_initvar_min_py38.py} | 0 ...10.py => test_new_style_annotations_min_py310.py} | 0 ...y39.py => test_new_style_annotations_min_py39.py} | 0 ...al_ignore_py37.py => test_positional_min_py38.py} | 0 7 files changed, 8 insertions(+), 8 deletions(-) rename tests/{test_flax_ignore_py310.py => test_flax_min_py38.py} (100%) rename tests/{test_initvar_ignore_py37.py => test_initvar_min_py38.py} (100%) rename tests/{test_new_style_annotations_only_py310.py => test_new_style_annotations_min_py310.py} (100%) rename tests/{test_new_style_annotations_above_py39.py => test_new_style_annotations_min_py39.py} (100%) rename tests/{test_positional_ignore_py37.py => test_positional_min_py38.py} (100%) diff --git a/pyproject.toml b/pyproject.toml index 6b57bd5e9..5c19e77b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,9 @@ dev = [ "pyright>=1.1.264", "mypy>=0.991", "numpy>=1.20.0", - "flax==0.6.0", # Pinning needed for Python 3.7 compatibility. + # As of 7/27/2023, flax install fails for Python 3.7 without pinning to an + # old version. But doing so breaks other Python versions. + "flax>=0.6.9;python_version>='3.8'", "pydantic>=1.10.2", "coverage[toml]>=6.5.0" ] diff --git a/tests/conftest.py b/tests/conftest.py index 2da5d1997..19cbe79de 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,14 +1,12 @@ import sys collect_ignore_glob = [] -if sys.version_info.major == 3 and sys.version_info.minor == 7: - collect_ignore_glob.append("*_ignore_py37.py") -if sys.version_info.major == 3 and sys.version_info.minor == 10: - collect_ignore_glob.append("*_ignore_py310.py") +if not (sys.version_info.major == 3 and sys.version_info.minor >= 8): + collect_ignore_glob.append("*_min_py38.py") if not (sys.version_info.major == 3 and sys.version_info.minor >= 9): - collect_ignore_glob.append("*_above_py39.py") + collect_ignore_glob.append("*_min_py39.py") -if not (sys.version_info.major == 3 and sys.version_info.minor == 10): - collect_ignore_glob.append("*_only_py310.py") +if not (sys.version_info.major == 3 and sys.version_info.minor >= 10): + collect_ignore_glob.append("*_min_py310.py") diff --git a/tests/test_flax_ignore_py310.py b/tests/test_flax_min_py38.py similarity index 100% rename from tests/test_flax_ignore_py310.py rename to tests/test_flax_min_py38.py diff --git a/tests/test_initvar_ignore_py37.py b/tests/test_initvar_min_py38.py similarity index 100% rename from tests/test_initvar_ignore_py37.py rename to tests/test_initvar_min_py38.py diff --git a/tests/test_new_style_annotations_only_py310.py b/tests/test_new_style_annotations_min_py310.py similarity index 100% rename from tests/test_new_style_annotations_only_py310.py rename to tests/test_new_style_annotations_min_py310.py diff --git a/tests/test_new_style_annotations_above_py39.py b/tests/test_new_style_annotations_min_py39.py similarity index 100% rename from tests/test_new_style_annotations_above_py39.py rename to tests/test_new_style_annotations_min_py39.py diff --git a/tests/test_positional_ignore_py37.py b/tests/test_positional_min_py38.py similarity index 100% rename from tests/test_positional_ignore_py37.py rename to tests/test_positional_min_py38.py From 5340498f878c01e59b7764893c5fa5b78349cb84 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Wed, 30 Aug 2023 03:10:30 +0900 Subject: [PATCH 313/491] Fix typo in README.md (#65) hierachical -> hierarchical --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b01a9269e..cd1b4feac 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ will work out-of-the-box. These examples only scratch the surface of what's possible. `tyro` aims to support all reasonable type annotations, which can help us define things like -hierachical structures, enums, unions, variable-length inputs, and subcommands. +hierarchical structures, enums, unions, variable-length inputs, and subcommands. See [documentation](https://brentyi.github.io/tyro) for examples. ### In the wild From 83592d67d9d4141d07eb3b0d09c24d135e570e08 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 1 Sep 2023 00:43:45 -0700 Subject: [PATCH 314/491] Better error messages (#63) * Improve error messages * Tweak error messages, fix types * Use difflib, add help messages * ruff + black --preview * allow_abbrev=False in subparsers * Fix exit codes * Fix type errors * Tune similarity thresholds * Fix metavar ordering regression * Tweak unrecognized argument filter * Improve error messages for unrecognized arguments with subparsers (by a lot!) * Fix unrecognized argument false positives * Python 3.7-compatible mypy * Polish unrecognized argument error message when subcommands are present * Fix for duplicate arguments * Fix `same_exists` default * Print helptext of similar arguments * Refine helptext in error messages * Remove delete * Add tests + heuristics * Tweaks for coverage * Fix or suppress mypy error * Fix test * Bump version * Error message cleanup, test fixes * Handle flags / custom actions in unrecognized argument errors * Fix Python 3.7 * Hack for jax error * Revert jax hack * Suppress long usage prints * Error message consistency * Test coverage * Fix test typo --- examples/01_basics/01_functions.py | 0 pyproject.toml | 4 +- tests/test_collections.py | 2 +- tests/test_errors.py | 315 +++++++++++++ tests/test_nested.py | 19 +- tyro/_argparse_formatter.py | 709 +++++++++++++++++++++++++++-- tyro/_arguments.py | 21 +- tyro/_calling.py | 13 +- tyro/_cli.py | 67 ++- tyro/_fields.py | 8 +- tyro/_instantiators.py | 11 +- tyro/_parsers.py | 17 +- tyro/_strings.py | 2 +- 13 files changed, 1102 insertions(+), 86 deletions(-) mode change 100755 => 100644 examples/01_basics/01_functions.py diff --git a/examples/01_basics/01_functions.py b/examples/01_basics/01_functions.py old mode 100755 new mode 100644 diff --git a/pyproject.toml b/pyproject.toml index 5c19e77b1..701c36333 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.5" +version = "0.5.6" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } @@ -41,7 +41,7 @@ dev = [ "attrs>=21.4.0", "torch>=1.10.0", "pyright>=1.1.264", - "mypy>=0.991", + "mypy>=1.4.1", "numpy>=1.20.0", # As of 7/27/2023, flax install fails for Python 3.7 without pinning to an # old version. But doing so breaks other Python versions. diff --git a/tests/test_collections.py b/tests/test_collections.py index a20e73c22..59287bb18 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -78,7 +78,7 @@ class A: assert tyro.cli(A, args=[]) == A(x=(1, 2)) target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stderr(target): + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): tyro.cli(A, args=["1", "2", "3", "4"]) assert "invalid choice" in target.getvalue() diff --git a/tests/test_errors.py b/tests/test_errors.py index ced3abaa1..bf80bd25b 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -1,4 +1,6 @@ +import contextlib import dataclasses +import io from typing import List, Tuple, TypeVar, Union import pytest @@ -148,3 +150,316 @@ def main(value: list) -> None: with pytest.raises(tyro.UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) + + +def test_similar_arguments_basic() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + + @dataclasses.dataclass + class Class: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Class, args="--reward.trac".split(" ")) + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Similar arguments" in error + + # --reward.track should appear in both the usage string and as a similar argument. + assert error.count("--reward.track") == 2 + assert error.count("--help") == 0 + + +def test_similar_arguments_subcommands() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Union[ClassA, ClassB], args="--reward.trac".split(" ")) # type: ignore + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Similar arguments:" in error + assert error.count("--reward.track") == 1 + assert error.count("--help") == 2 + + +def test_similar_arguments_subcommands_multiple() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + trace: int + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Union[ClassA, ClassB], args="--fjdkslaj --reward.trac".split(" ")) # type: ignore + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Arguments similar to --reward.trac" in error + assert error.count("--reward.track {True,False}") == 1 + assert error.count("--reward.trace INT") == 1 + assert error.count("--help") == 4 + + +def test_similar_arguments_subcommands_multiple_contains_match() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + trace: int + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Union[ClassA, ClassB], args="--rd.trac".split(" ")) # type: ignore + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Similar arguments" in error + assert error.count("--reward.track {True,False}") == 1 + assert error.count("--reward.trace INT") == 1 + assert error.count("--help") == 4 # 2 subcommands * 2 arguments. + + +def test_similar_arguments_subcommands_multiple_contains_match_alt() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + trace: int + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Union[ClassA, ClassB], args="--track".split(" ")) # type: ignore + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Similar arguments" in error + assert error.count("--reward.track {True,False}") == 1 + assert error.count("--help") == 2 # Should show two possible subcommands. + + +def test_similar_arguments_subcommands_overflow_different() -> None: + @dataclasses.dataclass + class RewardConfig: + track0: bool + track1: bool + track2: bool + track3: bool + track4: bool + track5: bool + track6: bool + track7: bool + track8: bool + track9: bool + track10: bool + track11: bool + track12: bool + track13: bool + track14: bool + track15: bool + track16: bool + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Union[ClassA, ClassB], args="--track".split(" ")) # type: ignore + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Similar arguments" in error + assert error.count("--reward.track") == 10 + assert "[...]" not in error + assert error.count("--help") == 20 + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + # --tracked is intentionally between 0.8 ~ 0.9 similarity to track{i} for test + # coverage. + tyro.cli(RewardConfig, args="--tracked".split(" ")) # type: ignore + + # Usage print should be clipped. + error = target.getvalue() + assert "help:" in error + + +def test_similar_arguments_subcommands_overflow_same() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + @dataclasses.dataclass + class ClassC: + reward: RewardConfig + + @dataclasses.dataclass + class ClassD: + reward: RewardConfig + + @dataclasses.dataclass + class ClassE: + reward: RewardConfig + + @dataclasses.dataclass + class ClassF: + reward: RewardConfig + + @dataclasses.dataclass + class ClassG: + reward: RewardConfig + + @dataclasses.dataclass + class ClassH: + reward: RewardConfig + + @dataclasses.dataclass + class ClassI: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli( # type: ignore + Union[ + ClassA, ClassB, ClassC, ClassD, ClassE, ClassF, ClassG, ClassH, ClassI + ], + args="--track".split(" "), + ) + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Similar arguments" in error + assert error.count("--reward.track") == 1 + assert "[...]" in error + assert error.count("--help") == 4 + + +def test_similar_arguments_subcommands_overflow_same_startswith_multiple() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + @dataclasses.dataclass + class ClassC: + reward: RewardConfig + + @dataclasses.dataclass + class ClassD: + reward: RewardConfig + + @dataclasses.dataclass + class ClassE: + reward: RewardConfig + rewarde: tyro.conf.Suppress[int] = 10 + + @dataclasses.dataclass + class ClassF: + reward: RewardConfig + + @dataclasses.dataclass + class ClassG: + reward: RewardConfig + + @dataclasses.dataclass + class ClassH: + reward: RewardConfig + + @dataclasses.dataclass + class ClassI: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli( # type: ignore + Union[ + ClassA, ClassB, ClassC, ClassD, ClassE, ClassF, ClassG, ClassH, ClassI + ], + args="--track --ffff".split(" "), + ) + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Arguments similar to --track" in error + assert error.count("--rewar") == 1 + assert "rewarde" not in error + assert "[...]" in error + assert error.count("--help") == 4 + + +def test_similar_flag() -> None: + @dataclasses.dataclass + class Args: + flag: bool = False + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli( + Args, + args="--lag".split(" "), + ) + + error = target.getvalue() + + # Printed in the usage message. + assert error.count("--flag | --no-flag") == 1 + + # Printed in the similar argument list. + assert error.count("--flag, --no-flag") == 1 diff --git a/tests/test_nested.py b/tests/test_nested.py index 38a1c2ba4..f92112d72 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -206,8 +206,11 @@ class ModelSettings: assert tyro.cli( ModelSettings, - args="output-head-settings:output-head-settings --output-head-settings.number-of-outputs 5 optimizer-settings:None".split( - " " + args=( + "output-head-settings:output-head-settings" + " --output-head-settings.number-of-outputs 5 optimizer-settings:None".split( + " " + ) ), ) == ModelSettings(OutputHeadSettings(5), None) @@ -215,8 +218,9 @@ class ModelSettings: tyro.conf.OmitSubcommandPrefixes[ tyro.conf.ConsolidateSubcommandArgs[ModelSettings] ], - args="output-head-settings:output-head-settings optimizer-settings:None --number-of-outputs 5".split( - " " + args=( + "output-head-settings:output-head-settings optimizer-settings:None" + " --number-of-outputs 5".split(" ") ), ) == ModelSettings(OutputHeadSettings(5), None) @@ -224,8 +228,11 @@ class ModelSettings: tyro.conf.OmitSubcommandPrefixes[ tyro.conf.ConsolidateSubcommandArgs[ModelSettings] ], - args="output-head-settings:output-head-settings optimizer-settings:optimizer-settings --name sgd --number-of-outputs 5".split( - " " + args=( + "output-head-settings:output-head-settings" + " optimizer-settings:optimizer-settings --name sgd --number-of-outputs 5".split( + " " + ) ), ) == ModelSettings(OutputHeadSettings(5), OptimizerSettings("sgd")) diff --git a/tyro/_argparse_formatter.py b/tyro/_argparse_formatter.py index 874947a03..808902ebf 100644 --- a/tyro/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -5,29 +5,36 @@ - Use `rich` for formatting. - Can be themed with an accent color. -This is largely built by fussing around in argparse implementation details, and is by -far the hackiest part of `tyro`. +This is largely built by fussing around in argparse implementation details, and is +extremely chaotic as a result. -TODO: we may want to just maintain our own fork of argparse. +TODO: the current implementation should be robust given our test coverage, but unideal +long-term. We should just maintain our own fork of argparse. """ import argparse import contextlib import dataclasses +import difflib import itertools import re as _re import shutil -from typing import Any, ContextManager, Generator, List, Optional +import sys +from gettext import gettext as _ +from typing import Any, Generator, List, NoReturn, Optional, Tuple from rich.columns import Columns from rich.console import Console, Group, RenderableType from rich.padding import Padding from rich.panel import Panel +from rich.rule import Rule from rich.style import Style from rich.table import Table from rich.text import Text from rich.theme import Theme +from typing_extensions import override -from . import _arguments, _strings +from . import _arguments, _strings, conf +from ._parsers import ParserSpecification @dataclasses.dataclass @@ -79,47 +86,44 @@ def monkeypatch_len(obj: Any) -> int: return len(obj) -def ansi_context() -> ContextManager[None]: +@contextlib.contextmanager +def ansi_context() -> Generator[None, None, None]: """Context for working with ANSI codes + argparse: - Applies a temporary monkey patch for making argparse ignore ANSI codes when wrapping usage text. - Enables support for Windows via colorama. """ - @contextlib.contextmanager - def inner() -> Generator[None, None, None]: - if not hasattr(argparse, "len"): - # Sketchy, but seems to work. - argparse.len = monkeypatch_len # type: ignore - try: - # Use Colorama to support coloring in Windows shells. - import colorama # type: ignore - - # Notes: - # - # (1) This context manager looks very nice and local, but under-the-hood - # does some global operations which look likely to cause unexpected - # behavior if another library relies on `colorama.init()` and - # `colorama.deinit()`. - # - # (2) SSHed into a non-Windows machine from a WinAPI terminal => this - # won't work. - # - # Fixing these issues doesn't seem worth it: it doesn't seem like there - # are low-effort solutions for either problem, and more modern terminals - # in Windows (PowerShell, MSYS2, ...) do support ANSI codes anyways. - with colorama.colorama_text(): - yield - - except ImportError: + if not hasattr(argparse, "len"): + # Sketchy, but seems to work. + argparse.len = monkeypatch_len # type: ignore + try: # pragma: no cover + # Use Colorama to support coloring in Windows shells. + import colorama # type: ignore + + # Notes: + # + # (1) This context manager looks very nice and local, but under-the-hood + # does some global operations which look likely to cause unexpected + # behavior if another library relies on `colorama.init()` and + # `colorama.deinit()`. + # + # (2) SSHed into a non-Windows machine from a WinAPI terminal => this + # won't work. + # + # Fixing these issues doesn't seem worth it: it doesn't seem like there + # are low-effort solutions for either problem, and more modern terminals + # in Windows (PowerShell, MSYS2, ...) do support ANSI codes anyways. + with colorama.colorama_text(): yield - del argparse.len # type: ignore - else: - # No-op when the context manager is nested. + except ImportError: yield - return inner() + del argparse.len # type: ignore + else: + # No-op when the context manager is nested. + yield def str_from_rich( @@ -131,6 +135,613 @@ def str_from_rich( return out.get().rstrip("\n") +@dataclasses.dataclass(frozen=True) +class _ArgumentInfo: + option_strings: Tuple[str, ...] + metavar: Optional[str] + usage_hint: str + help: Optional[str] + subcommand_match_score: float + """Priority value used when an argument is in the current subcommand tree.""" + + +# By default, unrecognized arguments won't raise an error in the case of: +# +# # We've misspelled `binary`! +# python 03_multiple_subcommands.py dataset:mnist --dataset.binayr True +# +# When there's a subcommand that follows dataset:mnist. Instead, +# --dataset.binayr is consumed and we get an error that `True` is not a valid +# subcommand. This can be really confusing when we have a lot of keyword +# arguments. +# +# Our current solution is to manually track unrecognized arguments in _parse_known_args, +# and in error() override other errors when unrecognized arguments are present. +global_unrecognized_args: List[str] = [] + + +class TyroArgumentParser(argparse.ArgumentParser): + _parser_specification: ParserSpecification + _parsing_known_args: bool + _args: List[str] + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + @override + def _parse_known_args(self, arg_strings, namespace): # pragma: no cover + """We override _parse_known_args() to improve error messages in the presence of + subcommands. Difference is marked with ... below.""" + + # + # Reset the unused argument list in the root parser. + # Subparsers will have spaces in self.prog. + if " " not in self.prog: + global global_unrecognized_args + global_unrecognized_args = [] + # + + # replace arg strings that are file references + if self.fromfile_prefix_chars is not None: + arg_strings = self._read_args_from_files(arg_strings) + + # map all mutually exclusive arguments to the other arguments + # they can't occur with + action_conflicts = {} + for mutex_group in self._mutually_exclusive_groups: + group_actions = mutex_group._group_actions + for i, mutex_action in enumerate(mutex_group._group_actions): + conflicts = action_conflicts.setdefault(mutex_action, []) + conflicts.extend(group_actions[:i]) + conflicts.extend(group_actions[i + 1 :]) + + # find all option indices, and determine the arg_string_pattern + # which has an 'O' if there is an option at an index, + # an 'A' if there is an argument, or a '-' if there is a '--' + option_string_indices = {} + arg_string_pattern_parts = [] + arg_strings_iter = iter(arg_strings) + for i, arg_string in enumerate(arg_strings_iter): + # all args after -- are non-options + if arg_string == "--": + arg_string_pattern_parts.append("-") + for arg_string in arg_strings_iter: + arg_string_pattern_parts.append("A") + + # otherwise, add the arg to the arg strings + # and note the index if it was an option + else: + option_tuple = self._parse_optional(arg_string) + if option_tuple is None: + pattern = "A" + else: + option_string_indices[i] = option_tuple + pattern = "O" + arg_string_pattern_parts.append(pattern) + + # join the pieces together to form the pattern + arg_strings_pattern = "".join(arg_string_pattern_parts) + + # converts arg strings to the appropriate and then takes the action + seen_actions = set() + seen_non_default_actions = set() + + def take_action(action, argument_strings, option_string=None): + seen_actions.add(action) + argument_values = self._get_values(action, argument_strings) + + # error if this argument is not allowed with other previously + # seen arguments, assuming that actions that use the default + # value don't really count as "present" + if argument_values is not action.default: + seen_non_default_actions.add(action) + for conflict_action in action_conflicts.get(action, []): + if conflict_action in seen_non_default_actions: + msg = _("not allowed with argument %s") + action_name = argparse._get_action_name(conflict_action) + raise argparse.ArgumentError(action, msg % action_name) + + # take the action if we didn't receive a SUPPRESS value + # (e.g. from a default) + if argument_values is not argparse.SUPPRESS: + action(self, namespace, argument_values, option_string) + + # function to convert arg_strings into an optional action + def consume_optional(start_index): + # get the optional identified at this index + option_tuple = option_string_indices[start_index] + action, option_string, explicit_arg = option_tuple + + # identify additional optionals in the same arg string + # (e.g. -xyz is the same as -x -y -z if no args are required) + match_argument = self._match_argument + action_tuples = [] + while True: + # if we found no optional action, skip it + if action is None: + # + # Manually track unused arguments to assist with error messages + # later. + if not self._parsing_known_args: + global global_unrecognized_args + global_unrecognized_args.append(option_string) + # + + extras.append(arg_strings[start_index]) + return start_index + 1 + + # if there is an explicit argument, try to match the + # optional's string arguments to only this + if explicit_arg is not None: + arg_count = match_argument(action, "A") + + # if the action is a single-dash option and takes no + # arguments, try to parse more single-dash options out + # of the tail of the option string + chars = self.prefix_chars + if ( + arg_count == 0 + and option_string[1] not in chars + and explicit_arg != "" + ): + action_tuples.append((action, [], option_string)) + char = option_string[0] + option_string = char + explicit_arg[0] + new_explicit_arg = explicit_arg[1:] or None + optionals_map = self._option_string_actions + if option_string in optionals_map: + action = optionals_map[option_string] + explicit_arg = new_explicit_arg + else: + msg = _("ignored explicit argument %r") + raise argparse.ArgumentError(action, msg % explicit_arg) + + # if the action expect exactly one argument, we've + # successfully matched the option; exit the loop + elif arg_count == 1: + stop = start_index + 1 + args = [explicit_arg] + action_tuples.append((action, args, option_string)) + break + + # error if a double-dash option did not use the + # explicit argument + else: + msg = _("ignored explicit argument %r") + raise argparse.ArgumentError(action, msg % explicit_arg) + + # if there is no explicit argument, try to match the + # optional's string arguments with the following strings + # if successful, exit the loop + else: + start = start_index + 1 + selected_patterns = arg_strings_pattern[start:] + arg_count = match_argument(action, selected_patterns) + stop = start + arg_count + args = arg_strings[start:stop] + action_tuples.append((action, args, option_string)) + break + + # add the Optional to the list and return the index at which + # the Optional's string args stopped + assert action_tuples + for action, args, option_string in action_tuples: + take_action(action, args, option_string) + return stop + + # the list of Positionals left to be parsed; this is modified + # by consume_positionals() + positionals = self._get_positional_actions() + + # function to convert arg_strings into positional actions + def consume_positionals(start_index): + # match as many Positionals as possible + match_partial = self._match_arguments_partial + selected_pattern = arg_strings_pattern[start_index:] + arg_counts = match_partial(positionals, selected_pattern) + + # slice off the appropriate arg strings for each Positional + # and add the Positional and its args to the list + for action, arg_count in zip(positionals, arg_counts): + args = arg_strings[start_index : start_index + arg_count] + start_index += arg_count + take_action(action, args) + + # slice off the Positionals that we just parsed and return the + # index at which the Positionals' string args stopped + positionals[:] = positionals[len(arg_counts) :] + return start_index + + # consume Positionals and Optionals alternately, until we have + # passed the last option string + extras = [] + start_index = 0 + if option_string_indices: + max_option_string_index = max(option_string_indices) + else: + max_option_string_index = -1 + while start_index <= max_option_string_index: + # consume any Positionals preceding the next option + next_option_string_index = min( + [index for index in option_string_indices if index >= start_index] + ) + if start_index != next_option_string_index: + positionals_end_index = consume_positionals(start_index) + + # only try to parse the next optional if we didn't consume + # the option string during the positionals parsing + if positionals_end_index > start_index: + start_index = positionals_end_index + continue + else: + start_index = positionals_end_index + + # if we consumed all the positionals we could and we're not + # at the index of an option string, there were extra arguments + if start_index not in option_string_indices: + strings = arg_strings[start_index:next_option_string_index] + extras.extend(strings) + start_index = next_option_string_index + + # consume the next optional and any arguments for it + start_index = consume_optional(start_index) + + # consume any positionals following the last Optional + stop_index = consume_positionals(start_index) + + # if we didn't consume all the argument strings, there were extras + extras.extend(arg_strings[stop_index:]) + + # make sure all required actions were present and also convert + # action defaults which were not given as arguments + required_actions = [] + for action in self._actions: + if action not in seen_actions: + if action.required: + required_actions.append(argparse._get_action_name(action)) + else: + # Convert action default now instead of doing it before + # parsing arguments to avoid calling convert functions + # twice (which may fail) if the argument was given, but + # only if it was defined already in the namespace + if ( + action.default is not None + and isinstance(action.default, str) + and hasattr(namespace, action.dest) + and action.default is getattr(namespace, action.dest) + ): + setattr( + namespace, + action.dest, + self._get_value(action, action.default), + ) + + if required_actions: + self.error( + _("the following arguments are required: %s") + % ", ".join(required_actions) + ) + + # make sure all required groups had one option present + for group in self._mutually_exclusive_groups: + if group.required: + for action in group._group_actions: + if action in seen_non_default_actions: + break + + # if no actions were used, report the error + else: + names = [ + argparse._get_action_name(action) + for action in group._group_actions + if action.help is not argparse.SUPPRESS + ] + msg = _("one of the arguments %s is required") + self.error(msg % " ".join(names)) # type: ignore + + # return the updated namespace and the extra arguments + return namespace, extras + + def _print_usage_succinct(self, console: Console) -> None: + """Print usage, but abridged if too long.""" + usage = self.format_usage().strip() + "\n" + if len(usage) < 400: + print(usage) + else: # pragma: no cover + console.print(f"[bold]help:[/bold] {self.prog} --help\n") + + @override + def error(self, message: str) -> NoReturn: + """Improve error messages from argparse. + + error(message: string) + + Prints a usage message incorporating the message to stderr and + exits. + + If you override this in a subclass, it should not return -- it + should either exit or raise an exception. + """ + + console = Console(theme=THEME.as_rich_theme()) + self._print_usage_succinct(console) + + extra_info: List[RenderableType] = [] + global global_unrecognized_args + if len(global_unrecognized_args) == 0 and message.startswith( + "unrecognized arguments: " + ): + global_unrecognized_args = message.partition(":")[2].strip().split(" ") + + if len(global_unrecognized_args) > 0: + message = f"Unrecognized arguments: {' '.join(global_unrecognized_args)}" + unrecognized_arguments = [ + arg + for arg in global_unrecognized_args + # If we pass in `--spell-chekc on`, we only want `spell-chekc` and not + # `on`. + if arg.startswith("--") + ] + + # Argument name => subcommands it came from. + arguments: List[_ArgumentInfo] = [] + has_subcommands = False + same_exists = False + + def _recursive_arg_search( + parser_spec: ParserSpecification, + subcommands: str, + subcommand_match_score: float, + ) -> None: + """Find all possible arguments that could have been passed in.""" + + # When tyro.conf.ConsolidateSubcommandArgs is turned on, arguments will + # only appear in the help message for "leaf" subparsers. + help_flag = ( + " (other subcommands) --help" + if parser_spec.consolidate_subcommand_args + and parser_spec.subparsers is not None + else " --help" + ) + for arg in parser_spec.args: + if arg.field.is_positional() or arg.lowered.is_fixed(): + # Skip positional arguments. + continue + + # Skip suppressed arguments. + if conf.Suppress in arg.field.markers or ( + conf.SuppressFixed in arg.field.markers + and conf.Fixed in arg.field.markers + ): + continue + + option_strings = (arg.lowered.name_or_flag,) + + # Handle actions, eg BooleanOptionalAction will map ("--flag",) to + # ("--flag", "--no-flag"). + if ( + arg.lowered.action is not None + # Actions are sometimes strings in Python 3.7, eg "append". + # We'll ignore these, but this kind of thing is a good reason + # for just forking argparse. + and callable(arg.lowered.action) + ): + option_strings = arg.lowered.action( + option_strings, dest="" # dest should not matter. + ).option_strings + + arguments.append( + _ArgumentInfo( + # Currently doesn't handle actions well, eg boolean optional + # arguments. + option_strings, + metavar=arg.lowered.metavar, + usage_hint=subcommands + help_flag, + help=arg.lowered.help, + subcommand_match_score=subcommand_match_score, + ) + ) + + # An unrecognized argument. + nonlocal same_exists + if ( + not same_exists + and arg.lowered.name_or_flag in unrecognized_arguments + ): + same_exists = True + + if parser_spec.subparsers is not None: + nonlocal has_subcommands + has_subcommands = True + for ( + subparser_name, + subparser, + ) in parser_spec.subparsers.parser_from_name.items(): + _recursive_arg_search( + subparser, + subcommands + " " + subparser_name, + subcommand_match_score=subcommand_match_score + + (1 if subparser_name in self._args else -0.001), + ) + + _recursive_arg_search( + self._parser_specification, + # Remove other subcommands. + self.prog.split(" ")[0], + 0, + ) + + if has_subcommands and same_exists: + misplaced_arguments = message.partition(":")[2].strip() + message = ( + "unrecognized or misplaced arguments: " + misplaced_arguments + if " " in misplaced_arguments + else "unrecognized or misplaced argument: " + misplaced_arguments + ) + + # Show similar arguments for keyword options. + for unrecognized_argument in unrecognized_arguments: + # Sort arguments by similarity. + scored_arguments: List[Tuple[_ArgumentInfo, float]] = [] + for argument in arguments: + # Compute a score for each argument. + assert unrecognized_argument.startswith("--") + + def get_score(option_string: str) -> float: + if option_string.endswith( + unrecognized_argument[2:] + ) or option_string.startswith(unrecognized_argument[2:]): + return 0.9 + elif len(unrecognized_argument) >= 4 and all( + map( + lambda part: part in option_string, + unrecognized_argument[2:].split("."), + ) + ): + return 0.9 + else: + return difflib.SequenceMatcher( + a=unrecognized_argument, b=option_string + ).ratio() + + scored_arguments.append( + (argument, max(map(get_score, argument.option_strings))) + ) + + # Add information about similar arguments. + prev_arg_option_strings: Optional[Tuple[str, ...]] = None + show_arguments: List[_ArgumentInfo] = [] + unique_counter = 0 + for argument, score in ( + # Sort scores greatest to least. + sorted( + scored_arguments, + key=lambda arg_score: ( + # Highest scores first. + -arg_score[1], + # Prefer arguments available in the currently specified + # subcommands. + -arg_score[0].subcommand_match_score, + # Cluster by flag name, usage hint, help message. + arg_score[0].option_strings[0], + arg_score[0].usage_hint, + arg_score[0].help, + ), + ) + ): + if score < 0.8: + break + if ( + score < 0.9 + and unique_counter >= 3 + and prev_arg_option_strings != argument.option_strings + ): + break + unique_counter += prev_arg_option_strings != argument.option_strings + + show_arguments.append(argument) + prev_arg_option_strings = argument.option_strings + + prev_arg_option_strings = None + prev_argument_help: Optional[str] = None + same_counter = 0 + dots_printed = False + if len(show_arguments) > 0: + # Add a header before the first similar argument. + extra_info.append(Rule(style=Style(color="red"))) + extra_info.append( + "Similar arguments:" + if len(unrecognized_arguments) == 1 + else f"Arguments similar to {unrecognized_argument}:" + ) + + unique_counter = 0 + for argument in show_arguments: + same_counter += 1 + if argument.option_strings != prev_arg_option_strings: + same_counter = 0 + if unique_counter >= 10: + break + unique_counter += 1 + + # For arguments with the same name, only show a limited number of + # subcommands / help messages. + if ( + len(show_arguments) >= 8 + and same_counter >= 4 + and argument.option_strings == prev_arg_option_strings + ): + if not dots_printed: + extra_info.append( + Padding( + "[...]", + (0, 0, 0, 12), + ) + ) + dots_printed = True + continue + + if not ( + has_subcommands + and argument.option_strings == prev_arg_option_strings + ): + extra_info.append( + Padding( + "[bold]" + + ( + ", ".join(argument.option_strings) + if argument.metavar is None + else ", ".join(argument.option_strings) + + " " + + argument.metavar + ) + + "[/bold]", + (0, 0, 0, 4), + ) + ) + + # Uncomment to show similarity metric. + # extra_info.append( + # Padding( + # f"[green]Similarity: {score:.02f}[/green]", (0, 0, 0, 8) + # ) + # ) + + if argument.help is not None and ( + # Only print help messages if it's not the same as the previous + # one. + argument.help != prev_argument_help + or argument.option_strings != prev_arg_option_strings + ): + extra_info.append(Padding(argument.help, (0, 0, 0, 8))) + + # Show the subcommand that this argument is available in. + if has_subcommands: + extra_info.append( + Padding( + f"in [green]{argument.usage_hint}[/green]", + (0, 0, 0, 12), + ) + ) + + prev_arg_option_strings = argument.option_strings + prev_argument_help = argument.help + + # print(self._parser_specification) + console.print( + Panel( + Group( + f"{message[0].upper() + message[1:]}" if len(message) > 0 else "", + *extra_info, + ), + title="[bold]Parsing error[/bold]", + title_align="left", + border_style=Style(color="bright_red"), + ) + ) + sys.exit(2) + + class TyroArgparseHelpFormatter(argparse.RawDescriptionHelpFormatter): def __init__(self, prog: str): indent_increment = 4 @@ -143,6 +754,7 @@ def __init__(self, prog: str): super().__init__(prog, indent_increment, max_help_position, width) + @override def _format_args(self, action, default_metavar): """Override _format_args() to ignore nargs and always expect single string metavars.""" @@ -166,6 +778,7 @@ def _format_args(self, action, default_metavar): ) return out + @override def add_argument(self, action): # pragma: no cover # Patch to avoid super long arguments from shifting the helptext of all of the # fields. @@ -186,9 +799,11 @@ def _split_lines(self, text, width): del textwrap.len # type: ignore return out + @override def _fill_text(self, text, width, indent): return "".join(indent + line for line in text.splitlines(keepends=True)) + @override def format_help(self): # Try with and without a fixed help position, then return the shorter help # message. @@ -210,6 +825,7 @@ def format_help(self): else: return out + @override class _Section(object): def __init__(self, formatter, parent, heading=None): self.formatter = formatter @@ -438,9 +1054,9 @@ def _tyro_format_nonroot(self): max_width = max(map(len, lines)) if self.formatter._tyro_rule is None: - # Note: we don't use rich.rule.Rule() because this will make all of - # the panels expand to fill the full width of the console. (this only - # impacts single-column layouts) + # We don't use rich.rule.Rule() because this will make all of the panels + # expand to fill the full width of the console. This only impacts + # single-column layouts. self.formatter._tyro_rule = Text.from_ansi( "─" * max_width, style=THEME.border, overflow="crop" ) @@ -490,7 +1106,7 @@ def _format_actions_usage(self, actions, groups): # pragma: no cover group_action_count - suppressed_actions_count ) - if not group.required: + if not group.required: # type: ignore if start in inserts: inserts[start] += " [" else: @@ -580,3 +1196,16 @@ def _format_actions_usage(self, actions, groups): # pragma: no cover # return the text return text + + @override + def _format_usage(self, usage, actions, groups, prefix) -> str: + # Format the usage label. + if prefix is None: + prefix = str_from_rich("[bold]usage[/bold]: ") + usage = super()._format_usage( + usage, + actions, + groups, + prefix, + ) + return "\n\n" + usage diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 348bcd0de..55f3c384f 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -9,10 +9,12 @@ import itertools import shlex from typing import ( + TYPE_CHECKING, Any, Callable, Dict, Iterable, + List, Mapping, Optional, Sequence, @@ -29,12 +31,15 @@ from ._typing import TypeForm from .conf import _markers -try: - # Python >=3.8. - from functools import cached_property -except ImportError: - # Python 3.7. - from backports.cached_property import cached_property # type: ignore +if TYPE_CHECKING: + cached_property = property +else: + try: + # Python >=3.8. + from functools import cached_property + except ImportError: + # Python 3.7. + from backports.cached_property import cached_property # type: ignore _T = TypeVar("_T") @@ -207,10 +212,10 @@ def is_fixed(self) -> bool: name_or_flag: str = "" default: Optional[Any] = None dest: Optional[str] = None - required: bool = False + required: Optional[bool] = None action: Optional[Any] = None nargs: Optional[Union[int, str]] = None - choices: Optional[Set[Any]] = None + choices: Optional[Union[Set[str], List[str]]] = None # Note: unlike in vanilla argparse, our metavar is always a string. We handle # sequences, multiple arguments, etc, manually. metavar: Optional[str] = None diff --git a/tyro/_calling.py b/tyro/_calling.py index 5931b5bf9..9eca6fe0f 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -3,6 +3,7 @@ from __future__ import annotations +import dataclasses from typing import Any, Callable, Dict, List, Sequence, Set, Tuple, TypeVar, Union from typing_extensions import get_args @@ -11,10 +12,14 @@ from .conf import _markers +@dataclasses.dataclass(frozen=True) class InstantiationError(Exception): """Exception raised when instantiation fail; this typically means that values from the CLI are invalid.""" + message: str + arg: _arguments.ArgumentDefinition + T = TypeVar("T") @@ -94,7 +99,8 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: value = arg.lowered.instantiator(value) except ValueError as e: raise InstantiationError( - f"Parsing error for {arg.lowered.name_or_flag}: {e.args[0]}" + e.args[0], + arg, ) else: assert arg.field.default not in _fields.MISSING_SINGLETONS @@ -102,8 +108,9 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: parsed_value = value_from_prefixed_field_name.get(prefixed_field_name) if parsed_value not in _fields.MISSING_SINGLETONS: raise InstantiationError( - f"{arg.lowered.name_or_flag}={parsed_value} was passed in, but" - " is a fixed argument that cannot be parsed" + f"{arg.lowered.name_or_flag} was passed in, but" + " is a fixed argument that cannot be parsed", + arg, ) elif ( prefixed_field_name diff --git a/tyro/_cli.py b/tyro/_cli.py index 1837bd912..c3d3c7996 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -264,10 +264,8 @@ def _cli_impl( if deprecated_kwargs.get("avoid_subparsers", False): f = conf.AvoidSubcommands[f] # type: ignore warnings.warn( - ( - "`avoid_subparsers=` is deprecated! use `tyro.conf.AvoidSubparsers[]`" - " instead." - ), + "`avoid_subparsers=` is deprecated! use `tyro.conf.AvoidSubparsers[]`" + " instead.", stacklevel=2, ) @@ -355,7 +353,7 @@ def _cli_impl( _arguments.USE_RICH = True # Map a callable to the relevant CLI arguments + subparsers. - parser_definition = _parsers.ParserSpecification.from_callable_or_type( + parser_spec = _parsers.ParserSpecification.from_callable_or_type( f, description=description, parent_classes=set(), # Used for recursive calls. @@ -366,16 +364,19 @@ def _cli_impl( # Generate parser! with _argparse_formatter.ansi_context(): - parser = argparse.ArgumentParser( + parser = _argparse_formatter.TyroArgumentParser( prog=prog, formatter_class=_argparse_formatter.TyroArgparseHelpFormatter, allow_abbrev=False, ) - parser_definition.apply(parser) + parser._parser_specification = parser_spec + parser._parsing_known_args = return_unknown_args + parser._args = args + parser_spec.apply(parser) # Print help message when no arguments are passed in. (but arguments are # expected) - if len(args) == 0 and parser_definition.has_required_args: + if len(args) == 0 and parser_spec.has_required_args: args = ["--help"] if return_parser: @@ -410,7 +411,7 @@ def _cli_impl( root_prefix=f"tyro_{parser.prog}", ) ) - raise SystemExit() + sys.exit() if return_unknown_args: namespace, unknown_args = parser.parse_known_args(args=args) @@ -429,17 +430,55 @@ def _cli_impl( # Attempt to call `f` using whatever was passed in. out, consumed_keywords = _calling.call_from_args( f, - parser_definition, + parser_spec, default_instance_internal, value_from_prefixed_field_name, field_name_prefix="", ) except _calling.InstantiationError as e: + assert isinstance(e, _calling.InstantiationError) + # Emulate argparse's error behavior when invalid arguments are passed in. - parser.print_usage() - print() - print(e.args[0]) - raise SystemExit() + from rich.console import Console, Group, RenderableType + from rich.padding import Padding + from rich.panel import Panel + from rich.rule import Rule + from rich.style import Style + + from ._argparse_formatter import THEME + + console = Console(theme=THEME.as_rich_theme()) + parser._print_usage_succinct(console) + console.print( + Panel( + Group( + "[bright_red][bold]Error parsing" + f" {e.arg.lowered.name_or_flag}[/bold]:[/bright_red] {e.message}", + *cast( # Cast to appease mypy... + List[RenderableType], + ( + [] + if e.arg.lowered.help is None + else [ + Rule(style=Style(color="red")), + "Argument helptext:", + Padding( + Group( + f"{e.arg.lowered.name_or_flag} [bold]{e.arg.lowered.metavar}[/bold]", + e.arg.lowered.help, + ), + pad=(0, 0, 0, 4), + ), + ] + ), + ), + ), + title="[bold]Value error[/bold]", + title_align="left", + border_style=Style(color="red"), + ) + ) + sys.exit(2) assert len(value_from_prefixed_field_name.keys() - consumed_keywords) == 0, ( f"Parsed {value_from_prefixed_field_name.keys()}, but only consumed" diff --git a/tyro/_fields.py b/tyro/_fields.py index 25642ea3e..60ca09b79 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -839,11 +839,9 @@ def _get_dataclass_field_default( return getattr(parent_default_instance, field.name) else: warnings.warn( - ( - f"Could not find field {field.name} in default instance" - f" {parent_default_instance}, which has" - f" type {type(parent_default_instance)}," - ), + f"Could not find field {field.name} in default instance" + f" {parent_default_instance}, which has" + f" type {type(parent_default_instance)},", stacklevel=2, ) diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 9fc35b6a2..1fd5e898c 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -46,6 +46,7 @@ Iterable, List, Optional, + Set, Tuple, TypeVar, Union, @@ -93,7 +94,7 @@ class InstantiatorMetadata: # Unlike in vanilla argparse, our metavar is always a string. We handle # sequences, multiple arguments, etc, manually. metavar: str - choices: Optional[Tuple[str, ...]] + choices: Optional[Set[str]] action: Optional[Literal["append"]] def check_choices(self, strings: List[str]) -> None: @@ -175,7 +176,9 @@ def instantiator(strings: List[str]) -> None: return instantiator, InstantiatorMetadata( nargs=1, metavar="{None}", - choices=("None",), + choices={ + "None", + }, action=None, ) @@ -246,7 +249,7 @@ def instantiator_base_case(strings: List[str]) -> Any: if auto_choices is None else "{" + ",".join(map(str, auto_choices)) + "}" ), - choices=auto_choices, + choices=None if auto_choices is None else set(auto_choices), action=None, ) @@ -671,7 +674,7 @@ def _instantiator_from_literal( InstantiatorMetadata( nargs=1, metavar="{" + ",".join(str_choices) + "}", - choices=str_choices, + choices=set(str_choices), action=None, ), ) diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 3a9b5aa5d..a8cb131ba 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -357,8 +357,12 @@ def from_field( for typ in get_args(typ) ] options = [ - # Cast seems unnecessary but needed in mypy... (1.4.1) - cast(Callable, none_proxy) if o is type(None) else o + ( + # Cast seems unnecessary but needed in mypy... (1.4.1) + cast(Callable, none_proxy) + if o is type(None) + else o + ) for o in options ] if not all( @@ -534,7 +538,16 @@ def apply( name, formatter_class=_argparse_formatter.TyroArgparseHelpFormatter, help=helptext, + allow_abbrev=False, ) + + # Attributes used for error message generation. + assert isinstance(subparser, _argparse_formatter.TyroArgumentParser) + assert isinstance(parent_parser, _argparse_formatter.TyroArgumentParser) + subparser._parsing_known_args = parent_parser._parsing_known_args + subparser._parser_specification = parent_parser._parser_specification + subparser._args = parent_parser._args + subparser_tree_leaves.extend(subparser_def.apply(subparser)) return tuple(subparser_tree_leaves) diff --git a/tyro/_strings.py b/tyro/_strings.py index 0abfce80a..e6b9527a2 100644 --- a/tyro/_strings.py +++ b/tyro/_strings.py @@ -3,7 +3,7 @@ import functools import re import textwrap -from typing import Iterable, List, Sequence, Tuple, Type, Union +from typing import Iterable, List, Sequence, Tuple, Type from typing_extensions import get_args, get_origin From 0fb5289988eb89e1cd169a66671855df78f0b61d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 2 Sep 2023 00:24:02 -0700 Subject: [PATCH 315/491] Typo --- tyro/conf/_confstruct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tyro/conf/_confstruct.py b/tyro/conf/_confstruct.py index 1cf4f0454..e1ce4d06a 100644 --- a/tyro/conf/_confstruct.py +++ b/tyro/conf/_confstruct.py @@ -46,7 +46,7 @@ def subcommand( NestedTypeA, subcommand("a", ...) ], Annotated[ - NestedTypeA, subcommand("b", ...) + NestedTypeB, subcommand("b", ...) ], ] ) From 5498eec8807771a007b77cf91f1da531473fb679 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 5 Sep 2023 23:50:33 -0700 Subject: [PATCH 316/491] Add conf.PositionalRequiredArgs (#66) * Add conf.PositionalRequiredArgs * Typo --- tests/test_conf.py | 14 ++++++++++++++ tyro/_fields.py | 5 +++++ tyro/conf/__init__.py | 1 + tyro/conf/_markers.py | 3 +++ 4 files changed, 23 insertions(+) diff --git a/tests/test_conf.py b/tests/test_conf.py index 2e93d9a9a..0bf4c7ad4 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -568,6 +568,20 @@ def main(x: tyro.conf.Positional[int], y: int) -> int: assert tyro.cli(main, args="--y 3 5".split(" ")) == 8 +def test_positional_required_args() -> None: + @dataclasses.dataclass + class Args: + x: int + y: int = 3 + + assert tyro.cli( + tyro.conf.PositionalRequiredArgs[Args], args="5 --y 3".split(" ") + ) == Args(5, 3) + assert tyro.cli( + tyro.conf.PositionalRequiredArgs[Args], args="--y 3 5".split(" ") + ) == Args(5, 3) + + def test_positional_order_swap() -> None: def main(x: int, y: tyro.conf.Positional[int]) -> int: return x + y diff --git a/tyro/_fields.py b/tyro/_fields.py index 60ca09b79..37420431e 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -110,6 +110,11 @@ def is_positional(self) -> bool: _markers.Positional in self.markers # Dummy dataclasses should have a single positional field. or self.name == _strings.dummy_field_name + or ( + # Make required arguments positional. + _markers.PositionalRequiredArgs in self.markers + and self.default in MISSING_SINGLETONS + ) ) def is_positional_call(self) -> bool: diff --git a/tyro/conf/__init__.py b/tyro/conf/__init__.py index 1a1ba8c58..ef9698523 100644 --- a/tyro/conf/__init__.py +++ b/tyro/conf/__init__.py @@ -17,6 +17,7 @@ from ._markers import OmitArgPrefixes as OmitArgPrefixes from ._markers import OmitSubcommandPrefixes as OmitSubcommandPrefixes from ._markers import Positional as Positional +from ._markers import PositionalRequiredArgs as PositionalRequiredArgs from ._markers import Suppress as Suppress from ._markers import SuppressFixed as SuppressFixed from ._markers import UseAppendAction as UseAppendAction diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index d791b8bb7..8ee8bcd9c 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -18,6 +18,9 @@ """A type `T` can be annotated as `Positional[T]` if we want to parse it as a positional argument.""" +PositionalRequiredArgs = Annotated[T, None] +"""Make all arguments without defaults positional.""" + # Private marker. For when an argument is not only positional in the CLI, but also in # the callable. _PositionalCall = Annotated[T, None] From 4d5f0581e3e1416ff4cb4831172645fc9a1e00a0 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 5 Sep 2023 23:55:25 -0700 Subject: [PATCH 317/491] More error message tweaks (#67) * More error message tweaks * Comments, subcommand verbosity * mypy + black fixes * Nits, version bump --- pyproject.toml | 2 +- tests/test_errors.py | 26 +-- tyro/_argparse_formatter.py | 350 +++++++++++++++++++++++------------- tyro/_cli.py | 7 +- 4 files changed, 240 insertions(+), 145 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 701c36333..3508560eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.6" +version = "0.5.7" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/tests/test_errors.py b/tests/test_errors.py index bf80bd25b..3ff99852b 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -170,8 +170,8 @@ class Class: assert "Similar arguments" in error # --reward.track should appear in both the usage string and as a similar argument. - assert error.count("--reward.track") == 2 - assert error.count("--help") == 0 + assert error.count("--reward.track") == 1 + assert error.count("--help") == 1 def test_similar_arguments_subcommands() -> None: @@ -195,7 +195,7 @@ class ClassB: assert "Unrecognized argument" in error assert "Similar arguments:" in error assert error.count("--reward.track") == 1 - assert error.count("--help") == 2 + assert error.count("--help") == 3 def test_similar_arguments_subcommands_multiple() -> None: @@ -221,7 +221,7 @@ class ClassB: assert "Arguments similar to --reward.trac" in error assert error.count("--reward.track {True,False}") == 1 assert error.count("--reward.trace INT") == 1 - assert error.count("--help") == 4 + assert error.count("--help") == 5 def test_similar_arguments_subcommands_multiple_contains_match() -> None: @@ -247,7 +247,7 @@ class ClassB: assert "Similar arguments" in error assert error.count("--reward.track {True,False}") == 1 assert error.count("--reward.trace INT") == 1 - assert error.count("--help") == 4 # 2 subcommands * 2 arguments. + assert error.count("--help") == 5 # 2 subcommands * 2 arguments + usage hint. def test_similar_arguments_subcommands_multiple_contains_match_alt() -> None: @@ -272,7 +272,9 @@ class ClassB: assert "Unrecognized argument" in error assert "Similar arguments" in error assert error.count("--reward.track {True,False}") == 1 - assert error.count("--help") == 2 # Should show two possible subcommands. + assert ( + error.count("--help") == 3 + ) # Should show two possible subcommands + usage hint. def test_similar_arguments_subcommands_overflow_different() -> None: @@ -313,7 +315,7 @@ class ClassB: assert "Similar arguments" in error assert error.count("--reward.track") == 10 assert "[...]" not in error - assert error.count("--help") == 20 + assert error.count("--help") == 21 target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): @@ -323,7 +325,7 @@ class ClassB: # Usage print should be clipped. error = target.getvalue() - assert "help:" in error + assert "For full helptext, run" in error def test_similar_arguments_subcommands_overflow_same() -> None: @@ -381,7 +383,7 @@ class ClassI: assert "Similar arguments" in error assert error.count("--reward.track") == 1 assert "[...]" in error - assert error.count("--help") == 4 + assert error.count("--help") == 5 def test_similar_arguments_subcommands_overflow_same_startswith_multiple() -> None: @@ -441,7 +443,7 @@ class ClassI: assert error.count("--rewar") == 1 assert "rewarde" not in error assert "[...]" in error - assert error.count("--help") == 4 + assert error.count("--help") == 5 def test_similar_flag() -> None: @@ -458,8 +460,8 @@ class Args: error = target.getvalue() - # Printed in the usage message. - assert error.count("--flag | --no-flag") == 1 + # We don't print usage text anymore. + assert error.count("--flag | --no-flag") == 0 # Printed in the similar argument list. assert error.count("--flag, --no-flag") == 1 diff --git a/tyro/_argparse_formatter.py b/tyro/_argparse_formatter.py index 808902ebf..3e0a6a274 100644 --- a/tyro/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -11,6 +11,9 @@ TODO: the current implementation should be robust given our test coverage, but unideal long-term. We should just maintain our own fork of argparse. """ + +from __future__ import annotations + import argparse import contextlib import dataclasses @@ -20,7 +23,7 @@ import shutil import sys from gettext import gettext as _ -from typing import Any, Generator, List, NoReturn, Optional, Tuple +from typing import Any, Dict, Generator, List, NoReturn, Optional, Set, Tuple from rich.columns import Columns from rich.console import Console, Group, RenderableType @@ -73,6 +76,109 @@ def set_accent_color(accent_color: Optional[str]) -> None: ) +def recursive_arg_search( + args: List[str], + parser_spec: ParserSpecification, + prog: str, + unrecognized_arguments: Set[str], +) -> Tuple[List[_ArgumentInfo], bool, bool]: + """Recursively search for arguments in a ParserSpecification. Used for error message + printing. + + Returns a list of arguments, whether the parser has subcommands or not, and -- if + `unrecognized_arguments` is passed in --- whether an unrecognized argument exists + under a different subparser. + + Args: + args: Arguments being parsed. Used for heuristics on subcommands. + parser_spec: Argument parser specification. + subcommands: Prog corresponding to parser_spec. + unrecognized_arguments: Used for same_exists return value. + """ + # Argument name => subcommands it came from. + arguments: List[_ArgumentInfo] = [] + has_subcommands = False + same_exists = False + + def _recursive_arg_search( + parser_spec: ParserSpecification, + prog: str, + subcommand_match_score: float, + ) -> None: + """Find all possible arguments that could have been passed in.""" + + # When tyro.conf.ConsolidateSubcommandArgs is turned on, arguments will + # only appear in the help message for "leaf" subparsers. + help_flag = ( + " (other subcommands) --help" + if parser_spec.consolidate_subcommand_args + and parser_spec.subparsers is not None + else " --help" + ) + for arg in parser_spec.args: + if arg.field.is_positional() or arg.lowered.is_fixed(): + # Skip positional arguments. + continue + + # Skip suppressed arguments. + if conf.Suppress in arg.field.markers or ( + conf.SuppressFixed in arg.field.markers + and conf.Fixed in arg.field.markers + ): + continue + + option_strings = (arg.lowered.name_or_flag,) + + # Handle actions, eg BooleanOptionalAction will map ("--flag",) to + # ("--flag", "--no-flag"). + if ( + arg.lowered.action is not None + # Actions are sometimes strings in Python 3.7, eg "append". + # We'll ignore these, but this kind of thing is a good reason + # for just forking argparse. + and callable(arg.lowered.action) + ): + option_strings = arg.lowered.action( + option_strings, dest="" # dest should not matter. + ).option_strings + + arguments.append( + _ArgumentInfo( + # Currently doesn't handle actions well, eg boolean optional + # arguments. + option_strings, + metavar=arg.lowered.metavar, + usage_hint=prog + help_flag, + help=arg.lowered.help, + subcommand_match_score=subcommand_match_score, + ) + ) + + # An unrecognized argument. + nonlocal same_exists + if not same_exists and arg.lowered.name_or_flag in unrecognized_arguments: + same_exists = True + + if parser_spec.subparsers is not None: + nonlocal has_subcommands + has_subcommands = True + for ( + subparser_name, + subparser, + ) in parser_spec.subparsers.parser_from_name.items(): + _recursive_arg_search( + subparser, + prog + " " + subparser_name, + # Leaky (!!) heuristic for if this subcommand is matched or not. + subcommand_match_score=subcommand_match_score + + (1 if subparser_name in args else -0.001), + ) + + _recursive_arg_search(parser_spec, prog, 0) + + return arguments, has_subcommands, same_exists + + # TODO: this is a prototype; for a v1.0.0 release we should revisit whether the global # state here is acceptable or not. THEME = TyroTheme() @@ -442,14 +548,6 @@ def consume_positionals(start_index): # return the updated namespace and the extra arguments return namespace, extras - def _print_usage_succinct(self, console: Console) -> None: - """Print usage, but abridged if too long.""" - usage = self.format_usage().strip() + "\n" - if len(usage) < 400: - print(usage) - else: # pragma: no cover - console.print(f"[bold]help:[/bold] {self.prog} --help\n") - @override def error(self, message: str) -> NoReturn: """Improve error messages from argparse. @@ -464,7 +562,6 @@ def error(self, message: str) -> NoReturn: """ console = Console(theme=THEME.as_rich_theme()) - self._print_usage_succinct(console) extra_info: List[RenderableType] = [] global global_unrecognized_args @@ -473,117 +570,33 @@ def error(self, message: str) -> NoReturn: ): global_unrecognized_args = message.partition(":")[2].strip().split(" ") + message_title = "Parsing error" + if len(global_unrecognized_args) > 0: + message_title = "Unrecognized arguments" message = f"Unrecognized arguments: {' '.join(global_unrecognized_args)}" - unrecognized_arguments = [ + unrecognized_arguments = set( arg for arg in global_unrecognized_args # If we pass in `--spell-chekc on`, we only want `spell-chekc` and not # `on`. if arg.startswith("--") - ] - - # Argument name => subcommands it came from. - arguments: List[_ArgumentInfo] = [] - has_subcommands = False - same_exists = False - - def _recursive_arg_search( - parser_spec: ParserSpecification, - subcommands: str, - subcommand_match_score: float, - ) -> None: - """Find all possible arguments that could have been passed in.""" - - # When tyro.conf.ConsolidateSubcommandArgs is turned on, arguments will - # only appear in the help message for "leaf" subparsers. - help_flag = ( - " (other subcommands) --help" - if parser_spec.consolidate_subcommand_args - and parser_spec.subparsers is not None - else " --help" - ) - for arg in parser_spec.args: - if arg.field.is_positional() or arg.lowered.is_fixed(): - # Skip positional arguments. - continue - - # Skip suppressed arguments. - if conf.Suppress in arg.field.markers or ( - conf.SuppressFixed in arg.field.markers - and conf.Fixed in arg.field.markers - ): - continue - - option_strings = (arg.lowered.name_or_flag,) - - # Handle actions, eg BooleanOptionalAction will map ("--flag",) to - # ("--flag", "--no-flag"). - if ( - arg.lowered.action is not None - # Actions are sometimes strings in Python 3.7, eg "append". - # We'll ignore these, but this kind of thing is a good reason - # for just forking argparse. - and callable(arg.lowered.action) - ): - option_strings = arg.lowered.action( - option_strings, dest="" # dest should not matter. - ).option_strings - - arguments.append( - _ArgumentInfo( - # Currently doesn't handle actions well, eg boolean optional - # arguments. - option_strings, - metavar=arg.lowered.metavar, - usage_hint=subcommands + help_flag, - help=arg.lowered.help, - subcommand_match_score=subcommand_match_score, - ) - ) - - # An unrecognized argument. - nonlocal same_exists - if ( - not same_exists - and arg.lowered.name_or_flag in unrecognized_arguments - ): - same_exists = True - - if parser_spec.subparsers is not None: - nonlocal has_subcommands - has_subcommands = True - for ( - subparser_name, - subparser, - ) in parser_spec.subparsers.parser_from_name.items(): - _recursive_arg_search( - subparser, - subcommands + " " + subparser_name, - subcommand_match_score=subcommand_match_score - + (1 if subparser_name in self._args else -0.001), - ) - - _recursive_arg_search( - self._parser_specification, - # Remove other subcommands. - self.prog.split(" ")[0], - 0, + ) + arguments, has_subcommands, same_exists = recursive_arg_search( + args=self._args, + parser_spec=self._parser_specification, + prog=self.prog.partition(" ")[0], + unrecognized_arguments=unrecognized_arguments, ) if has_subcommands and same_exists: - misplaced_arguments = message.partition(":")[2].strip() - message = ( - "unrecognized or misplaced arguments: " + misplaced_arguments - if " " in misplaced_arguments - else "unrecognized or misplaced argument: " + misplaced_arguments - ) + message = f"Unrecognized or misplaced arguments: {' '.join(global_unrecognized_args)}" # Show similar arguments for keyword options. for unrecognized_argument in unrecognized_arguments: # Sort arguments by similarity. scored_arguments: List[Tuple[_ArgumentInfo, float]] = [] - for argument in arguments: + for arg_info in arguments: # Compute a score for each argument. assert unrecognized_argument.startswith("--") @@ -605,14 +618,14 @@ def get_score(option_string: str) -> float: ).ratio() scored_arguments.append( - (argument, max(map(get_score, argument.option_strings))) + (arg_info, max(map(get_score, arg_info.option_strings))) ) # Add information about similar arguments. prev_arg_option_strings: Optional[Tuple[str, ...]] = None show_arguments: List[_ArgumentInfo] = [] unique_counter = 0 - for argument, score in ( + for arg_info, score in ( # Sort scores greatest to least. sorted( scored_arguments, @@ -634,13 +647,13 @@ def get_score(option_string: str) -> float: if ( score < 0.9 and unique_counter >= 3 - and prev_arg_option_strings != argument.option_strings + and prev_arg_option_strings != arg_info.option_strings ): break - unique_counter += prev_arg_option_strings != argument.option_strings + unique_counter += prev_arg_option_strings != arg_info.option_strings - show_arguments.append(argument) - prev_arg_option_strings = argument.option_strings + show_arguments.append(arg_info) + prev_arg_option_strings = arg_info.option_strings prev_arg_option_strings = None prev_argument_help: Optional[str] = None @@ -656,9 +669,9 @@ def get_score(option_string: str) -> float: ) unique_counter = 0 - for argument in show_arguments: + for arg_info in show_arguments: same_counter += 1 - if argument.option_strings != prev_arg_option_strings: + if arg_info.option_strings != prev_arg_option_strings: same_counter = 0 if unique_counter >= 10: break @@ -669,7 +682,7 @@ def get_score(option_string: str) -> float: if ( len(show_arguments) >= 8 and same_counter >= 4 - and argument.option_strings == prev_arg_option_strings + and arg_info.option_strings == prev_arg_option_strings ): if not dots_printed: extra_info.append( @@ -683,17 +696,17 @@ def get_score(option_string: str) -> float: if not ( has_subcommands - and argument.option_strings == prev_arg_option_strings + and arg_info.option_strings == prev_arg_option_strings ): extra_info.append( Padding( "[bold]" + ( - ", ".join(argument.option_strings) - if argument.metavar is None - else ", ".join(argument.option_strings) + ", ".join(arg_info.option_strings) + if arg_info.metavar is None + else ", ".join(arg_info.option_strings) + " " - + argument.metavar + + arg_info.metavar ) + "[/bold]", (0, 0, 0, 4), @@ -707,36 +720,115 @@ def get_score(option_string: str) -> float: # ) # ) - if argument.help is not None and ( + if arg_info.help is not None and ( # Only print help messages if it's not the same as the previous # one. - argument.help != prev_argument_help - or argument.option_strings != prev_arg_option_strings + arg_info.help != prev_argument_help + or arg_info.option_strings != prev_arg_option_strings ): - extra_info.append(Padding(argument.help, (0, 0, 0, 8))) + extra_info.append(Padding(arg_info.help, (0, 0, 0, 8))) # Show the subcommand that this argument is available in. if has_subcommands: extra_info.append( Padding( - f"in [green]{argument.usage_hint}[/green]", + f"in [green]{arg_info.usage_hint}[/green]", (0, 0, 0, 12), ) ) - prev_arg_option_strings = argument.option_strings - prev_argument_help = argument.help + prev_arg_option_strings = arg_info.option_strings + prev_argument_help = arg_info.help + + elif message.startswith("the following arguments are required:"): + message_title = "Required arguments" + + info_from_required_arg: Dict[str, Optional[_ArgumentInfo]] = {} + for arg in message.partition(":")[2].strip().split(", "): + info_from_required_arg[arg] = None + + arguments, has_subcommands, same_exists = recursive_arg_search( + args=self._args, + parser_spec=self._parser_specification, + prog=self.prog.partition(" ")[0], + unrecognized_arguments=set(), + ) + del same_exists + + for arg_info in arguments: + # Iterate over each option string separately. This can help us support + # aliases in the future. + for option_string in arg_info.option_strings: + # If the option string was found... + if option_string in info_from_required_arg and ( + # And it's the first time it was found... + info_from_required_arg[option_string] is None + # Or we found a better one... + or arg_info.subcommand_match_score + > info_from_required_arg[option_string].subcommand_match_score # type: ignore + ): + # Record the argument info. + info_from_required_arg[option_string] = arg_info + + # Try to print help text for required arguments. + first = True + for maybe_arg in info_from_required_arg.values(): + if maybe_arg is None: + # No argument info found. This will currently happen for + # subcommands. + continue + + if first: + extra_info.extend( + [ + Rule(style=Style(color="red")), + "Argument helptext:", + ] + ) + first = False + + extra_info.append( + Padding( + "[bold]" + + ( + ", ".join(maybe_arg.option_strings) + if maybe_arg.metavar is None + else ", ".join(maybe_arg.option_strings) + + " " + + maybe_arg.metavar + ) + + "[/bold]", + (0, 0, 0, 4), + ) + ) + if maybe_arg.help is not None: + extra_info.append(Padding(maybe_arg.help, (0, 0, 0, 8))) + if has_subcommands: + # We are explicit about where the argument helptext is being + # extracted from because the `subcommand_match_score` heuristic + # above is flawed. + # + # The stars really need to be aligned for it to fail, but this makes + # sure that if it does fail that it's obvious to the user. + extra_info.append( + Padding( + f"in [green]{maybe_arg.usage_hint}[/green]", + (0, 0, 0, 12), + ) + ) - # print(self._parser_specification) console.print( Panel( Group( f"{message[0].upper() + message[1:]}" if len(message) > 0 else "", *extra_info, + Rule(style=Style(color="red")), + f"For full helptext, run [bold]{self.prog} --help[/bold]", ), - title="[bold]Parsing error[/bold]", + title=f"[bold]{message_title}[/bold]", title_align="left", border_style=Style(color="bright_red"), + expand=False, ) ) sys.exit(2) diff --git a/tyro/_cli.py b/tyro/_cli.py index c3d3c7996..921c87dad 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -376,8 +376,8 @@ def _cli_impl( # Print help message when no arguments are passed in. (but arguments are # expected) - if len(args) == 0 and parser_spec.has_required_args: - args = ["--help"] + # if len(args) == 0 and parser_spec.has_required_args: + # args = ["--help"] if return_parser: _arguments.USE_RICH = True @@ -448,7 +448,6 @@ def _cli_impl( from ._argparse_formatter import THEME console = Console(theme=THEME.as_rich_theme()) - parser._print_usage_succinct(console) console.print( Panel( Group( @@ -469,6 +468,8 @@ def _cli_impl( ), pad=(0, 0, 0, 4), ), + Rule(style=Style(color="red")), + f"For full helptext, see [bold]{parser.prog} --help[/bold]", ] ), ), From bb20982385ef8665ed204823a331e2f606a41eb6 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 6 Sep 2023 00:30:19 -0700 Subject: [PATCH 318/491] Fix tyro.conf for pydantic>=2 (#69) * Fix `tyro.conf` for pydantic 2 * add test showing pydantic failing on positional annotation * Nit --------- Co-authored-by: Sander Teunissen --- .gitignore | 2 ++ pyproject.toml | 2 +- tests/test_attrs.py | 3 +-- tests/test_pydantic.py | 9 +++++++++ tyro/_fields.py | 5 +++++ 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 3abe36ed2..c0ac32cb7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +# pyenv and direnv alternative +.rtx.toml *.swp *.pyc *.egg-info diff --git a/pyproject.toml b/pyproject.toml index 3508560eb..67e134815 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ dev = [ # As of 7/27/2023, flax install fails for Python 3.7 without pinning to an # old version. But doing so breaks other Python versions. "flax>=0.6.9;python_version>='3.8'", - "pydantic>=1.10.2", + "pydantic>=2.3.0", "coverage[toml]>=6.5.0" ] diff --git a/tests/test_attrs.py b/tests/test_attrs.py index 459f63b8e..c8a2ed445 100644 --- a/tests/test_attrs.py +++ b/tests/test_attrs.py @@ -13,7 +13,7 @@ def test_attrs_basic() -> None: @attr.s class ManyTypesA: - i: int = attr.ib() + i: tyro.conf.Positional[int] = attr.ib() s: str = attr.ib() f: float = attr.ib() p: pathlib.Path = attr.ib() @@ -22,7 +22,6 @@ class ManyTypesA: assert tyro.cli( ManyTypesA, args=[ - "--i", "5", "--s", "5", diff --git a/tests/test_pydantic.py b/tests/test_pydantic.py index a296ceb55..18e2a51bf 100644 --- a/tests/test_pydantic.py +++ b/tests/test_pydantic.py @@ -97,3 +97,12 @@ def test_pydantic_field_helptext_from_docstring() -> None: assert "Documentation 1" in helptext assert "Documentation 2" in helptext assert "Documentation 3" in helptext + + +def test_pydantic_positional_annotation() -> None: + class AnnotatedAsPositional(BaseModel): + name: tyro.conf.Positional[str] + """This is annotated as a positional argument.""" + + result = tyro.cli(AnnotatedAsPositional, args=["myname"]) + assert isinstance(result, AnnotatedAsPositional) diff --git a/tyro/_fields.py b/tyro/_fields.py index 37420431e..3722c15ff 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -493,6 +493,11 @@ def _field_list_from_pydantic( FieldDefinition.make( name=name, typ=pd_field.annotation, + markers=tuple( + meta + for meta in pd_field.metadata + if isinstance(meta, _markers._Marker) + ), default=( MISSING_NONPROP if pd_field.is_required() From 33240361838207e0298f6b167474c70882605f8e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 6 Sep 2023 23:01:55 -0700 Subject: [PATCH 319/491] "Perhaps you meant:" --- pyproject.toml | 2 +- tests/test_errors.py | 12 ++++++------ tyro/_argparse_formatter.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 67e134815..7b8376a91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.7" +version = "0.5.8" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/tests/test_errors.py b/tests/test_errors.py index 3ff99852b..e96098024 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -167,7 +167,7 @@ class Class: error = target.getvalue() assert "Unrecognized argument" in error - assert "Similar arguments" in error + assert "Perhaps you meant:" in error # --reward.track should appear in both the usage string and as a similar argument. assert error.count("--reward.track") == 1 @@ -193,7 +193,7 @@ class ClassB: error = target.getvalue() assert "Unrecognized argument" in error - assert "Similar arguments:" in error + assert "Perhaps you meant::" in error assert error.count("--reward.track") == 1 assert error.count("--help") == 3 @@ -244,7 +244,7 @@ class ClassB: error = target.getvalue() assert "Unrecognized argument" in error - assert "Similar arguments" in error + assert "Perhaps you meant:" in error assert error.count("--reward.track {True,False}") == 1 assert error.count("--reward.trace INT") == 1 assert error.count("--help") == 5 # 2 subcommands * 2 arguments + usage hint. @@ -270,7 +270,7 @@ class ClassB: error = target.getvalue() assert "Unrecognized argument" in error - assert "Similar arguments" in error + assert "Perhaps you meant:" in error assert error.count("--reward.track {True,False}") == 1 assert ( error.count("--help") == 3 @@ -312,7 +312,7 @@ class ClassB: error = target.getvalue() assert "Unrecognized argument" in error - assert "Similar arguments" in error + assert "Perhaps you meant:" in error assert error.count("--reward.track") == 10 assert "[...]" not in error assert error.count("--help") == 21 @@ -380,7 +380,7 @@ class ClassI: error = target.getvalue() assert "Unrecognized argument" in error - assert "Similar arguments" in error + assert "Perhaps you meant:" in error assert error.count("--reward.track") == 1 assert "[...]" in error assert error.count("--help") == 5 diff --git a/tyro/_argparse_formatter.py b/tyro/_argparse_formatter.py index 3e0a6a274..5cbb57410 100644 --- a/tyro/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -663,7 +663,7 @@ def get_score(option_string: str) -> float: # Add a header before the first similar argument. extra_info.append(Rule(style=Style(color="red"))) extra_info.append( - "Similar arguments:" + "Perhaps you meant:" if len(unrecognized_arguments) == 1 else f"Arguments similar to {unrecognized_argument}:" ) From a47b2f88e5f5508185d8cc45f0aa6ff97ac47916 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 10 Sep 2023 00:58:10 -0700 Subject: [PATCH 320/491] Fix test --- tests/test_errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_errors.py b/tests/test_errors.py index e96098024..86acdde63 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -193,7 +193,7 @@ class ClassB: error = target.getvalue() assert "Unrecognized argument" in error - assert "Perhaps you meant::" in error + assert "Perhaps you meant:" in error assert error.count("--reward.track") == 1 assert error.count("--help") == 3 From c31ff2dc856bec76b610428f18ef99554a864e75 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 24 Sep 2023 17:14:37 -0700 Subject: [PATCH 321/491] Dependency cleanup --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7b8376a91..46412a835 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,16 +25,16 @@ classifiers = [ dependencies = [ "docstring-parser>=0.14.1", "typing-extensions>=4.3.0", - "PyYAML>=6.0", "backports.cached-property>=1.0.2; python_version<'3.8'", "colorama>=0.4.0; platform_system=='Windows'", - "frozendict>=2.3.4", "rich>=11.1.0", "shtab>=1.5.6" ] [project.optional-dependencies] dev = [ + "PyYAML>=6.0", + "frozendict>=2.3.4", "pytest>=7.1.2", "pytest-cov>=3.0.0", "omegaconf>=2.2.2", From e73fbd2c784973fb1056a8e09380ed854b61e666 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 26 Sep 2023 11:57:33 -0700 Subject: [PATCH 322/491] Add `use_underscores` option (#76) * Add `use_underscores` argument * Dictionary tests * Remove debug print * Format --- tests/helptext_utils.py | 10 ++-- tests/test_dcargs.py | 31 +++++++++++ tests/test_dict_namedtuple.py | 84 ++++++++++++++++++++++++++++++ tests/test_helptext.py | 98 +++++++++++++++++++++++++++++++++++ tyro/_arguments.py | 8 +-- tyro/_cli.py | 70 +++++++++++++++---------- tyro/_strings.py | 62 ++++++++++++++++------ 7 files changed, 313 insertions(+), 50 deletions(-) diff --git a/tests/helptext_utils.py b/tests/helptext_utils.py index fa8e8e845..58553e1f5 100644 --- a/tests/helptext_utils.py +++ b/tests/helptext_utils.py @@ -11,13 +11,15 @@ import tyro._strings -def get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: +def get_helptext( + f: Callable, args: List[str] = ["--help"], use_underscores: bool = False +) -> str: target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): - tyro.cli(f, args=args) + tyro.cli(f, args=args, use_underscores=use_underscores) # Check tyro.extras.get_parser(). - parser = tyro.extras.get_parser(f) + parser = tyro.extras.get_parser(f, use_underscores=use_underscores) assert isinstance(parser, argparse.ArgumentParser) # Returned parser should have formatting information stripped. External tools rarely @@ -44,7 +46,7 @@ def get_helptext(f: Callable, args: List[str] = ["--help"]) -> str: target2 = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): tyro._arguments.USE_RICH = False - tyro.cli(f, args=args) + tyro.cli(f, args=args, use_underscores=use_underscores) tyro._arguments.USE_RICH = True if target2.getvalue() != tyro._strings.strip_ansi_sequences(target.getvalue()): diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 3d21d512b..61e5f3890 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -688,3 +688,34 @@ class A: assert tyro.cli(A, args="--x".split(" ")).x == () assert tyro.cli(A, args="--y".split(" ")).y == [] + + +def test_unknown_args_with_consistent_duplicates_use_underscores() -> None: + @dataclasses.dataclass + class A: + a_b: List[int] = dataclasses.field(default_factory=list) + c_d: List[int] = dataclasses.field(default_factory=list) + + # Tests logic for consistent duplicate arguments when performing argument fixing. + # i.e., we can fix arguments if the separator is consistent (all _'s or all -'s). + a, unknown_args = tyro.cli( + A, + args=[ + "--a-b", + "5", + "--a-b", + "7", + "--c_d", + "5", + "--c_d", + "7", + "--e-f", + "--e-f", + "--g_h", + "--g_h", + ], + return_unknown_args=True, + use_underscores=True, + ) + assert a == A(a_b=[7], c_d=[7]) + assert unknown_args == ["--e-f", "--e-f", "--g_h", "--g_h"] diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index 6aa61bd16..bcf33e73f 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -311,6 +311,36 @@ class HelptextNamedTuple(NamedTuple): def test_nested_dict() -> None: + loaded_config = { + "batch_size": 32, + "optimizer": { + "learning_rate": 1e-4, + "epsilon": 1e-8, + "scheduler": {"schedule_type": "constant"}, + }, + } + backup_config = copy.deepcopy(loaded_config) + overrided_config = tyro.cli( + dict, + default=loaded_config, + args=[ + "--batch-size", + "16", + "--optimizer.scheduler.schedule_type", + "exponential", + ], + ) + + # Overridden config should be different from loaded config. + assert overrided_config != loaded_config + assert overrided_config["batch_size"] == 16 + assert overrided_config["optimizer"]["scheduler"]["schedule_type"] == "exponential" + + # Original loaded config should not be mutated. + assert loaded_config == backup_config + + +def test_nested_dict_use_underscores() -> None: loaded_config = { "batch_size": 32, "optimizer": { @@ -329,6 +359,7 @@ def test_nested_dict() -> None: "--optimizer.scheduler.schedule-type", "exponential", ], + use_underscores=True, ) # Overridden config should be different from loaded config. @@ -372,6 +403,59 @@ def test_nested_dict_hyphen() -> None: assert loaded_config == backup_config +def test_nested_dict_hyphen_use_underscores() -> None: + # We do a lot of underscore <=> conversion in the code; this is just to make sure it + # doesn't break anything! + loaded_config = { + "batch-size": 32, + "optimizer": { + "learning-rate": 1e-4, + "epsilon": 1e-8, + "scheduler": {"schedule-type": "constant"}, + }, + } + backup_config = copy.deepcopy(loaded_config) + overrided_config = tyro.cli( + dict, + default=loaded_config, + args=[ + "--batch-size", + "16", + "--optimizer.scheduler.schedule-type", + "exponential", + ], + use_underscores=True, + ) + + # Overridden config should be different from loaded config. + assert overrided_config != loaded_config + assert overrided_config["batch-size"] == 16 + assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" + + # Original loaded config should not be mutated. + assert loaded_config == backup_config + + overrided_config = tyro.cli( + dict, + default=loaded_config, + args=[ + "--batch_size", + "16", + "--optimizer.scheduler.schedule_type", + "exponential", + ], + use_underscores=True, + ) + + # Overridden config should be different from loaded config. + assert overrided_config != loaded_config + assert overrided_config["batch-size"] == 16 + assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" + + # Original loaded config should not be mutated. + assert loaded_config == backup_config + + def test_nested_dict_annotations() -> None: loaded_config = { "optimizer": { diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 1e8055c85..47d682edd 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -587,3 +587,101 @@ def main(child: Child) -> None: helptext = get_helptext(main) assert "--child.x | --child.no-x" in helptext + + +def test_multiple_subparsers_helptext_hyphens() -> None: + @dataclasses.dataclass + class SubcommandOne: + """2% milk.""" # % symbol is prone to bugs in argparse. + + arg_x: int = 0 + arg_flag: bool = False + + @dataclasses.dataclass + class SubcommandTwo: + arg_y: int = 1 + + @dataclasses.dataclass + class SubcommandThree: + arg_z: int = 2 + + @dataclasses.dataclass + class MultipleSubparsers: + # Field a description. + a: Union[SubcommandOne, SubcommandTwo, SubcommandThree] + # Field b description. + b: Union[SubcommandOne, SubcommandTwo, SubcommandThree] + # Field c description. + c: Union[SubcommandOne, SubcommandTwo, SubcommandThree] = dataclasses.field( + default_factory=SubcommandThree + ) + + helptext = get_helptext(MultipleSubparsers) + + assert "2% milk." in helptext + assert "Field a description." in helptext + assert "Field b description." not in helptext + assert "Field c description." not in helptext + + helptext = get_helptext( + MultipleSubparsers, args=["a:subcommand-one", "b:subcommand-one", "--help"] + ) + + assert "2% milk." in helptext + assert "Field a description." not in helptext + assert "Field b description." not in helptext + assert "Field c description." in helptext + assert "(default: c:subcommand-three)" in helptext + assert "--b.arg-x" in helptext + assert "--b.no-arg-flag" in helptext + assert "--b.arg-flag" in helptext + + +def test_multiple_subparsers_helptext_underscores() -> None: + @dataclasses.dataclass + class SubcommandOne: + """2% milk.""" # % symbol is prone to bugs in argparse. + + arg_x: int = 0 + arg_flag: bool = False + + @dataclasses.dataclass + class SubcommandTwo: + arg_y: int = 1 + + @dataclasses.dataclass + class SubcommandThree: + arg_z: int = 2 + + @dataclasses.dataclass + class MultipleSubparsers: + # Field a description. + a: Union[SubcommandOne, SubcommandTwo, SubcommandThree] + # Field b description. + b: Union[SubcommandOne, SubcommandTwo, SubcommandThree] + # Field c description. + c: Union[SubcommandOne, SubcommandTwo, SubcommandThree] = dataclasses.field( + default_factory=SubcommandThree + ) + + helptext = get_helptext(MultipleSubparsers, use_underscores=True) + + assert "2% milk." in helptext + assert "Field a description." in helptext + assert "Field b description." not in helptext + assert "Field c description." not in helptext + + helptext = get_helptext( + MultipleSubparsers, + args=["a:subcommand_one", "b:subcommand_one", "--help"], + use_underscores=True, + ) + + assert "2% milk." in helptext + assert "Field a description." not in helptext + assert "Field b description." not in helptext + assert "Field c description." in helptext + assert "(default: c:subcommand_three)" in helptext + assert "--b.arg_x" in helptext + assert "--b.no_arg_flag" in helptext + assert "--b.arg_flag" in helptext diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 55f3c384f..8854edb54 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -67,11 +67,13 @@ def __init__( if option_string.startswith("--"): if "." not in option_string: - option_string = "--no-" + option_string[2:] + option_string = ( + "--no" + _strings.get_delimeter() + option_string[2:] + ) else: - # Loose heuristic for where to add the no- prefix. + # Loose heuristic for where to add the no-/no_ prefix. left, _, right = option_string.rpartition(".") - option_string = left + ".no-" + right + option_string = left + ".no" + _strings.get_delimeter() + right self._no_strings.add(option_string) _option_strings.append(option_string) diff --git a/tyro/_cli.py b/tyro/_cli.py index 921c87dad..7fe1fd241 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -50,6 +50,7 @@ def cli( args: Optional[Sequence[str]] = None, default: Optional[OutT] = None, return_unknown_args: Literal[False] = False, + use_underscores: bool = False, ) -> OutT: ... @@ -63,6 +64,7 @@ def cli( args: Optional[Sequence[str]] = None, default: Optional[OutT] = None, return_unknown_args: Literal[True], + use_underscores: bool = False, ) -> Tuple[OutT, List[str]]: ... @@ -79,6 +81,7 @@ def cli( # of the callable itself. default: None = None, return_unknown_args: Literal[False] = False, + use_underscores: bool = False, ) -> OutT: ... @@ -95,6 +98,7 @@ def cli( # of the callable itself. default: None = None, return_unknown_args: Literal[True], + use_underscores: bool = False, ) -> Tuple[OutT, List[str]]: ... @@ -107,6 +111,7 @@ def cli( args: Optional[Sequence[str]] = None, default: Optional[OutT] = None, return_unknown_args: bool = False, + use_underscores: bool = False, **deprecated_kwargs, ) -> Union[OutT, Tuple[OutT, List[str]]]: """Call or instantiate `f`, with inputs populated from an automatically generated @@ -163,6 +168,10 @@ def cli( return_unknown_args: If True, return a tuple of the output of `f` and a list of unknown arguments. Mirrors the unknown arguments returned from `argparse.ArgumentParser.parse_known_args()`. + use_underscores: If True, use underscores as a word delimeter instead of hyphens. + This primarily impacts helptext; underscores and hyphens are treated equivalently + when parsing happens. We default helptext to hyphens to follow the GNU style guide. + https://www.gnu.org/software/libc/manual/html_node/Argument-Syntax.html Returns: The output of `f(...)` or an instance `f`. If `f` is a class, the two are @@ -174,16 +183,18 @@ def cli( # memory address conflicts. _unsafe_cache.clear_cache() - output = _cli_impl( - f, - prog=prog, - description=description, - args=args, - default=default, - return_parser=False, - return_unknown_args=return_unknown_args, - **deprecated_kwargs, - ) + with _strings.delimeter_context("_" if use_underscores else "-"): + output = _cli_impl( + f, + prog=prog, + description=description, + args=args, + default=default, + return_parser=False, + return_unknown_args=return_unknown_args, + use_underscores=use_underscores, + **deprecated_kwargs, + ) # Prevent unnecessary memory usage. _unsafe_cache.clear_cache() @@ -201,6 +212,7 @@ def get_parser( prog: Optional[str] = None, description: Optional[str] = None, default: Optional[OutT] = None, + use_underscores: bool = False, ) -> argparse.ArgumentParser: ... @@ -212,6 +224,7 @@ def get_parser( prog: Optional[str] = None, description: Optional[str] = None, default: Optional[OutT] = None, + use_underscores: bool = False, ) -> argparse.ArgumentParser: ... @@ -224,24 +237,27 @@ def get_parser( prog: Optional[str] = None, description: Optional[str] = None, default: Optional[OutT] = None, + use_underscores: bool = False, ) -> argparse.ArgumentParser: """Get the `argparse.ArgumentParser` object generated under-the-hood by `tyro.cli()`. Useful for tools like `sphinx-argparse`, `argcomplete`, etc. For tab completion, we recommend using `tyro.cli()`'s built-in `--tyro-write-completion` flag.""" - return cast( - argparse.ArgumentParser, - _cli_impl( - f, - prog=prog, - description=description, - args=None, - default=default, - return_parser=True, - return_unknown_args=False, - ), - ) + with _strings.delimeter_context("_" if use_underscores else "-"): + return cast( + argparse.ArgumentParser, + _cli_impl( + f, + prog=prog, + description=description, + args=None, + default=default, + return_parser=True, + return_unknown_args=False, + use_underscores=use_underscores, + ), + ) def _cli_impl( @@ -302,19 +318,21 @@ def _cli_impl( args = list(sys.argv[1:]) if args is None else list(args) # Fix arguments. This will modify all option-style arguments replacing - # underscores with dashes. This is to support the common convention of using - # underscores in variable names, but dashes in command line arguments. + # underscores with hyphens, or vice versa if use_underscores=True. # If two options are ambiguous, e.g., --a_b and --a-b, raise a runtime error. modified_args: Dict[str, str] = {} for index, arg in enumerate(args): if not arg.startswith("--"): continue + delimeter = _strings.get_delimeter() + to_swap_delimeter = "-" if delimeter == "_" else "_" + if "=" in arg: arg, _, val = arg.partition("=") - fixed = arg.replace("_", "-") + "=" + val + fixed = "--" + arg[2:].replace(to_swap_delimeter, delimeter) + "=" + val else: - fixed = arg.replace("_", "-") + fixed = "--" + arg[2:].replace(to_swap_delimeter, delimeter) if ( return_unknown_args and fixed in modified_args diff --git a/tyro/_strings.py b/tyro/_strings.py index e6b9527a2..1caa03a03 100644 --- a/tyro/_strings.py +++ b/tyro/_strings.py @@ -1,21 +1,39 @@ """Utilities and constants for working with strings.""" +import contextlib import functools import re import textwrap from typing import Iterable, List, Sequence, Tuple, Type -from typing_extensions import get_args, get_origin +from typing_extensions import Literal, get_args, get_origin from . import _resolver dummy_field_name = "__tyro_dummy_field__" +DELIMETER: Literal["-", "_"] = "-" def _strip_dummy_field_names(parts: Iterable[str]) -> Iterable[str]: return filter(lambda name: len(name) > 0 and name != dummy_field_name, parts) +@contextlib.contextmanager +def delimeter_context(delimeter: Literal["-", "_"]): + """Context for setting the delimeter. Determines if `field_a` is populated as + `--field-a` or `--field_a`. Not thread-safe.""" + global DELIMETER + delimeter_restore = DELIMETER + DELIMETER = delimeter + yield + DELIMETER = delimeter_restore + + +def get_delimeter() -> Literal["-", "_"]: + """Get delimeter used to separate words.""" + return DELIMETER + + def make_field_name(parts: Sequence[str]) -> str: """Join parts of a field name together. Used for nesting. @@ -28,13 +46,18 @@ def make_field_name(parts: Sequence[str]) -> str: out.append(".") # Replace all underscores with hyphens, except ones at the start of a string. - num_underscore_prefix = 0 - for i in range(len(p)): - if p[i] == "_": - num_underscore_prefix += 1 - else: - break - p = "_" * num_underscore_prefix + p[num_underscore_prefix:].replace("_", "-") + if get_delimeter() == "-": + num_underscore_prefix = 0 + for i in range(len(p)): + if p[i] == "_": + num_underscore_prefix += 1 + else: + break + p = "_" * num_underscore_prefix + ( + p[num_underscore_prefix:].replace("_", "-") + ) + else: + p = p.replace("-", "_") out.append(p) return "".join(out) @@ -52,13 +75,13 @@ def dedent(text: str) -> str: return f"{first_line.strip()}\n{textwrap.dedent(rest)}" -_camel_separator_pattern = functools.lru_cache(maxsize=1)( - lambda: re.compile("((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))") -) - - def hyphen_separated_from_camel_case(name: str) -> str: - return _camel_separator_pattern().sub(r"-\1", name).lower() + out = ( + re.compile("((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))") + .sub(get_delimeter() + r"\1", name) + .lower() + ) + return out def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: @@ -85,7 +108,7 @@ def get_name(cls: Type) -> str: if orig is not None and hasattr(orig, "__name__"): parts = [orig.__name__] # type: ignore parts.extend(map(get_name, get_args(cls))) - return "-".join(parts) + return get_delimeter().join(parts) elif hasattr(cls, "__name__"): return hyphen_separated_from_camel_case(cls.__name__) else: @@ -97,7 +120,7 @@ def get_name(cls: Type) -> str: return get_name(cls), prefix_name # type: ignore return ( - "-".join( + get_delimeter().join( map( lambda x: _subparser_name_from_type(x)[0], [cls] + list(type_from_typevar.values()), @@ -113,7 +136,12 @@ def subparser_name_from_type(prefix: str, cls: Type) -> str: ) if len(prefix) == 0 or not use_prefix: return suffix - return f"{prefix}:{suffix}".replace("_", "-") + + if get_delimeter() == "-": + return f"{prefix}:{suffix}".replace("_", "-") + else: + assert get_delimeter() == "_" + return f"{prefix}:{suffix}" @functools.lru_cache(maxsize=None) From 2b8579fb25e37630f7d9b5767c9e6839aad67c8e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 26 Sep 2023 13:34:40 -0700 Subject: [PATCH 323/491] Update docs build --- docs/requirements.txt | 10 +++++----- .../examples/02_nesting/03_multiple_subcommands.rst | 8 -------- examples/02_nesting/03_multiple_subcommands.py | 1 - 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index b39d0c586..7260ab5aa 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ -sphinx==5.0.2 -furo==2022.6.21 -docutils==0.17.1 -sphinx-autoapi==1.8.4 -m2r2==0.3.2 +sphinx==7.2.6 +furo==2023.9.10 +docutils==0.20.1 +sphinx-autoapi==3.0.0 +m2r2==0.3.3.post2 git+https://github.com/brentyi/sphinxcontrib-programoutput.git git+https://github.com/brentyi/ansi.git diff --git a/docs/source/examples/02_nesting/03_multiple_subcommands.rst b/docs/source/examples/02_nesting/03_multiple_subcommands.rst index 78b3d9ea2..480ddbee4 100644 --- a/docs/source/examples/02_nesting/03_multiple_subcommands.rst +++ b/docs/source/examples/02_nesting/03_multiple_subcommands.rst @@ -75,14 +75,6 @@ Multiple unions over nested types are populated using a series of subcommands. ------------ -.. raw:: html - - python 02_nesting/03_multiple_subcommands.py - -.. program-output:: python ../../examples/02_nesting/03_multiple_subcommands.py - ------------- - .. raw:: html python 02_nesting/03_multiple_subcommands.py --help diff --git a/examples/02_nesting/03_multiple_subcommands.py b/examples/02_nesting/03_multiple_subcommands.py index 99d4bf79a..08536e014 100644 --- a/examples/02_nesting/03_multiple_subcommands.py +++ b/examples/02_nesting/03_multiple_subcommands.py @@ -3,7 +3,6 @@ Multiple unions over nested types are populated using a series of subcommands. Usage: -`python ./03_multiple_subcommands.py` `python ./03_multiple_subcommands.py --help` `python ./03_multiple_subcommands.py dataset:mnist --help` `python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --help` From 7a8c6f736cc81134b6afdb36511ba7fcd5aa8c7a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 26 Sep 2023 22:06:45 -0700 Subject: [PATCH 324/491] Fix yaml import error --- pyproject.toml | 2 +- tyro/extras/_serialization.py | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 46412a835..4bd46d11c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.8" +version = "0.5.9" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/tyro/extras/_serialization.py b/tyro/extras/_serialization.py index c61e674b1..3acc40f00 100644 --- a/tyro/extras/_serialization.py +++ b/tyro/extras/_serialization.py @@ -1,11 +1,15 @@ """Type-safe, human-readable serialization helpers for dataclasses.""" +from __future__ import annotations import dataclasses import enum import functools -from typing import IO, Any, Optional, Set, Type, TypeVar, Union +from typing import IO, TYPE_CHECKING, Any, Optional, Set, Type, TypeVar, Union + +if TYPE_CHECKING: + # Since serialization functionality is deprecated, the yaml dependency is optional. + import yaml -import yaml from typing_extensions import get_args, get_origin from .. import _fields, _resolver @@ -70,6 +74,8 @@ def handle_type(typ: Type[Any]) -> Set[Type[Any]]: def _make_loader(cls: Type[Any]) -> Type[yaml.Loader]: + import yaml + class DataclassLoader(yaml.Loader): pass @@ -120,6 +126,8 @@ def make_enum_constructor(typ: Type[Any]): def _make_dumper(instance: Any) -> Type[yaml.Dumper]: + import yaml + class DataclassDumper(yaml.Dumper): def ignore_aliases(self, data): return super().ignore_aliases(data) or data is _fields.MISSING_PROP @@ -194,6 +202,8 @@ def from_yaml( Returns: Instantiated dataclass. """ + import yaml + out = yaml.load(stream, Loader=_make_loader(cls)) origin_cls = get_origin(cls) assert isinstance(out, origin_cls if origin_cls is not None else cls) @@ -222,4 +232,6 @@ def to_yaml(instance: Any) -> str: Returns: YAML string. """ + import yaml + return "# tyro YAML.\n" + yaml.dump(instance, Dumper=_make_dumper(instance)) From 86a69436ea6e52a776e8709a9fbd5553c8274fc4 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 5 Oct 2023 16:48:57 +0200 Subject: [PATCH 325/491] Patch underscore prefix edge case (#78) https://github.com/brentyi/tyro/issues/77 --- README.md | 55 +++++++++++++++++++++++++------------------ docs/requirements.txt | 7 ------ pyproject.toml | 2 +- tests/test_nested.py | 28 ++++++++++++++++++++++ tyro/_calling.py | 1 - tyro/_strings.py | 41 ++++++++++++++++---------------- 6 files changed, 81 insertions(+), 53 deletions(-) delete mode 100644 docs/requirements.txt diff --git a/README.md b/README.md index cd1b4feac..80d69a9fb 100644 --- a/README.md +++ b/README.md @@ -37,13 +37,32 @@
-tyro is a tool for building command-line -interfaces and configuration objects in Python. - -Our core interface, `tyro.cli()`, generates command-line interfaces from -type-annotated callables. - ---- +tyro is a tool for generating command-line +interfaces and configuration objects from type-annotated Python. We: + +- Introduce a single-function core API, `tyro.cli()`, which is minimal enough to + use in throwaway scripts but flexible enough to be hardened in larger + projects. +- Support a broad range of Python type constructs, including basics (`int`, + `str`, `bool`, `float`, `pathlib.Path`, ...), both fixed- and variable-length + containers (`list[T]`, `tuple[T1, T2, ...]`, `set[T]`, `dict[K, V]`), unions + (`X | Y`, `Union[X, Y]`), literals (`Literal`), and generics (`TypeVar`). +- Understand hierarchy, nesting, and tools you may already use, like + `dataclasses`, `pydantic`, and `attrs`. +- Generate helptext automatically from defaults, annotations, and docstrings. +- Provide flexible support for subcommands, as well as choosing between and + overriding values in configuration objects. +- Enable tab completion in both your IDE and terminal. +- Expose fine-grained configuration via PEP 529 runtime annotations + (`tyro.conf.*`). + +`tyro`'s use cases overlaps significantly with many other tools. The differences +are a result of several API goals: + +- Focusing on a single, uninvasive function. +- Prioritizing compatibility with language servers and type checkers. +- Avoiding assumptions on serialization formats (like YAML or JSON) for + configuration objects. ### Brief walkthrough @@ -65,10 +84,10 @@ print(total) ``` This pattern is dramatically cleaner than manually parsing `sys.argv`, but has -several issues: it lacks type checking and IDE support (consider: jumping to +several issues: it requires a significant amount of parsing-specific +boilerplate, lacks type checking and IDE support (consider: jumping to definitions, finding references, docstrings, refactoring and renaming tools), -requires a significant amount of parsing-specific boilerplate, and becomes -difficult to manage for larger projects. +and becomes difficult to manage for larger projects. The basic goal of `tyro.cli()` is to provide a wrapper for `argparse` that solves these issues. @@ -122,17 +141,6 @@ args = tyro.cli(Args) print(args.a + args.b) ``` -Unlike directly using `argparse`, both the function-based and dataclass-based -approaches are compatible with static analysis; tab completion and type checking -will work out-of-the-box. - -**(3) Additional features.** - -These examples only scratch the surface of what's possible. `tyro` aims to -support all reasonable type annotations, which can help us define things like -hierarchical structures, enums, unions, variable-length inputs, and subcommands. -See [documentation](https://brentyi.github.io/tyro) for examples. - ### In the wild `tyro` is still a new library, but being stress tested in several projects! @@ -148,7 +156,8 @@ See [documentation](https://brentyi.github.io/tyro) for examples. implementation of [instant-ngp](https://nvlabs.github.io/instant-ngp/), implemented in JAX. - [NVIDIAGameWorks/kaolin-wisp](https://github.com/NVIDIAGameWorks/kaolin-wisp) - combines `tyro` with [`hydra-zen`](https://github.com/mit-ll-responsible-ai/hydra-zen) - for neural fields in PyTorch. + combines `tyro` with + [`hydra-zen`](https://github.com/mit-ll-responsible-ai/hydra-zen) for neural + fields in PyTorch. - [openrlbenchmark/openrlbenchmark](https://github.com/openrlbenchmark/openrlbenchmark) is a collection of tracked experiments for reinforcement learning. diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 7260ab5aa..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -sphinx==7.2.6 -furo==2023.9.10 -docutils==0.20.1 -sphinx-autoapi==3.0.0 -m2r2==0.3.3.post2 -git+https://github.com/brentyi/sphinxcontrib-programoutput.git -git+https://github.com/brentyi/ansi.git diff --git a/pyproject.toml b/pyproject.toml index 4bd46d11c..8b3f5efb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.9" +version = "0.5.10" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/tests/test_nested.py b/tests/test_nested.py index f92112d72..5739e7e5b 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -989,3 +989,31 @@ def train( container=DatasetContainer(ImageNet(50)) ) assert tyro.cli(train, args=["sgd"]) == Sgd(container=DatasetContainer(Mnist())) + + +def test_underscore_prefix() -> None: + """https://github.com/brentyi/tyro/issues/77""" + + @dataclasses.dataclass + class PrivateConfig: + pass + + @dataclasses.dataclass + class BaseConfig: + _private: PrivateConfig = dataclasses.field( + default_factory=lambda: PrivateConfig() + ) + + @dataclasses.dataclass + class Level1(BaseConfig): + pass + + @dataclasses.dataclass + class Level2(BaseConfig): + child: Level1 = dataclasses.field(default_factory=lambda: Level1()) + + @dataclasses.dataclass + class Level3(BaseConfig): + child: Level2 = dataclasses.field(default_factory=lambda: Level2()) + + tyro.cli(Level3, args=[]) diff --git a/tyro/_calling.py b/tyro/_calling.py index 9eca6fe0f..f8daa752f 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -64,7 +64,6 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: # Resolve field type. field_type = field.typ - if prefixed_field_name in arg_from_prefixed_field_name: assert prefixed_field_name not in consumed_keywords diff --git a/tyro/_strings.py b/tyro/_strings.py index 1caa03a03..2d495e10d 100644 --- a/tyro/_strings.py +++ b/tyro/_strings.py @@ -34,33 +34,32 @@ def get_delimeter() -> Literal["-", "_"]: return DELIMETER +def replace_delimeter_in_part(p: str) -> str: + """Replace hyphens with underscores (or vice versa) except when at the start.""" + if get_delimeter() == "-": + num_underscore_prefix = 0 + for i in range(len(p)): + if p[i] == "_": + num_underscore_prefix += 1 + else: + break + p = "_" * num_underscore_prefix + (p[num_underscore_prefix:].replace("_", "-")) + else: + p = p.replace("-", "_") + return p + + def make_field_name(parts: Sequence[str]) -> str: """Join parts of a field name together. Used for nesting. ('parent_1', 'child') => 'parent-1.child' ('parents', '1', '_child_node') => 'parents.1._child-node' + ('parents', '1', 'middle._child_node') => 'parents.1.middle._child-node' """ out: List[str] = [] - for i, p in enumerate(_strip_dummy_field_names(parts)): - if i > 0: - out.append(".") - - # Replace all underscores with hyphens, except ones at the start of a string. - if get_delimeter() == "-": - num_underscore_prefix = 0 - for i in range(len(p)): - if p[i] == "_": - num_underscore_prefix += 1 - else: - break - p = "_" * num_underscore_prefix + ( - p[num_underscore_prefix:].replace("_", "-") - ) - else: - p = p.replace("-", "_") - out.append(p) - - return "".join(out) + for p in _strip_dummy_field_names(parts): + out.extend(map(replace_delimeter_in_part, p.split("."))) + return ".".join(out) def make_subparser_dest(name: str) -> str: @@ -138,7 +137,7 @@ def subparser_name_from_type(prefix: str, cls: Type) -> str: return suffix if get_delimeter() == "-": - return f"{prefix}:{suffix}".replace("_", "-") + return f"{prefix}:{make_field_name(suffix.split('.'))}" else: assert get_delimeter() == "_" return f"{prefix}:{suffix}" From d2906ae5037f63a0f2360722c0e7988b2d6ee45a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 5 Oct 2023 16:54:05 +0200 Subject: [PATCH 326/491] Update docs build, README --- README.md | 219 ++++++++++++++++++++---------------------- docs/requirements.txt | 7 ++ docs/source/index.md | 5 +- 3 files changed, 114 insertions(+), 117 deletions(-) create mode 100644 docs/requirements.txt diff --git a/README.md b/README.md index 80d69a9fb..631178661 100644 --- a/README.md +++ b/README.md @@ -38,126 +38,115 @@
tyro is a tool for generating command-line -interfaces and configuration objects from type-annotated Python. We: +interfaces and configuration objects from type-annotated Python. -- Introduce a single-function core API, `tyro.cli()`, which is minimal enough to - use in throwaway scripts but flexible enough to be hardened in larger - projects. -- Support a broad range of Python type constructs, including basics (`int`, - `str`, `bool`, `float`, `pathlib.Path`, ...), both fixed- and variable-length - containers (`list[T]`, `tuple[T1, T2, ...]`, `set[T]`, `dict[K, V]`), unions - (`X | Y`, `Union[X, Y]`), literals (`Literal`), and generics (`TypeVar`). -- Understand hierarchy, nesting, and tools you may already use, like +Our single-function core API, `tyro.cli()`, + +- Generates CLI interfaces from a comprehensive set of Python type constructs. +- Generates helptext automatically from defaults, annotations, and docstrings. +- Understands hierarchy, nesting, and tools you may already use, like `dataclasses`, `pydantic`, and `attrs`. -- Generate helptext automatically from defaults, annotations, and docstrings. -- Provide flexible support for subcommands, as well as choosing between and +- Provides flexible support for subcommands, as well as choosing between and overriding values in configuration objects. -- Enable tab completion in both your IDE and terminal. -- Expose fine-grained configuration via PEP 529 runtime annotations +- Enables tab completion in both your IDE and terminal. +- Supports fine-grained configuration via PEP 529 runtime annotations (`tyro.conf.*`). -`tyro`'s use cases overlaps significantly with many other tools. The differences -are a result of several API goals: - -- Focusing on a single, uninvasive function. -- Prioritizing compatibility with language servers and type checkers. -- Avoiding assumptions on serialization formats (like YAML or JSON) for - configuration objects. - -### Brief walkthrough - -To summarize how `tyro.cli()` can be used, let's consider a script based on -`argparse`. We define two inputs and print the sum: - -```python -"""Sum two numbers from argparse.""" -import argparse - -parser = argparse.ArgumentParser() -parser.add_argument("--a", type=int, required=True) -parser.add_argument("--b", type=int, default=3) -args = parser.parse_args() - -total = args.a + args.b - -print(total) -``` - -This pattern is dramatically cleaner than manually parsing `sys.argv`, but has -several issues: it requires a significant amount of parsing-specific -boilerplate, lacks type checking and IDE support (consider: jumping to -definitions, finding references, docstrings, refactoring and renaming tools), -and becomes difficult to manage for larger projects. - -The basic goal of `tyro.cli()` is to provide a wrapper for `argparse` that -solves these issues. - -**(1) Command-line interfaces from functions.** - -We can write the same script as above using `tyro.cli()`: - -```python -"""Sum two numbers by calling a function with tyro.""" -import tyro - -def add(a: int, b: int = 3) -> int: - return a + b - -# Populate the inputs of add(), call it, then return the output. -total = tyro.cli(add) - -print(total) -``` - -Or, more succinctly: - -```python -"""Sum two numbers by calling a function with tyro.""" -import tyro - -def add(a: int, b: int = 3) -> None: - print(a + b) - -tyro.cli(add) # Returns `None`. -``` - -**(2) Command-line interfaces from config objects.** - -A class in Python can be treated as a function that returns an instance. This -makes it easy to populate explicit configuration structures: - -```python -"""Sum two numbers by instantiating a dataclass with tyro.""" -from dataclasses import dataclass - -import tyro - -@dataclass -class Args: - a: int - b: int = 3 - -args = tyro.cli(Args) -print(args.a + args.b) -``` +For examples and the API reference, see our +[documentation](https://brentyi.github.io/tyro). ### In the wild -`tyro` is still a new library, but being stress tested in several projects! - -- [nerfstudio-project/nerfstudio](https://github.com/nerfstudio-project/nerfstudio/) - provides a set of tools for end-to-end training, testing, and rendering of - neural radiance fields. -- [Sea-Snell/JAXSeq](https://github.com/Sea-Snell/JAXSeq/) is a library for - distributed training of large language models in JAX. -- [kevinzakka/obj2mjcf](https://github.com/kevinzakka/obj2mjcf) is an interface - for processing composite Wavefront OBJ files for Mujoco. -- [blurgyy/jaxngp](https://github.com/blurgyy/jaxngp) is a CUDA-accelerated - implementation of [instant-ngp](https://nvlabs.github.io/instant-ngp/), - implemented in JAX. -- [NVIDIAGameWorks/kaolin-wisp](https://github.com/NVIDIAGameWorks/kaolin-wisp) - combines `tyro` with - [`hydra-zen`](https://github.com/mit-ll-responsible-ai/hydra-zen) for neural - fields in PyTorch. -- [openrlbenchmark/openrlbenchmark](https://github.com/openrlbenchmark/openrlbenchmark) - is a collection of tracked experiments for reinforcement learning. +`tyro` has been stress-tested in several projects, including: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + nerfstudio-project/nerfstudio +
GitHub star count + + 		+ Open-source tools for neural radiance fields. +
+ + Sea-Snell/JAXSeq +
GitHub star count + + 		Library for distributed training of large language models in JAX.
+ + kevinzakka/obj2mjcf +
GitHub star count + + 		Interface for processing composite Wavefront OBJ files for Mujoco.
+ + blurgyy/jaxngp +
GitHub star count + + 		+ CUDA-accelerated implementation of + instant-ngp, in JAX. +
+ + NVIDIAGameWorks/kaolin-wisp +
GitHub star count + + 		PyTorch library for neural fields.
+ + autonomousvision/sdfstudio +
GitHub star count + + 		Unified framework for surface reconstruction.
+ + openrlbenchmark/openrlbenchmark +
GitHub star count + + 		Collection of tracked experiments for reinforcement learning.
diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..7260ab5aa --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,7 @@ +sphinx==7.2.6 +furo==2023.9.10 +docutils==0.20.1 +sphinx-autoapi==3.0.0 +m2r2==0.3.3.post2 +git+https://github.com/brentyi/sphinxcontrib-programoutput.git +git+https://github.com/brentyi/ansi.git diff --git a/docs/source/index.md b/docs/source/index.md index 79248c018..206d9835b 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -2,8 +2,9 @@ |build| |nbsp| |mypy| |nbsp| |lint| |nbsp| |coverage| |nbsp| |versions| -:code:`tyro` is a tool for building scalable command-line interfaces and -configuration objects in Python. +:code:`tyro` is a tool for generating command-line interfaces and configuration +objects from type-annotated Python, with a goal of being lightweight enough for +throwaway scripts but flexible enough for large projects. Our core interface, :func:`tyro.cli()`, generates command-line interfaces from type-annotated callables. From c7400d9f040c7f9ecbd4c091984caac0586130ae Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 19 Oct 2023 10:08:15 -0700 Subject: [PATCH 327/491] Sync docs updates --- README.md | 58 ++++++++++++++----- .../{03_containers.rst => 03_collections.rst} | 16 ++--- docs/source/index.md | 27 +++++++-- examples/01_basics/03_containers.py | 34 ----------- 4 files changed, 75 insertions(+), 60 deletions(-) rename docs/source/examples/01_basics/{03_containers.rst => 03_collections.rst} (64%) delete mode 100644 examples/01_basics/03_containers.py diff --git a/README.md b/README.md index 631178661..2a7a78d4f 100644 --- a/README.md +++ b/README.md @@ -38,18 +38,27 @@
tyro is a tool for generating command-line -interfaces and configuration objects from type-annotated Python. - -Our single-function core API, `tyro.cli()`, - -- Generates CLI interfaces from a comprehensive set of Python type constructs. -- Generates helptext automatically from defaults, annotations, and docstrings. -- Understands hierarchy, nesting, and tools you may already use, like - `dataclasses`, `pydantic`, and `attrs`. -- Provides flexible support for subcommands, as well as choosing between and - overriding values in configuration objects. -- Enables tab completion in both your IDE and terminal. -- Supports fine-grained configuration via PEP 529 runtime annotations +interfaces and configuration objects in Python. + +Our core API, `tyro.cli()`, + +- **Generates CLI interfaces** from a comprehensive set of Python type + constructs. +- **Populates helptext automatically** from defaults, annotations, and + docstrings. +- **Understands nesting** of `dataclasses`, `pydantic`, and `attrs` structures. +- **Prioritizes static analysis** for type checking and autocompletion with + tools like + [Pylance](https://marketplace.visualstudio.com/items?itemName=ms-python.vscode-pylance), + [Pyright](https://github.com/microsoft/pyright), and + [mypy](https://github.com/python/mypy). + +For power users, it also: + +- **Supports subcommands**, as well as choosing between and overriding values in + configuration objects. +- **Generates shell completion** scripts for `bash`, `zsh`, and `tcsh`. +- **Supports fine-grained configuration** via PEP 529 runtime annotations (`tyro.conf.*`). For examples and the API reference, see our @@ -57,7 +66,8 @@ For examples and the API reference, see our ### In the wild -`tyro` has been stress-tested in several projects, including: +`tyro` is designed to be lightweight enough for throwaway scripts, while +facilitating type safety and modularity for larger projects. Examples: @@ -150,3 +160,25 @@ For examples and the API reference, see our
Collection of tracked experiments for reinforcement learning.
+ +### Alternatives + +`tyro` bakes many opinions into its design decisions. If any of them don't make +sense, feel free to file an issue! + +You might also consider one of many alternative libraries. Some that we +particularly like: + +- [simple-parsing](https://github.com/lebrice/SimpleParsing) and + [jsonargparse](https://github.com/omni-us/jsonargparse), which provide deeper + integration with configuration file formats like YAML and JSON. +- [clipstick](https://github.com/sander76/clipstick), which focuses on + generating CLIs from Pydantic models. +- [datargs](https://github.com/roee30/datargs) provides a minimal API for + dataclasses. +- [fire](https://github.com/google/python-fire) and + [clize](https://github.com/epsy/clize), which support arguments without type + annotations. + +We also have some notes on `tyro`'s design goals and other alternatives in the +docs [here](https://brentyi.github.io/tyro/goals_and_alternatives/). diff --git a/docs/source/examples/01_basics/03_containers.rst b/docs/source/examples/01_basics/03_collections.rst similarity index 64% rename from docs/source/examples/01_basics/03_containers.rst rename to docs/source/examples/01_basics/03_collections.rst index f30129ba6..d939cd48c 100644 --- a/docs/source/examples/01_basics/03_containers.rst +++ b/docs/source/examples/01_basics/03_collections.rst @@ -1,12 +1,12 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -Containers +Multi-value Arguments ========================================== Arguments of both fixed and variable lengths can be annotated with standard Python -container types: ``typing.List[T]``\ , ``typing.Tuple[T1, T2]``\ , etc. In Python >=3.9, +collection types: ``typing.List[T]``\ , ``typing.Tuple[T1, T2]``\ , etc. In Python >=3.9, ``list[T]`` and ``tuple[T]`` are also supported. @@ -42,22 +42,22 @@ container types: ``typing.List[T]``\ , ``typing.Tuple[T1, T2]``\ , etc. In Pytho .. raw:: html - python 01_basics/03_containers.py --help + python 01_basics/03_collections.py --help -.. program-output:: python ../../examples/01_basics/03_containers.py --help +.. program-output:: python ../../examples/01_basics/03_collections.py --help ------------ .. raw:: html - python 01_basics/03_containers.py --dataset-sources ./data --image-dimensions 16 16 + python 01_basics/03_collections.py --dataset-sources ./data --image-dimensions 16 16 -.. program-output:: python ../../examples/01_basics/03_containers.py --dataset-sources ./data --image-dimensions 16 16 +.. program-output:: python ../../examples/01_basics/03_collections.py --dataset-sources ./data --image-dimensions 16 16 ------------ .. raw:: html - python 01_basics/03_containers.py --dataset-sources ./data + python 01_basics/03_collections.py --dataset-sources ./data -.. program-output:: python ../../examples/01_basics/03_containers.py --dataset-sources ./data +.. program-output:: python ../../examples/01_basics/03_collections.py --dataset-sources ./data diff --git a/docs/source/index.md b/docs/source/index.md index 206d9835b..49566a4c5 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -3,11 +3,28 @@ |build| |nbsp| |mypy| |nbsp| |lint| |nbsp| |coverage| |nbsp| |versions| :code:`tyro` is a tool for generating command-line interfaces and configuration -objects from type-annotated Python, with a goal of being lightweight enough for -throwaway scripts but flexible enough for large projects. - -Our core interface, :func:`tyro.cli()`, generates command-line interfaces from -type-annotated callables. +objects in Python. + +Our core API, `tyro.cli()`, + +- **Generates CLI interfaces** from a comprehensive set of Python type + constructs. +- **Populates helptext automatically** from defaults, annotations, and + docstrings. +- **Understands nesting** of `dataclasses`, `pydantic`, and `attrs` structures. +- **Prioritizes static analysis** for type checking and autocompletion with + tools like + [Pylance](https://marketplace.visualstudio.com/items?itemName=ms-python.vscode-pylance), + [Pyright](https://github.com/microsoft/pyright), and + [mypy](https://github.com/python/mypy). + +For power users, it also: + +- **Supports subcommands**, as well as choosing between and overriding values in + configuration objects. +- **Generates shell completion** scripts for `bash`, `zsh`, and `tcsh`. +- **Supports fine-grained configuration** via PEP 529 runtime annotations + (`tyro.conf.*`). To get started, we recommend browsing the examples to the left. diff --git a/examples/01_basics/03_containers.py b/examples/01_basics/03_containers.py deleted file mode 100644 index 9c3a881de..000000000 --- a/examples/01_basics/03_containers.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Containers - -Arguments of both fixed and variable lengths can be annotated with standard Python -container types: `typing.List[T]`, `typing.Tuple[T1, T2]`, etc. In Python >=3.9, -`list[T]` and `tuple[T]` are also supported. - -Usage: -`python ./03_containers.py --help` -`python ./03_containers.py --dataset-sources ./data --image-dimensions 16 16` -`python ./03_containers.py --dataset-sources ./data` -""" - -import dataclasses -import pathlib -from typing import Tuple - -import tyro - - -@dataclasses.dataclass(frozen=True) -class TrainConfig: - # Example of a variable-length tuple. `typing.List`, `typing.Sequence`, - # `typing.Set`, `typing.Dict`, etc are all supported as well. - dataset_sources: Tuple[pathlib.Path, ...] - """Paths to load training data from. This can be multiple!""" - - # Fixed-length tuples are also okay. - image_dimensions: Tuple[int, int] = (32, 32) - """Height and width of some image data.""" - - -if __name__ == "__main__": - config = tyro.cli(TrainConfig) - print(config) From f2133831e912b74d11d5f71da3a99a120f258ac4 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 24 Oct 2023 13:28:57 -0700 Subject: [PATCH 328/491] Improved dictionaries: `TypedDict` support, `typing.[Required,NotRequired]`, defaults (#81) * Support typing.[Required,NotRequired], tweak dictionary defaults * Test for generic collections * More tests for edge cases * Appease mypy * Add Python 3.12 to build * Remove 3.12 until dependencies are available * Add more comprehensive Python 3.11 tests * Rename generated tests * Exclude generated tests from mypy * Tweak mypy exclude * Tweak again --- README.md | 13 +- docs/source/index.md | 8 +- examples/04_additional/02_dictionaries.py | 19 +- pyproject.toml | 1 + tests/conftest.py | 3 + tests/test_collections.py | 7 + tests/test_dict_namedtuple.py | 77 +- tests/test_errors.py | 8 +- tests/test_py311_generated/README.md | 2 + tests/test_py311_generated/_generate.py | 9 + .../test_attrs_generated.py | 102 ++ .../test_boolean_optional_generated.py | 67 ++ .../test_collections_generated.py | 467 ++++++++ .../test_completion_generated.py | 43 + .../test_conf_generated.py | 902 +++++++++++++++ .../test_dcargs_generated.py | 724 ++++++++++++ .../test_dict_namedtuple_generated.py | 543 +++++++++ .../test_dynamic_dataclasses_generated.py | 14 + .../test_errors_generated.py | 471 ++++++++ .../test_flax_min_py38_generated.py | 65 ++ .../test_forward_ref_generated.py | 48 + .../test_functools_generated.py | 133 +++ ...st_generics_and_serialization_generated.py | 422 +++++++ .../test_helptext_generated.py | 698 +++++++++++ .../test_initvar_min_py38_generated.py | 27 + .../test_is_nested_type_generated.py | 52 + .../test_missing_generated.py | 73 ++ ...est_missing_optional_packages_generated.py | 25 + .../test_mixed_unions_generated.py | 87 ++ .../test_nested_generated.py | 1028 +++++++++++++++++ .../test_nested_in_containers_generated.py | 385 ++++++ ...w_style_annotations_min_py310_generated.py | 64 + ...ew_style_annotations_min_py39_generated.py | 52 + .../test_partial_generated.py | 61 + .../test_positional_min_py38_generated.py | 118 ++ .../test_pydantic_generated.py | 108 ++ .../test_strings_generated.py | 70 ++ .../test_union_from_mapping_generated.py | 48 + .../test_unsafe_cache_generated.py | 31 + ...t_unsupported_but_should_work_generated.py | 58 + tyro/_fields.py | 49 +- tyro/_instantiators.py | 11 + 42 files changed, 7146 insertions(+), 47 deletions(-) create mode 100644 tests/test_py311_generated/README.md create mode 100644 tests/test_py311_generated/_generate.py create mode 100644 tests/test_py311_generated/test_attrs_generated.py create mode 100644 tests/test_py311_generated/test_boolean_optional_generated.py create mode 100644 tests/test_py311_generated/test_collections_generated.py create mode 100644 tests/test_py311_generated/test_completion_generated.py create mode 100644 tests/test_py311_generated/test_conf_generated.py create mode 100644 tests/test_py311_generated/test_dcargs_generated.py create mode 100644 tests/test_py311_generated/test_dict_namedtuple_generated.py create mode 100644 tests/test_py311_generated/test_dynamic_dataclasses_generated.py create mode 100644 tests/test_py311_generated/test_errors_generated.py create mode 100644 tests/test_py311_generated/test_flax_min_py38_generated.py create mode 100644 tests/test_py311_generated/test_forward_ref_generated.py create mode 100644 tests/test_py311_generated/test_functools_generated.py create mode 100644 tests/test_py311_generated/test_generics_and_serialization_generated.py create mode 100644 tests/test_py311_generated/test_helptext_generated.py create mode 100644 tests/test_py311_generated/test_initvar_min_py38_generated.py create mode 100644 tests/test_py311_generated/test_is_nested_type_generated.py create mode 100644 tests/test_py311_generated/test_missing_generated.py create mode 100644 tests/test_py311_generated/test_missing_optional_packages_generated.py create mode 100644 tests/test_py311_generated/test_mixed_unions_generated.py create mode 100644 tests/test_py311_generated/test_nested_generated.py create mode 100644 tests/test_py311_generated/test_nested_in_containers_generated.py create mode 100644 tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py create mode 100644 tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py create mode 100644 tests/test_py311_generated/test_partial_generated.py create mode 100644 tests/test_py311_generated/test_positional_min_py38_generated.py create mode 100644 tests/test_py311_generated/test_pydantic_generated.py create mode 100644 tests/test_py311_generated/test_strings_generated.py create mode 100644 tests/test_py311_generated/test_union_from_mapping_generated.py create mode 100644 tests/test_py311_generated/test_unsafe_cache_generated.py create mode 100644 tests/test_py311_generated/test_unsupported_but_should_work_generated.py diff --git a/README.md b/README.md index 2a7a78d4f..4b8c19b73 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,7 @@ interfaces and configuration objects in Python. Our core API, `tyro.cli()`, -- **Generates CLI interfaces** from a comprehensive set of Python type - constructs. +- **Generates CLI interfaces** from Python type signatures. - **Populates helptext automatically** from defaults, annotations, and docstrings. - **Understands nesting** of `dataclasses`, `pydantic`, and `attrs` structures. @@ -53,12 +52,12 @@ Our core API, `tyro.cli()`, [Pyright](https://github.com/microsoft/pyright), and [mypy](https://github.com/python/mypy). -For power users, it also: +For advanced users, it also supports: -- **Supports subcommands**, as well as choosing between and overriding values in +- **Subcommands**, as well as choosing between and overriding values in configuration objects. -- **Generates shell completion** scripts for `bash`, `zsh`, and `tcsh`. -- **Supports fine-grained configuration** via PEP 529 runtime annotations +- **Completion script generation** for `bash`, `zsh`, and `tcsh`. +- **Fine-grained configuration** via PEP 529 runtime annotations (`tyro.conf.*`). For examples and the API reference, see our @@ -174,7 +173,7 @@ particularly like: integration with configuration file formats like YAML and JSON. - [clipstick](https://github.com/sander76/clipstick), which focuses on generating CLIs from Pydantic models. -- [datargs](https://github.com/roee30/datargs) provides a minimal API for +- [datargs](https://github.com/roee30/datargs), which provides a minimal API for dataclasses. - [fire](https://github.com/google/python-fire) and [clize](https://github.com/epsy/clize), which support arguments without type diff --git a/docs/source/index.md b/docs/source/index.md index 49566a4c5..d274ce0f7 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -18,12 +18,12 @@ Our core API, `tyro.cli()`, [Pyright](https://github.com/microsoft/pyright), and [mypy](https://github.com/python/mypy). -For power users, it also: +For advanced users, it also supports: -- **Supports subcommands**, as well as choosing between and overriding values in +- **Subcommands**, as well as choosing between and overriding values in configuration objects. -- **Generates shell completion** scripts for `bash`, `zsh`, and `tcsh`. -- **Supports fine-grained configuration** via PEP 529 runtime annotations +- **Completion script generation** for `bash`, `zsh`, and `tcsh`. +- **Fine-grained configuration** via PEP 529 runtime annotations (`tyro.conf.*`). To get started, we recommend browsing the examples to the left. diff --git a/examples/04_additional/02_dictionaries.py b/examples/04_additional/02_dictionaries.py index d09641ba0..15d4bbfd0 100644 --- a/examples/04_additional/02_dictionaries.py +++ b/examples/04_additional/02_dictionaries.py @@ -11,10 +11,12 @@ from typing import Dict, Tuple, TypedDict +from typing_extensions import NotRequired + import tyro -class DictionarySchema( +class DictionarySchemaA( TypedDict, # Setting `total=False` specifies that not all keys need to exist. total=False, @@ -23,17 +25,26 @@ class DictionarySchema( betas: Tuple[float, float] +class DictionarySchemaB(TypedDict): + learning_rate: NotRequired[float] + """NotRequired[] specifies that a particular key doesn't need to exist.""" + betas: Tuple[float, float] + + def main( - typed_dict: DictionarySchema, + typed_dict_a: DictionarySchemaA, + typed_dict_b: DictionarySchemaB, standard_dict: Dict[str, float] = { "learning_rate": 3e-4, "beta1": 0.9, "beta2": 0.999, }, ) -> None: - assert isinstance(typed_dict, dict) + assert isinstance(typed_dict_a, dict) + assert isinstance(typed_dict_b, dict) assert isinstance(standard_dict, dict) - print("Typed dict:", typed_dict) + print("Typed dict A:", typed_dict_a) + print("Typed dict B:", typed_dict_b) print("Standard dict:", standard_dict) diff --git a/pyproject.toml b/pyproject.toml index 8b3f5efb1..35fb6ee36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ profile = "black" python_version = "3.8" ignore_missing_imports = true warn_unused_configs = true +exclude = "^tests/test_py311_generated/.*" [tool.coverage.report] exclude_lines = [ diff --git a/tests/conftest.py b/tests/conftest.py index 19cbe79de..1cf00b0bf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,3 +10,6 @@ if not (sys.version_info.major == 3 and sys.version_info.minor >= 10): collect_ignore_glob.append("*_min_py310.py") + +if not (sys.version_info.major == 3 and sys.version_info.minor >= 11): + collect_ignore_glob.append("test_py311_generated/*.py") diff --git a/tests/test_collections.py b/tests/test_collections.py index 59287bb18..ca689b4ee 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -458,3 +458,10 @@ def main(x: tuple = (0, 1, 2, "hello")) -> Any: return x assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == (0, 1, 2, "3") + + +def test_no_type_collections(): + assert tyro.cli(dict, args="a b c d".split(" ")) == {"a": "b", "c": "d"} + assert tyro.cli(list, args="a b c d".split(" ")) == ["a", "b", "c", "d"] + assert tyro.cli(tuple, args="a b c d".split(" ")) == ("a", "b", "c", "d") + assert tyro.cli(set, args="a b c d".split(" ")) == {"a", "b", "c", "d"} diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index bcf33e73f..a1cedb076 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -6,7 +6,7 @@ from typing import Any, Dict, Mapping, NamedTuple, Tuple, Union, cast import pytest -from typing_extensions import Literal, TypedDict +from typing_extensions import Literal, NotRequired, Required, TypedDict import tyro import tyro._strings @@ -98,10 +98,49 @@ class ManyTypesTypedDict(TypedDict, total=False): args="--i 5 --s 5".split(" "), ) == dict(i=5, s="5") + assert tyro.cli(ManyTypesTypedDict, args=[]) == dict() assert tyro.cli(ManyTypesTypedDict, args="--i 5".split(" ")) == dict(i=5) assert tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) == dict(s="5") +def test_total_false_required_typeddict() -> None: + class ManyTypesTypedDict(TypedDict, total=False): + i: int + s: Required[str] + + assert tyro.cli( + ManyTypesTypedDict, + args="--i 5 --s 5".split(" "), + ) == dict(i=5, s="5") + + with pytest.raises(SystemExit): + # `s` is Required[]. + assert tyro.cli(ManyTypesTypedDict, args="--i 5".split(" ")) == dict(i=5) + assert tyro.cli( + ManyTypesTypedDict, args="--i 5".split(" "), default={"s": "5"} + ) == dict(i=5, s="5") + assert tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) == dict(s="5") + + +def test_total_true_not_required_typeddict() -> None: + class ManyTypesTypedDict(TypedDict, total=True): + i: NotRequired[int] + s: str + + assert tyro.cli( + ManyTypesTypedDict, + args="--i 5 --s 5".split(" "), + ) == dict(i=5, s="5") + + with pytest.raises(SystemExit): + # `s` is Required[]. + assert tyro.cli(ManyTypesTypedDict, args="--i 5".split(" ")) == dict(i=5) + assert tyro.cli( + ManyTypesTypedDict, args="--i 5".split(" "), default={"s": "5"} + ) == dict(i=5, s="5") + assert tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) == dict(s="5") + + def test_total_false_nested_typeddict() -> None: class ChildTypedDict(TypedDict, total=False): i: int @@ -110,20 +149,16 @@ class ChildTypedDict(TypedDict, total=False): class ParentTypedDict(TypedDict, total=False): child: ChildTypedDict - with pytest.raises(tyro.UnsupportedTypeAnnotationError): - tyro.cli( - ParentTypedDict, - args="--child.i 5 --child.s 5".split(" "), - ) + assert tyro.cli( + ParentTypedDict, + args="--child.i 5 --child.s 5".split(" "), + ) == {"child": {"i": 5, "s": "5"}} - with pytest.raises(tyro.UnsupportedTypeAnnotationError): - assert ( - tyro.cli( - ParentTypedDict, - args=[""], - ) - == {} - ) + # total=False is ~ignored on the parent. + assert tyro.cli( + ParentTypedDict, + args=[], + ) == {"child": {}} def test_total_false_typeddict_with_nested() -> None: @@ -135,17 +170,17 @@ class ManyTypesTypedDict(TypedDict, total=False): i: int s: Inner - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + # --s.j is (unfortunately) still required. + with pytest.raises(SystemExit): tyro.cli( ManyTypesTypedDict, args="".split(" "), ) - with pytest.raises(tyro.UnsupportedTypeAnnotationError): - tyro.cli( - ManyTypesTypedDict, - args="--x.i 5 --x.s 5 5".split(" "), - ) + assert tyro.cli( + ManyTypesTypedDict, + args="--s.j 5".split(" "), + ) == {"s": Inner(5.0)} def test_total_false_typeddict_with_tuple() -> None: @@ -486,7 +521,7 @@ def test_nested_dict_annotations() -> None: del overrided_config overrided_config = tyro.cli( - Dict[str, Dict[str, str]], + Dict[str, Dict[str, Dict]], default=loaded_config, args=[ "--optimizer.scheduler.schedule-type", diff --git a/tests/test_errors.py b/tests/test_errors.py index 86acdde63..46c8c88de 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -114,7 +114,9 @@ def test_tuple_needs_default() -> None: def main(arg: tuple) -> None: # type: ignore pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + # This formerly raised an error, but now defaults to Tuple[str, ...]. + # with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(SystemExit): tyro.cli(main, args=["--help"]) @@ -148,7 +150,9 @@ def test_ambiguous_sequence() -> None: def main(value: list) -> None: return None - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + # This formerly raised an error, but now defaults to List[str]. + # with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(SystemExit): tyro.cli(main, args=["--help"]) diff --git a/tests/test_py311_generated/README.md b/tests/test_py311_generated/README.md new file mode 100644 index 000000000..534da8d6f --- /dev/null +++ b/tests/test_py311_generated/README.md @@ -0,0 +1,2 @@ +This folder contains autogenerated tests, which replaces `typing_extension` +imports with `typing` imports. These can sometimes break `A is B`-style checks. diff --git a/tests/test_py311_generated/_generate.py b/tests/test_py311_generated/_generate.py new file mode 100644 index 000000000..e07e8c252 --- /dev/null +++ b/tests/test_py311_generated/_generate.py @@ -0,0 +1,9 @@ +"""Generate a Python 3.11 version of tests. This will use imports from `typing` instead +of `typing_extensions`.""" + +import pathlib + +for test_path in pathlib.Path(__file__).absolute().parent.parent.glob("test_*.py"): + ( + pathlib.Path(__file__).absolute().parent / (test_path.stem + "_generated.py") + ).write_text(test_path.read_text().replace("typing_extensions", "typing")) diff --git a/tests/test_py311_generated/test_attrs_generated.py b/tests/test_py311_generated/test_attrs_generated.py new file mode 100644 index 000000000..c8a2ed445 --- /dev/null +++ b/tests/test_py311_generated/test_attrs_generated.py @@ -0,0 +1,102 @@ +import contextlib +import io +import pathlib +from typing import cast + +import attr +import pytest +from attrs import define, field + +import tyro + + +def test_attrs_basic() -> None: + @attr.s + class ManyTypesA: + i: tyro.conf.Positional[int] = attr.ib() + s: str = attr.ib() + f: float = attr.ib() + p: pathlib.Path = attr.ib() + + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( + ManyTypesA, + args=[ + "5", + "--s", + "5", + "--f", + "5", + "--p", + "~", + ], + ) == ManyTypesA(i=5, s="5", f=5.0, p=pathlib.Path("~")) + + +def test_attrs_defaults() -> None: + @attr.s + class ManyTypesB: + i: int = attr.ib() + s: str = attr.ib() + f: float = attr.ib(default=1.0) + + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( + ManyTypesB, + args=[ + "--i", + "5", + "--s", + "5", + ], + ) == ManyTypesB(i=5, s="5", f=1.0) + + +def test_attrs_helptext() -> None: + @attr.s + class Helptext: + """This docstring should be printed as a description.""" + + x: int = attr.ib() # Documentation 1 + + # Documentation 2 + y: int = attr.ib() + + z: int = attr.ib(default=3) + """Documentation 3""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(Helptext, args=["--help"]) + helptext = f.getvalue() + assert tyro._strings.strip_ansi_sequences(cast(str, Helptext.__doc__)) in helptext + + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + assert "Documentation 3" in helptext + + +def test_attrs_next_gen_and_factory() -> None: + @define + class Helptext: + """This docstring should be printed as a description.""" + + x: int # Documentation 1 + + # Documentation 2 + y: int + + z: int = field(factory=lambda: 3) + """Documentation 3""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(Helptext, args=["--help"]) + helptext = f.getvalue() + assert tyro._strings.strip_ansi_sequences(cast(str, Helptext.__doc__)) in helptext + + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + assert "Documentation 3" in helptext diff --git a/tests/test_py311_generated/test_boolean_optional_generated.py b/tests/test_py311_generated/test_boolean_optional_generated.py new file mode 100644 index 000000000..e3358bcf2 --- /dev/null +++ b/tests/test_py311_generated/test_boolean_optional_generated.py @@ -0,0 +1,67 @@ +import dataclasses + +import tyro + + +def test_flag_default_false() -> None: + """Test for argparse.BooleanOptionalAction-style usage.""" + + @dataclasses.dataclass + class A: + x: bool + + assert tyro.cli( + A, + args=["--x"], + default=A(False), + ) == A(True) + + assert tyro.cli( + A, + args=["--no-x"], + default=A(False), + ) == A(False) + + assert tyro.cli( + A, + args=[], + default=A(False), + ) == A(False) + + assert tyro.cli( + tyro.conf.FlagConversionOff[A], + args=["--x", "True"], + default=A(False), + ) == A(True) + + +def test_flag_default_true() -> None: + """Test for argparse.BooleanOptionalAction-style usage.""" + + @dataclasses.dataclass + class A: + x: bool + + assert tyro.cli( + A, + args=["--x"], + default=A(True), + ) == A(True) + + assert tyro.cli( + A, + args=["--no-x"], + default=A(True), + ) == A(False) + + assert tyro.cli( + A, + args=[], + default=A(True), + ) == A(True) + + assert tyro.cli( + tyro.conf.FlagConversionOff[A], + args=["--x", "True"], + default=A(False), + ) == A(True) diff --git a/tests/test_py311_generated/test_collections_generated.py b/tests/test_py311_generated/test_collections_generated.py new file mode 100644 index 000000000..4b8dd3d61 --- /dev/null +++ b/tests/test_py311_generated/test_collections_generated.py @@ -0,0 +1,467 @@ +import collections +import contextlib +import dataclasses +import enum +import io +from typing import ( + Any, + Deque, + Dict, + FrozenSet, + List, + Literal, + Optional, + Sequence, + Set, + Tuple, + Union, +) + +import pytest + +import tyro + + +def test_tuples_fixed() -> None: + @dataclasses.dataclass + class A: + x: Tuple[int, int, int] + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_tuples_fixed_mixed() -> None: + @dataclasses.dataclass + class A: + x: Tuple[int, str, int] + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, "2", 3)) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_tuples_with_default() -> None: + @dataclasses.dataclass + class A: + x: Tuple[int, int, int] = (0, 1, 2) + + assert tyro.cli(A, args=[]) == A(x=(0, 1, 2)) + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + + +def test_tuple_with_literal_and_default() -> None: + @dataclasses.dataclass + class A: + x: Tuple[Literal[1, 2, 3], ...] = (1, 2) + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + assert tyro.cli(A, args=[]) == A(x=(1, 2)) + assert tyro.cli(A, args=["--x"]) == A(x=()) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3", "4"]) + + +def test_positional_tuple_with_literal_and_default() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.Positional[Tuple[Literal[1, 2, 3], ...]] = (1, 2) + + assert tyro.cli(A, args=["1", "2", "3"]) == A(x=(1, 2, 3)) + assert tyro.cli(A, args=[]) == A(x=(1, 2)) + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(A, args=["1", "2", "3", "4"]) + assert "invalid choice" in target.getvalue() + + +def test_tuples_fixed_multitype() -> None: + @dataclasses.dataclass + class A: + x: Tuple[int, str, float] + + assert tyro.cli(A, args=["--x", "1", "2", "3.5"]) == A(x=(1, "2", 3.5)) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_tuples_fixed_bool() -> None: + @dataclasses.dataclass + class A: + x: Tuple[bool, bool, bool] + + assert tyro.cli(A, args=["--x", "True", "True", "False"]) == A( + x=(True, True, False) + ) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_tuples_variable() -> None: + @dataclasses.dataclass + class A: + x: Tuple[int, ...] + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + assert tyro.cli(A, args=["--x"]) == A(x=()) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_tuples_variable_bool() -> None: + @dataclasses.dataclass + class A: + x: Tuple[bool, ...] + + assert tyro.cli(A, args=["--x", "True", "True", "False"]) == A( + x=(True, True, False) + ) + assert tyro.cli(A, args=["--x"]) == A(x=()) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_tuples_variable_optional() -> None: + @dataclasses.dataclass + class A: + x: Optional[Tuple[int, ...]] = None + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=(1, 2, 3)) + assert tyro.cli(A, args=["--x"]) == A(x=()) + assert tyro.cli(A, args=[]) == A(x=None) + + +def test_sequences() -> None: + @dataclasses.dataclass + class A: + x: Sequence[int] + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_lists() -> None: + @dataclasses.dataclass + class A: + x: List[int] + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_list_with_literal() -> None: + @dataclasses.dataclass + class A: + x: List[Literal[1, 2, 3]] + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3", "4"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_list_with_enums() -> None: + class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + @dataclasses.dataclass + class A: + x: List[Color] + + assert tyro.cli(A, args=["--x", "RED", "RED", "BLUE"]) == A( + x=[Color.RED, Color.RED, Color.BLUE] + ) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "RED", "RED", "YELLOW"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_lists_with_default() -> None: + class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + @dataclasses.dataclass + class A: + x: List[Color] = dataclasses.field( + default_factory=[Color.RED, Color.GREEN].copy + ) + + assert tyro.cli(A, args=[]) == A(x=[Color.RED, Color.GREEN]) + assert tyro.cli(A, args=["--x", "RED", "GREEN", "BLUE"]) == A( + x=[Color.RED, Color.GREEN, Color.BLUE] + ) + + +def test_lists_bool() -> None: + @dataclasses.dataclass + class A: + x: List[bool] + + assert tyro.cli(A, args=["--x", "True", "False", "True"]) == A( + x=[True, False, True] + ) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_sets() -> None: + @dataclasses.dataclass + class A: + x: Set[int] + + assert tyro.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) + assert tyro.cli(A, args=["--x"]) == A(set()) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_frozen_sets() -> None: + @dataclasses.dataclass + class A: + x: FrozenSet[int] + + assert tyro.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x=frozenset({1, 2, 3})) + assert tyro.cli(A, args=["--x"]) == A(x=frozenset()) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_deque() -> None: + @dataclasses.dataclass + class A: + x: Deque[int] + + assert tyro.cli(A, args=["--x", "1", "2", "3", "3"]) == A( + x=collections.deque([1, 2, 3, 3]) + ) + assert tyro.cli(A, args=["--x"]) == A(collections.deque()) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_sets_with_default() -> None: + @dataclasses.dataclass + class A: + x: Set[int] = dataclasses.field(default_factory={0, 1, 2}.copy) + + assert tyro.cli(A, args=[]) == A(x={0, 1, 2}) + assert tyro.cli(A, args=["--x", "1", "2", "3", "3"]) == A(x={1, 2, 3}) + assert tyro.cli(A, args=["--x"]) == A(x=set()) + + +def test_optional_sequences() -> None: + @dataclasses.dataclass + class A: + x: Optional[Sequence[int]] = None + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + assert tyro.cli(A, args=["--x", "None"]) == A(x=None) + assert tyro.cli(A, args=[]) == A(x=None) + + +def test_optional_lists() -> None: + @dataclasses.dataclass + class A: + x: Optional[List[int]] = None + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + assert tyro.cli(A, args=["--x", "None"]) == A(x=None) + assert tyro.cli(A, args=[]) == A(x=None) + + +def test_nested_optional_types() -> None: + """We support "None" as a special-case keyword. (note: this is a bit weird because + Optional[str] might interpret "None" as either a string or an actual `None` + value)""" + + @dataclasses.dataclass + class A: + x: Tuple[Optional[int], ...] + + assert tyro.cli(A, args=["--x", "0", "1"]) == A((0, 1)) + assert tyro.cli(A, args=["--x", "0", "None", "1"]) == A((0, None, 1)) + + +def test_union_over_collections() -> None: + def main(a: Union[Tuple[float, ...], Tuple[int, ...]]) -> Any: + return a + + assert tyro.cli(main, args="--a 3.3 3.3 7.0".split(" ")) == (3.3, 3.3, 7.0) + assert tyro.cli(main, args="--a 3 3 7".split(" ")) == (3, 3, 7) + + +def test_union_over_collections_2() -> None: + def main(a: Union[Tuple[str, float, str], Tuple[str, str, float]]) -> Any: + return a + + assert tyro.cli(main, args="--a 3.3 hey 7.0".split(" ")) == ("3.3", "hey", 7.0) + assert tyro.cli(main, args="--a 3 3 hey".split(" ")) == ("3", 3.0, "hey") + + +def test_union_over_collections_3() -> None: + def main(a: Union[Tuple[int, int], Tuple[int, int, int]]) -> Tuple[int, ...]: + return a + + assert tyro.cli(main, args=["--a", "5", "5"]) == (5, 5) + assert tyro.cli(main, args=["--a", "1", "2", "3"]) == (1, 2, 3) + + with pytest.raises(SystemExit): + tyro.cli(main, args=["--a", "5", "5", "2", "1"]) + + with pytest.raises(SystemExit): + tyro.cli(main, args=["--a"]) + with pytest.raises(SystemExit): + tyro.cli(main, args=[]) + + +def test_choices_in_tuples_0() -> None: + @dataclasses.dataclass + class A: + x: Tuple[bool, bool] + + assert tyro.cli(A, args=["--x", "True", "False"]) == A((True, False)) + + +def test_choices_in_tuples_1() -> None: + @dataclasses.dataclass + class A: + x: Tuple[bool, Literal["True", "False"]] + + assert tyro.cli(A, args=["--x", "True", "False"]) == A((True, "False")) + + +def test_choices_in_tuples_2() -> None: + @dataclasses.dataclass + class A: + x: Tuple[bool, Literal["True", "False", "None"]] + + assert tyro.cli(A, args=["--x", "True", "False"]).x == (True, "False") + assert tyro.cli(A, args=["--x", "False", "None"]).x == (False, "None") + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "None", "False"]) + + +def test_nested_tuple_types() -> None: + @dataclasses.dataclass + class A: + x: Tuple[Tuple[int, int], Tuple[str, str]] + + assert tyro.cli(A, args="--x 5 5 5 5".split(" ")).x == ((5, 5), ("5", "5")) + + +def test_variable_nested_tuple() -> None: + def main(x: Tuple[Tuple[int, str], ...]) -> tuple: + return x + + assert tyro.cli(main, args="--x 1 1 2 2".split(" ")) == ((1, "1"), (2, "2")) + with pytest.raises(SystemExit): + tyro.cli(main, args="--x 1 1 2".split(" ")) + + +def test_super_nested() -> None: + def main( + x: Optional[ + List[ + Tuple[ + Optional[int], + Literal[3, 4], + Union[Tuple[int, int], Tuple[str, str]], + ] + ] + ] = None + ) -> Any: + return x + + assert tyro.cli(main, args=[]) is None + assert tyro.cli(main, args="--x None".split(" ")) is None + assert tyro.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] + assert tyro.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] + assert tyro.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ + (2, 3, ("x", "2")), + (2, 3, (1, 2)), + ] + with pytest.raises(SystemExit): + tyro.cli(main, args=["--help"]) + + +def test_dict_no_annotation() -> None: + def main(x: Dict[str, Any] = {"int": 5, "str": "5"}): + return x + + assert tyro.cli(main, args=[]) == {"int": 5, "str": "5"} + assert tyro.cli(main, args="--x.int 3 --x.str 7".split(" ")) == { + "int": 3, + "str": "7", + } + + +def test_double_dict_no_annotation() -> None: + def main( + x: Dict[str, Any] = { + "wow": {"int": 5, "str": "5"}, + } + ): + return x + + assert tyro.cli(main, args=[]) == {"wow": {"int": 5, "str": "5"}} + assert tyro.cli(main, args="--x.wow.int 3 --x.wow.str 7".split(" ")) == { + "wow": { + "int": 3, + "str": "7", + } + } + + +def test_list_narrowing() -> None: + def main(x: list = [0, 1, 2, "hello"]) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", 5] + + +def test_set_narrowing() -> None: + def main(x: set = {0, 1, 2, "hello"}) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", 5} + + +def test_tuple_narrowing() -> None: + def main(x: tuple = (0, 1, 2, "hello")) -> Any: + return x + + assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == (0, 1, 2, "3") + + +def test_no_type_collections(): + assert tyro.cli(dict, args="a b c d".split(" ")) == {"a": "b", "c": "d"} + assert tyro.cli(list, args="a b c d".split(" ")) == ["a", "b", "c", "d"] + assert tyro.cli(tuple, args="a b c d".split(" ")) == ("a", "b", "c", "d") + assert tyro.cli(set, args="a b c d".split(" ")) == {"a", "b", "c", "d"} diff --git a/tests/test_py311_generated/test_completion_generated.py b/tests/test_py311_generated/test_completion_generated.py new file mode 100644 index 000000000..e1d6a9419 --- /dev/null +++ b/tests/test_py311_generated/test_completion_generated.py @@ -0,0 +1,43 @@ +import contextlib +import dataclasses +import io +from typing import Union + +import pytest + +import tyro + + +# https://github.com/brentyi/tyro/issues/9 +@dataclasses.dataclass(frozen=True) +class Subtype: + data: int = 1 + + +@dataclasses.dataclass(frozen=True) +class TypeA: + subtype: Subtype = Subtype(1) + + +@dataclasses.dataclass(frozen=True) +class TypeB: + subtype: Subtype = Subtype(2) + + +@dataclasses.dataclass(frozen=True) +class Wrapper: + supertype: Union[TypeA, TypeB] = TypeA() + + +def test_bash(): + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Wrapper, args=["--tyro-print-completion", "bash"]) + assert "# AUTOMATICALLY GENERATED by `shtab`" in target.getvalue() + + +def test_zsh(): + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Wrapper, args=["--tyro-print-completion", "zsh"]) + assert "# AUTOMATICALLY GENERATED by `shtab`" in target.getvalue() diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py new file mode 100644 index 000000000..d9370e7d5 --- /dev/null +++ b/tests/test_py311_generated/test_conf_generated.py @@ -0,0 +1,902 @@ +import argparse +import dataclasses +from typing import Annotated, Any, Dict, Generic, List, Tuple, TypeVar, Union + +import pytest +from helptext_utils import get_helptext + +import tyro + + +def test_omit_subcommand_prefix() -> None: + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + flag: bool = True + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + bc: tyro.conf.OmitSubcommandPrefixes[ + Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + ] + + assert ( + tyro.cli( + DefaultInstanceSubparser, + args=[ + "--x", + "1", + "bc:default-instance-http-server", + "--y", + "5", + "--no-flag", + ], + ) + == tyro.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--y", "5"], + default=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=3, flag=False) + ), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5, flag=False)) + ) + assert ( + tyro.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--y", "8"], + ) + == tyro.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--y", "8"], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + ) + + +def test_avoid_subparser_with_default() -> None: + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + bc: tyro.conf.AvoidSubcommands[ + Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + ] + + assert ( + tyro.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], + ) + == tyro.cli( + DefaultInstanceSubparser, + args=["--x", "1", "--bc.y", "5"], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) + ) + assert ( + tyro.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], + ) + == tyro.cli( + DefaultInstanceSubparser, + args=["--bc.y", "8"], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + ) + + +def test_avoid_subparser_with_default_recursive() -> None: + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + + assert ( + tyro.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], + ) + == tyro.cli( + tyro.conf.AvoidSubcommands[DefaultInstanceSubparser], + args=["--x", "1", "--bc.y", "5"], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) + ) + assert tyro.cli( + DefaultInstanceSubparser, + args=["bc:default-instance-smtp-server", "--bc.z", "3"], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), + ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) + assert ( + tyro.cli( + tyro.conf.AvoidSubcommands[DefaultInstanceSubparser], + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], + ) + == tyro.cli( + tyro.conf.AvoidSubcommands[DefaultInstanceSubparser], + args=["--bc.y", "8"], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + ) + + +def test_subparser_in_nested_with_metadata() -> None: + @dataclasses.dataclass(frozen=True) + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + @dataclasses.dataclass + class Nested2: + subcommand: Union[ + Annotated[A, tyro.conf.subcommand("command-a", default=A(7))], + Annotated[B, tyro.conf.subcommand("command-b", default=B(9))], + ] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2 + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert tyro.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert tyro.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert tyro.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert tyro.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_subparser_in_nested_with_metadata_generic() -> None: + @dataclasses.dataclass(frozen=True) + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + T = TypeVar("T") + + @dataclasses.dataclass + class Nested2(Generic[T]): + subcommand: T + + @dataclasses.dataclass + class Nested1: + nested2: Nested2[ + Union[ + Annotated[A, tyro.conf.subcommand("command-a", default=A(7))], + Annotated[B, tyro.conf.subcommand("command-b", default=B(9))], + ] + ] + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert tyro.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert tyro.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert tyro.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert tyro.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_subparser_in_nested_with_metadata_generic_alt() -> None: + @dataclasses.dataclass(frozen=True) + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + T = TypeVar("T") + + @dataclasses.dataclass + class Nested2(Generic[T]): + subcommand: Union[ + Annotated[T, tyro.conf.subcommand("command-a", default=A(7))], + Annotated[B, tyro.conf.subcommand("command-b", default=B(9))], + ] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2[A] + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert tyro.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert tyro.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert tyro.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert tyro.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_subparser_in_nested_with_metadata_default_matching() -> None: + @dataclasses.dataclass(frozen=True) + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + default_one = B(3) + default_three = B(9) + + @dataclasses.dataclass + class Nested: + subcommand: Union[ + Annotated[B, tyro.conf.subcommand("one", default=default_one)], + Annotated[B, tyro.conf.subcommand("two")], + Annotated[B, tyro.conf.subcommand("three", default=default_three)], + ] + + # Match by hash. + def main_one(x: Nested = Nested(default_one)) -> None: + pass + + assert "default: x.subcommand:one" in get_helptext(main_one) + + # Match by value. + def main_two(x: Nested = Nested(B(9))) -> None: + pass + + assert "default: x.subcommand:three" in get_helptext(main_two) + + # Match by type. + def main_three(x: Nested = Nested(B(15))) -> None: + pass + + assert "default: x.subcommand:one" in get_helptext(main_three) + + +def test_flag() -> None: + """When boolean flags have no default value, they must be explicitly specified.""" + + @dataclasses.dataclass + class A: + x: bool + + assert tyro.cli( + A, + args=["--x"], + default=A(False), + ) == A(True) + + assert tyro.cli( + tyro.conf.FlagConversionOff[A], + args=["--x", "True"], + default=A(False), + ) == A(True) + + +def test_fixed() -> None: + """When an argument is fixed, we shouldn't be able to override it from the CLI.""" + + @dataclasses.dataclass + class A: + x: tyro.conf.Fixed[bool] + + assert tyro.cli( + A, + args=[], + default=A(True), + ) == A(True) + + with pytest.raises(SystemExit): + assert tyro.cli( + tyro.conf.FlagConversionOff[A], + args=["--x", "True"], + default=A(False), + ) == A(True) + + +def test_fixed_recursive() -> None: + """When an argument is fixed, we shouldn't be able to override it from the CLI.""" + + @dataclasses.dataclass + class A: + x: bool + + assert tyro.cli( + A, + args=["--x"], + default=A(False), + ) == A(True) + + with pytest.raises(SystemExit): + assert tyro.cli( + tyro.conf.Fixed[tyro.conf.FlagConversionOff[A]], + args=["--x", "True"], + default=A(False), + ) == A(True) + + +def test_suppressed_group() -> None: + """Reproduction of https://github.com/nerfstudio-project/nerfstudio/issues/882.""" + + @dataclasses.dataclass + class Inner: + a: int + b: int + + def main( + value: int, + inner: tyro.conf.Suppress[Inner] = Inner(1, 2), + ) -> int: + return value + inner.a + inner.b + + assert tyro.cli(main, args=["--value", "5"]) == 8 + + +def test_fixed_group() -> None: + """Inspired by https://github.com/nerfstudio-project/nerfstudio/issues/882.""" + + @dataclasses.dataclass + class Inner: + a: int + b: int + + def main( + value: int, + inner: tyro.conf.Fixed[Inner] = Inner(1, 2), + ) -> int: + return value + inner.a + inner.b + + assert tyro.cli(main, args=["--value", "5"]) == 8 + + +def test_fixed_suppressed_group() -> None: + """Reproduction of https://github.com/nerfstudio-project/nerfstudio/issues/882.""" + + @dataclasses.dataclass + class Inner: + a: int + b: int + + def main( + value: int, + inner: tyro.conf.Fixed[Inner] = Inner(1, 2), + ) -> int: + return value + inner.a + inner.b + + assert tyro.cli(main, args=["--value", "5"]) == 8 + + +def test_suppressed() -> None: + @dataclasses.dataclass + class Struct: + a: int = 5 + b: tyro.conf.Suppress[str] = "7" + + def main(x: Any = Struct()): + pass + + helptext = get_helptext(main) + assert "--x.a" in helptext + assert "--x.b" not in helptext + + +def test_suppress_manual_fixed() -> None: + @dataclasses.dataclass + class Struct: + a: int = 5 + b: tyro.conf.SuppressFixed[tyro.conf.Fixed[str]] = "7" + + def main(x: Any = Struct()): + pass + + helptext = get_helptext(main) + assert "--x.a" in helptext + assert "--x.b" not in helptext + + +def test_suppress_auto_fixed() -> None: + @dataclasses.dataclass + class Struct: + a: int = 5 + + def b(x): + return 5 + + def main(x: tyro.conf.SuppressFixed[Any] = Struct()): + pass + + helptext = get_helptext(main) + assert "--x.a" in helptext + assert "--x.b" not in helptext + + +def test_argconf_help() -> None: + @dataclasses.dataclass + class Struct: + a: Annotated[ + int, tyro.conf.arg(name="nice", help="Hello world", metavar="NUMBER") + ] = 5 + b: tyro.conf.Suppress[str] = "7" + + def main(x: Any = Struct()) -> int: + return x.a + + helptext = get_helptext(main) + assert "Hello world" in helptext + assert "INT" not in helptext + assert "NUMBER" in helptext + assert "--x.a" not in helptext + assert "--x.nice" in helptext + assert "--x.b" not in helptext + + assert tyro.cli(main, args=[]) == 5 + assert tyro.cli(main, args=["--x.nice", "3"]) == 3 + + +def test_argconf_no_prefix_help() -> None: + @dataclasses.dataclass + class Struct: + a: Annotated[ + int, + tyro.conf.arg( + name="nice", help="Hello world", metavar="NUMBER", prefix_name=False + ), + ] = 5 + b: tyro.conf.Suppress[str] = "7" + + def main(x: Any = Struct()) -> int: + return x.a + + helptext = get_helptext(main) + assert "Hello world" in helptext + assert "INT" not in helptext + assert "NUMBER" in helptext + assert "--x.a" not in helptext + assert "--x.nice" not in helptext + assert "--nice" in helptext + assert "--x.b" not in helptext + + assert tyro.cli(main, args=[]) == 5 + with pytest.raises(SystemExit): + assert tyro.cli(main, args=["--x.nice", "3"]) == 3 + assert tyro.cli(main, args=["--nice", "3"]) == 3 + + +def test_positional() -> None: + def main(x: tyro.conf.Positional[int], y: int) -> int: + return x + y + + assert tyro.cli(main, args="5 --y 3".split(" ")) == 8 + assert tyro.cli(main, args="--y 3 5".split(" ")) == 8 + + +def test_positional_required_args() -> None: + @dataclasses.dataclass + class Args: + x: int + y: int = 3 + + assert tyro.cli( + tyro.conf.PositionalRequiredArgs[Args], args="5 --y 3".split(" ") + ) == Args(5, 3) + assert tyro.cli( + tyro.conf.PositionalRequiredArgs[Args], args="--y 3 5".split(" ") + ) == Args(5, 3) + + +def test_positional_order_swap() -> None: + def main(x: int, y: tyro.conf.Positional[int]) -> int: + return x + y + + assert tyro.cli(main, args="5 --x 3".split(" ")) == 8 + assert tyro.cli(main, args="--x 3 5".split(" ")) == 8 + + +def test_omit_subcommand_prefix_and_consolidate_subcommand_args() -> None: + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + flag: bool = True + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + bc: tyro.conf.OmitSubcommandPrefixes[ + Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + ] = dataclasses.field(default_factory=DefaultInstanceHTTPServer) + + assert ( + tyro.cli( + tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], + args=[ + "bc:default-instance-http-server", + "--x", + "1", + "--y", + "5", + "--no-flag", + ], + ) + == tyro.cli( + tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], + args=[ + "bc:default-instance-http-server", + "--x", + "1", + "--y", + "5", + ], + default=DefaultInstanceSubparser( + x=1, bc=DefaultInstanceHTTPServer(y=3, flag=False) + ), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5, flag=False)) + ) + assert ( + tyro.cli( + tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], + args=[ + "bc:default-instance-http-server", + "--x", + "1", + "--y", + "8", + ], + ) + == tyro.cli( + tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], + args=[ + "bc:default-instance-http-server", + "--x", + "1", + "--y", + "8", + ], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + ) + + # Despite all defaults being set, a subcommand should be required. + with pytest.raises(SystemExit): + tyro.cli(tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[]) + + +def test_omit_subcommand_prefix_and_consolidate_subcommand_args_in_function() -> None: + @tyro.conf.configure(tyro.conf.subcommand(name="http-server")) + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + flag: bool = True + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + + @tyro.conf.configure( + tyro.conf.OmitSubcommandPrefixes, + tyro.conf.ConsolidateSubcommandArgs, + ) + def func(parent: DefaultInstanceSubparser) -> DefaultInstanceSubparser: + return parent + + assert tyro.cli( + func, + args=[ + "parent.bc:http-server", + "--parent.x", + "1", + # --y and --no-flag are in a subcommand with prefix omission. + "--y", + "5", + "--no-flag", + ], + ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5, flag=False)) + assert tyro.cli( + func, + args=[ + "parent.bc:http-server", + "--parent.x", + "1", + # --y is in a subcommand with prefix omission. + "--y", + "8", + ], + ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + + +def test_append_lists() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[List[int]] + + assert tyro.cli(A, args="--x 1 --x 2 --x 3".split(" ")) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=[]) == A(x=[]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3"]) + + +def test_append_tuple() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Tuple[int, ...]] + + assert tyro.cli(A, args="--x 1 --x 2 --x 3".split(" ")) == A(x=(1, 2, 3)) + assert tyro.cli(A, args=[]) == A(x=()) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3"]) + + +def test_append_tuple_with_default() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Tuple[int, ...]] = (1, 2) + + assert tyro.cli(A, args="--x 1 --x 2 --x 3".split(" ")) == A(x=(1, 2, 1, 2, 3)) + assert tyro.cli(A, args=[]) == A(x=(1, 2)) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3"]) + + +def test_append_nested_tuple_fixed_length() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Tuple[Tuple[str, int], ...]] + + assert tyro.cli(A, args="--x 1 1 --x 2 2 --x 3 3".split(" ")) == A( + x=(("1", 1), ("2", 2), ("3", 3)) + ) + assert tyro.cli(A, args=[]) == A(x=()) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3"]) + + +def test_append_nested_tuple_with_default_fixed_length() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Tuple[Tuple[str, int], ...]] = (("1", 1), ("2", 2)) + + assert tyro.cli(A, args="--x 1 1 --x 2 2 --x 3 3".split(" ")) == A( + x=(("1", 1), ("2", 2), ("1", 1), ("2", 2), ("3", 3)) + ) + assert tyro.cli(A, args=[]) == A(x=(("1", 1), ("2", 2))) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3"]) + + +def test_append_dict() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Dict[str, int]] + + assert tyro.cli(A, args="--x 1 1 --x 2 2 --x 3 3".split(" ")) == A( + x={"1": 1, "2": 2, "3": 3} + ) + assert tyro.cli(A, args=[]) == A(x={}) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1", "2", "3"]) + + +def test_append_dict_with_default() -> None: + """Append has no impact when a dictionary has a default value.""" + + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Dict[str, int]] = dataclasses.field( + default_factory=lambda: {"1": 1} + ) + + assert tyro.cli(A, args=[]) == A(x={"1": 1}) + assert tyro.cli(A, args=["--x.1", "2"]) == A(x={"1": 2}) + with pytest.raises(SystemExit): + assert tyro.cli(A, args="--x 2 2 --x 3 3".split(" ")) == A( + x={"1": 1, "2": 2, "3": 3} + ) + + +def test_append_nested_tuple() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Tuple[Tuple[str, ...], ...]] + + assert tyro.cli(A, args="--x 1 2 3 --x 4 5".split(" ")) == A( + x=(("1", "2", "3"), ("4", "5")) + ) + assert tyro.cli(A, args=[]) == A(x=()) + + +def test_append_nested_tuple_with_default() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Tuple[Tuple[str, ...], ...]] = (("1", "2"),) + + assert tyro.cli(A, args="--x 1 2 3 --x 4 5".split(" ")) == A( + x=(("1", "2"), ("1", "2", "3"), ("4", "5")) + ) + assert tyro.cli(A, args=[]) == A(x=(("1", "2"),)) + + +def test_append_nested_list() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[List[List[int]]] + + assert tyro.cli(A, args="--x 1 2 3 --x 4 5".split(" ")) == A(x=[[1, 2, 3], [4, 5]]) + assert tyro.cli(A, args=[]) == A(x=[]) + + +def test_append_nested_dict() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[List[Dict[str, int]]] + + assert tyro.cli(A, args="--x 1 2 3 4 --x 4 5".split(" ")) == A( + x=[{"1": 2, "3": 4}, {"4": 5}] + ) + assert tyro.cli(A, args=[]) == A(x=[]) + + +def test_append_nested_dict_double() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Dict[str, Dict[str, int]]] + + assert tyro.cli(A, args="--x 0 1 2 3 4 --x 4 5 6".split(" ")) == A( + x={"0": {"1": 2, "3": 4}, "4": {"5": 6}} + ) + assert tyro.cli(A, args=[]) == A(x={}) + + +def test_duplicated_arg() -> None: + # Loosely inspired by: https://github.com/brentyi/tyro/issues/49 + @dataclasses.dataclass + class ModelConfig: + num_slots: Annotated[int, tyro.conf.arg(prefix_name=False)] + + @dataclasses.dataclass + class TrainConfig: + num_slots: int + model: ModelConfig + + with pytest.raises(argparse.ArgumentError): + tyro.cli(TrainConfig, args="--num-slots 3".split(" ")) + + +def test_omit_arg_prefixes() -> None: + # Loosely inspired by: https://github.com/brentyi/tyro/issues/49 + @dataclasses.dataclass + class ModelConfig: + num_slots: int + + @dataclasses.dataclass + class TrainConfig: + model: ModelConfig + + assert tyro.cli( + tyro.conf.OmitSubcommandPrefixes[TrainConfig], + args="--model.num-slots 3".split(" "), + ) == TrainConfig(ModelConfig(num_slots=3)) + + assert tyro.cli( + tyro.conf.OmitArgPrefixes[TrainConfig], args="--num-slots 3".split(" ") + ) == TrainConfig(ModelConfig(num_slots=3)) diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py new file mode 100644 index 000000000..b03e7e43a --- /dev/null +++ b/tests/test_py311_generated/test_dcargs_generated.py @@ -0,0 +1,724 @@ +import argparse +import copy +import dataclasses +import enum +import os +import pathlib +from typing import ( + Annotated, + Any, + AnyStr, + Callable, + ClassVar, + Dict, + Final, + List, + Literal, + Optional, + Tuple, + TypeAlias, + TypeVar, + Union, +) + +import pytest +import torch + +import tyro + + +def test_no_args() -> None: + def main() -> int: + return 5 + + assert tyro.cli(main, args=[]) == 5 + with pytest.raises(SystemExit): + tyro.cli(main, args=["3"]) + + +def test_basic() -> None: + @dataclasses.dataclass + class ManyTypes: + i: int + s: str + f: float + p: pathlib.Path + + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( + ManyTypes, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--p", + "~", + ], + ) == ManyTypes(i=5, s="5", f=5.0, p=pathlib.Path("~")) + + # We can directly pass a function to `tyro.cli()`: + def function(i: int, s: str, f: float, p: pathlib.Path) -> ManyTypes: + return ManyTypes(i=i, s=s, f=f, p=p) + + assert tyro.cli( + function, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--p", + "~", + ], + ) == ManyTypes(i=5, s="5", f=5.0, p=pathlib.Path("~")) + + # We can directly pass a generic class to `tyro.cli()`: + class Wrapper: + def __init__(self, i: int, s: str, f: float, p: pathlib.Path): + self.inner = ManyTypes(i=i, s=s, f=f, p=p) + + assert tyro.cli( + Wrapper, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--p", + "~", + ], + ).inner == ManyTypes(i=5, s="5", f=5.0, p=pathlib.Path("~")) + + +def test_init_false() -> None: + @dataclasses.dataclass + class InitFalseDataclass: + i: int + s: str + f: float + dir: pathlib.Path + ignored: str = dataclasses.field(default="hello", init=False) + + assert tyro.cli( + InitFalseDataclass, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--dir", + "~", + ], + ) == InitFalseDataclass(i=5, s="5", f=5.0, dir=pathlib.Path("~")) + + with pytest.raises(SystemExit): + tyro.cli( + InitFalseDataclass, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--dir", + "~", + "--ignored", + "blah", + ], + ) + + +def test_required() -> None: + @dataclasses.dataclass + class A: + x: int + + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_flag() -> None: + """When boolean flags have no default value, they must be explicitly specified.""" + + @dataclasses.dataclass + class A: + x: bool + + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "true"]) + assert tyro.cli(A, args=["--x", "True"]) == A(True) + + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "0"]) + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "false"]) + assert tyro.cli(A, args=["--x", "False"]) == A(False) + + +def test_flag_default_false() -> None: + """When boolean flags default to False, a --flag-name flag must be passed in to flip it to True.""" + + @dataclasses.dataclass + class A: + x: bool = False + + assert tyro.cli(A, args=[]) == A(False) + assert tyro.cli(A, args=["--x"]) == A(True) + + +def test_flag_default_true() -> None: + """When boolean flags default to True, a --no-flag-name flag must be passed in to flip it to False.""" + + @dataclasses.dataclass + class A: + x: bool = True + + assert tyro.cli(A, args=[]) == A(True) + assert tyro.cli(A, args=["--no-x"]) == A(False) + + +def test_flag_default_true_nested() -> None: + """When boolean flags default to True, a --no-flag-name flag must be passed in to flip it to False.""" + + @dataclasses.dataclass + class NestedDefaultTrue: + x: bool = True + + @dataclasses.dataclass + class A: + x: NestedDefaultTrue + + assert tyro.cli(A, args=[]) == A(NestedDefaultTrue(True)) + assert tyro.cli(A, args=["--x.no-x"]) == A(NestedDefaultTrue(False)) + + +def test_default() -> None: + @dataclasses.dataclass + class A: + x: int = 5 + + assert tyro.cli(A, args=[]) == A() + + +def test_default_factory() -> None: + @dataclasses.dataclass + class A: + x: int = dataclasses.field(default_factory=lambda: 5) + + assert tyro.cli(A, args=[]) == A() + + +def test_optional() -> None: + @dataclasses.dataclass + class A: + x: Optional[int] = None + + assert tyro.cli(A, args=[]) == A(x=None) + + +def test_union_basic() -> None: + def main(x: Union[int, str]) -> Union[int, str]: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == 6 + assert tyro.cli(main, args=["--x", "five"]) == "five" + + +def test_union_with_list() -> None: + def main(x: Union[int, str, List[bool]]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == 6 + assert tyro.cli(main, args=["--x", "five"]) == "five" + assert tyro.cli(main, args=["--x", "True"]) == "True" + assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] + + +def test_union_literal() -> None: + def main(x: Union[Literal[1, 2], Literal[3, 4, 5], str]) -> Union[int, str]: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == "6" + assert tyro.cli(main, args=["--x", "five"]) == "five" + + +def test_func_typevar() -> None: + T = TypeVar("T", int, str) + + def main(x: T) -> T: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "five"]) == "five" + + +def test_func_typevar_bound() -> None: + T = TypeVar("T", bound=int) + + def main(x: T) -> T: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + with pytest.raises(SystemExit): + tyro.cli(main, args=["--x", "five"]) + + +def test_enum() -> None: + class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + @dataclasses.dataclass + class EnumClassA: + color: Color + + @dataclasses.dataclass + class EnumClassB: + color: Color = Color.GREEN + + assert tyro.cli(EnumClassA, args=["--color", "RED"]) == EnumClassA(color=Color.RED) + assert tyro.cli(EnumClassB, args=[]) == EnumClassB() + + +def test_literal() -> None: + @dataclasses.dataclass + class A: + x: Literal[0, 1, 2] + + assert tyro.cli(A, args=["--x", "1"]) == A(x=1) + with pytest.raises(SystemExit): + assert tyro.cli(A, args=["--x", "3"]) + + +# Hack for mypy. Not needed for pyright. +Choices = int +Choices = tyro.extras.literal_type_from_choices([0, 1, 2]) # type: ignore + + +def test_dynamic_literal() -> None: + @dataclasses.dataclass + class A: + x: Choices + + assert tyro.cli(A, args=["--x", "1"]) == A(x=1) + with pytest.raises(SystemExit): + assert tyro.cli(A, args=["--x", "3"]) + + +def test_literal_bool() -> None: + def main(x: Literal[True]) -> bool: + return x + + assert tyro.cli(main, args=["--x", "True"]) is True + with pytest.raises(SystemExit): + tyro.cli(main, args=["--x", "False"]) + + def main2(x: Literal[True, False]) -> bool: + return x + + assert tyro.cli(main2, args=["--x", "True"]) is True + assert tyro.cli(main2, args=["--x", "False"]) is False + with pytest.raises(SystemExit): + tyro.cli(main2, args=["--x", "Tru"]) + + +def test_literal_enum() -> None: + class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + @dataclasses.dataclass + class A: + x: Literal[Color.RED, Color.GREEN] + + assert tyro.cli(A, args=["--x", "RED"]) == A(x=Color.RED) + assert tyro.cli(A, args=["--x", "GREEN"]) == A(x=Color.GREEN) + with pytest.raises(SystemExit): + assert tyro.cli(A, args=["--x", "BLUE"]) + + +def test_optional_literal() -> None: + @dataclasses.dataclass + class A: + x: Optional[Literal[0, 1, 2]] = None + + assert tyro.cli(A, args=["--x", "1"]) == A(x=1) + with pytest.raises(SystemExit): + assert tyro.cli(A, args=["--x", "3"]) + assert tyro.cli(A, args=[]) == A(x=None) + + +def test_multitype_literal() -> None: + def main(x: Literal[0, "5"]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "0"]) == 0 + assert tyro.cli(main, args=["--x", "5"]) == "5" + with pytest.raises(SystemExit): + tyro.cli(main, args=["--x", "6"]) + + +def test_annotated() -> None: + """Annotated[] is a no-op.""" + + @dataclasses.dataclass + class A: + x: Annotated[int, "some label"] = 3 + + assert tyro.cli(A, args=["--x", "5"]) == A(x=5) + + +def test_annotated_optional() -> None: + """Annotated[] is a no-op.""" + + @dataclasses.dataclass + class A: + x: Annotated[Optional[int], "some label"] = 3 + + assert tyro.cli(A, args=[]) == A(x=3) + assert tyro.cli(A, args=["--x", "5"]) == A(x=5) + + +def test_optional_annotated() -> None: + """Annotated[] is a no-op.""" + + @dataclasses.dataclass + class A: + x: Optional[Annotated[int, "some label"]] = 3 + + assert tyro.cli(A, args=[]) == A(x=3) + assert tyro.cli(A, args=["--x", "5"]) == A(x=5) + + +def test_final() -> None: + """Final[] is a no-op.""" + + @dataclasses.dataclass + class A: + x: Final[int] = 3 + + assert tyro.cli(A, args=["--x", "5"]) == A(x=5) + + +def test_final_optional() -> None: + @dataclasses.dataclass + class A: + x: Final[Optional[int]] = 3 + + assert tyro.cli(A, args=[]) == A(x=3) + assert tyro.cli(A, args=["--x", "5"]) == A(x=5) + + +def test_classvar() -> None: + """ClassVar[] types should be skipped.""" + + @dataclasses.dataclass + class A: + x: ClassVar[int] = 5 + + with pytest.raises(SystemExit): + tyro.cli(A, args=["--x", "1"]) + assert tyro.cli(A, args=[]) == A() + + +def test_parse_empty_description() -> None: + """If the file has no dosctring, it should be treated as an empty string.""" + + @dataclasses.dataclass + class A: + x: int = 0 + + assert tyro.cli(A, description=None, args=[]) == A(x=0) + + +SomeTypeAlias: TypeAlias = int + + +def test_type_alias() -> None: + def add(a: SomeTypeAlias, b: SomeTypeAlias) -> SomeTypeAlias: + return a + b + + assert tyro.cli(add, args=["--a", "5", "--b", "7"]) == 12 + + +def test_any() -> None: + def main(x: Any = 5) -> Any: + return x + + assert tyro.cli(main, args=[]) == 5 + + +def test_bytes() -> None: + def main(x: bytes) -> bytes: + return x + + assert tyro.cli(main, args=["--x", "hello"]) == b"hello" + + +def test_any_str() -> None: + def main(x: AnyStr) -> AnyStr: + return x + + # Use bytes when provided ascii-compatible inputs. + assert tyro.cli(main, args=["--x", "hello"]) == b"hello" + assert tyro.cli(main, args=["--x", "hello„"]) == "hello„" + + +def test_fixed() -> None: + def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]: + return x + + assert tyro.cli(main, args=[])(3) == 6 + with pytest.raises(SystemExit): + tyro.cli(main, args=["--x", "something"]) + + +def test_fixed_dataclass_type() -> None: + def dummy(): + return 5 # noqa + + def main(x: Callable = dummy) -> Callable: + return x + + assert tyro.cli(main, args=[]) is dummy + with pytest.raises(SystemExit): + tyro.cli(main, args=["--x", "something"]) + + +def test_missing_singleton() -> None: + assert tyro.MISSING is copy.deepcopy(tyro.MISSING) + + +def test_torch_device() -> None: + def main(device: torch.device) -> torch.device: + return device + + assert tyro.cli(main, args=["--device", "cpu"]) == torch.device("cpu") + + +def test_torch_device_2() -> None: + assert tyro.cli(torch.device, args=["cpu"]) == torch.device("cpu") + + +def test_just_int() -> None: + assert tyro.cli(int, args=["123"]) == 123 + + +def test_just_dict() -> None: + assert tyro.cli(Dict[str, str], args="key value key2 value2".split(" ")) == { + "key": "value", + "key2": "value2", + } + + +def test_just_list() -> None: + assert tyro.cli(List[int], args="1 2 3 4".split(" ")) == [1, 2, 3, 4] + + +def test_just_tuple() -> None: + # Need a type: ignore for mypy. Seems like a mypy bug. + assert tyro.cli(Tuple[int, int, int, int], args="1 2 3 4".split(" ")) == ( # type: ignore + 1, + 2, + 3, + 4, + ) + + +def test_return_parser() -> None: + def main() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + return parser + + assert isinstance(tyro.cli(main, args=[]), argparse.ArgumentParser) + + +def test_pathlike_custom_class() -> None: + class CustomPath(pathlib.PurePosixPath): + def __new__(cls, *args: Union[str, os.PathLike]): + return super().__new__(cls, *args) + + def main(a: CustomPath) -> CustomPath: + return a + + assert tyro.cli(main, args=["--a", "/dev/null"]) == CustomPath("/dev/null") + + +def test_class_with_new_and_no_init() -> None: + class A(object): + def __new__(cls, x: int = 5): + return cls._custom_initializer(x) + + @classmethod + def _custom_initializer(cls, x: int = 5): + self = object.__new__(cls) + self.x = x # type: ignore + return self + + def __eq__(self, other) -> bool: + return self.x == other.x # type: ignore + + assert tyro.cli(A, args=["--x", "5"]) == A(x=5) + + +def test_return_unknown_args() -> None: + @dataclasses.dataclass + class A: + x: int = 0 + + a, unknown_args = tyro.cli( + A, args=["positional", "--x", "5", "--y", "7"], return_unknown_args=True + ) + assert a == A(x=5) + assert unknown_args == ["positional", "--y", "7"] + + +def test_unknown_args_with_arg_fixing() -> None: + @dataclasses.dataclass + class A: + x: int = 0 + + a, unknown_args = tyro.cli( + A, + args=["--x", "5", "--a_b", "--a-c"], + return_unknown_args=True, + ) + assert a == A(x=5) + # Should return the unfixed arguments + assert unknown_args == ["--a_b", "--a-c"] + + +def test_allow_ambiguous_args_when_not_returning_unknown_args() -> None: + @dataclasses.dataclass + class A: + a_b: List[int] = dataclasses.field(default_factory=list) + + a = tyro.cli( + A, + args=["--a_b", "5", "--a-b", "7"], + ) + assert a == A(a_b=[7]) + + +def test_disallow_ambiguous_args_when_returning_unknown_args() -> None: + @dataclasses.dataclass + class A: + x: int = 0 + + # If there's an argument that's ambiguous then we should raise an error when we're + # returning unknown args. + with pytest.raises(RuntimeError, match="Ambiguous .* --a_b and --a-b"): + tyro.cli( + A, + args=["--x", "5", "--a_b", "--a-b"], + return_unknown_args=True, + ) + + +def test_unknown_args_with_consistent_duplicates() -> None: + @dataclasses.dataclass + class A: + a_b: List[int] = dataclasses.field(default_factory=list) + c_d: List[int] = dataclasses.field(default_factory=list) + + # Tests logic for consistent duplicate arguments when performing argument fixing. + # i.e., we can fix arguments if the separator is consistent (all _'s or all -'s). + a, unknown_args = tyro.cli( + A, + args=[ + "--a-b", + "5", + "--a-b", + "7", + "--c_d", + "5", + "--c_d", + "7", + "--e-f", + "--e-f", + "--g_h", + "--g_h", + ], + return_unknown_args=True, + ) + assert a == A(a_b=[7], c_d=[7]) + assert unknown_args == ["--e-f", "--e-f", "--g_h", "--g_h"] + + +def test_pathlike(): + def main(x: os.PathLike) -> os.PathLike: + return x + + assert tyro.cli(main, args=["--x", "/dev/null"]) == pathlib.Path("/dev/null") + + +def test_variadics() -> None: + def main(*args: int, **kwargs: float) -> Tuple[Tuple[int, ...], Dict[str, float]]: + return args, kwargs + + assert tyro.cli( + main, args="--args 1 2 3 --kwargs learning_rate 1e-4 beta1 0.99".split(" ") + ) == ((1, 2, 3), {"learning_rate": 1e-4, "beta1": 0.99}) + + +def test_empty_container() -> None: + @dataclasses.dataclass + class A: + x: Tuple[int, ...] = (1, 2, 3) + y: Union[int, str, List[bool]] = dataclasses.field( + default_factory=lambda: [False, False, True] + ) + + assert tyro.cli(A, args="--x".split(" ")).x == () + assert tyro.cli(A, args="--y".split(" ")).y == [] + + +def test_unknown_args_with_consistent_duplicates_use_underscores() -> None: + @dataclasses.dataclass + class A: + a_b: List[int] = dataclasses.field(default_factory=list) + c_d: List[int] = dataclasses.field(default_factory=list) + + # Tests logic for consistent duplicate arguments when performing argument fixing. + # i.e., we can fix arguments if the separator is consistent (all _'s or all -'s). + a, unknown_args = tyro.cli( + A, + args=[ + "--a-b", + "5", + "--a-b", + "7", + "--c_d", + "5", + "--c_d", + "7", + "--e-f", + "--e-f", + "--g_h", + "--g_h", + ], + return_unknown_args=True, + use_underscores=True, + ) + assert a == A(a_b=[7], c_d=[7]) + assert unknown_args == ["--e-f", "--e-f", "--g_h", "--g_h"] diff --git a/tests/test_py311_generated/test_dict_namedtuple_generated.py b/tests/test_py311_generated/test_dict_namedtuple_generated.py new file mode 100644 index 000000000..5fcce0e06 --- /dev/null +++ b/tests/test_py311_generated/test_dict_namedtuple_generated.py @@ -0,0 +1,543 @@ +import contextlib +import copy +import dataclasses +import io +import pathlib +from typing import ( + Any, + Dict, + Literal, + Mapping, + NamedTuple, + NotRequired, + Required, + Tuple, + TypedDict, + Union, + cast, +) + +import pytest + +import tyro +import tyro._strings + + +def test_basic_dict() -> None: + def main(params: Dict[str, int]) -> Dict[str, int]: + return params + + assert tyro.cli(main, args="--params hey 5 hello 2".split(" ")) == { + "hey": 5, + "hello": 2, + } + assert tyro.cli(main, args="--params hey 5 hello 2".split(" ")) == { + "hey": 5, + "hello": 2, + } + assert tyro.cli(main, args="--params".split(" ")) == {} + with pytest.raises(SystemExit): + tyro.cli(main, args="--params hey 5 hello hey".split(" ")) + with pytest.raises(SystemExit): + tyro.cli(main, args="--params hey 5 hello".split(" ")) + + +def test_dict_with_default() -> None: + def main(params: Mapping[Literal[1, 3, 5, 7], bool] = {5: False, 1: True}) -> Any: + return params + + assert tyro.cli(main, args=[]) == {5: False, 1: True} + assert tyro.cli(main, args="--params.5 --params.no-1".split(" ")) == { + 5: True, + 1: False, + } + with pytest.raises(SystemExit): + tyro.cli(main, args="--params".split(" ")) + + +def test_tuple_in_dict() -> None: + def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> dict: + return x + + assert tyro.cli(main, args="--x 1 1 2 2 3 3 4 4".split(" ")) == { + (1, 1): (2, 2), + (3, 3): (4, 4), + } + + +def test_basic_typeddict() -> None: + class ManyTypesTypedDict(TypedDict): + i: int + s: str + + assert tyro.cli( + ManyTypesTypedDict, + args="--i 5 --s 5".split(" "), + ) == dict(i=5, s="5") + + with pytest.raises(SystemExit): + tyro.cli(ManyTypesTypedDict, args="--i 5".split(" ")) + + with pytest.raises(SystemExit): + tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) + + +def test_positional_in_typeddict() -> None: + class ManyTypesTypedDict(TypedDict): + i: tyro.conf.Positional[int] + s: str + + assert tyro.cli( + ManyTypesTypedDict, + args="5 --s 5".split(" "), + ) == dict(i=5, s="5") + + with pytest.raises(SystemExit): + tyro.cli(ManyTypesTypedDict, args="5".split(" ")) + + with pytest.raises(SystemExit): + tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) + + +def test_total_false_typeddict() -> None: + class ManyTypesTypedDict(TypedDict, total=False): + i: int + s: str + + assert tyro.cli( + ManyTypesTypedDict, + args="--i 5 --s 5".split(" "), + ) == dict(i=5, s="5") + + assert tyro.cli(ManyTypesTypedDict, args=[]) == dict() + assert tyro.cli(ManyTypesTypedDict, args="--i 5".split(" ")) == dict(i=5) + assert tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) == dict(s="5") + + +def test_total_false_required_typeddict() -> None: + class ManyTypesTypedDict(TypedDict, total=False): + i: int + s: Required[str] + + assert tyro.cli( + ManyTypesTypedDict, + args="--i 5 --s 5".split(" "), + ) == dict(i=5, s="5") + + with pytest.raises(SystemExit): + # `s` is Required[]. + assert tyro.cli(ManyTypesTypedDict, args="--i 5".split(" ")) == dict(i=5) + assert tyro.cli( + ManyTypesTypedDict, args="--i 5".split(" "), default={"s": "5"} + ) == dict(i=5, s="5") + assert tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) == dict(s="5") + + +def test_total_true_not_required_typeddict() -> None: + class ManyTypesTypedDict(TypedDict, total=True): + i: NotRequired[int] + s: str + + assert tyro.cli( + ManyTypesTypedDict, + args="--i 5 --s 5".split(" "), + ) == dict(i=5, s="5") + + with pytest.raises(SystemExit): + # `s` is Required[]. + assert tyro.cli(ManyTypesTypedDict, args="--i 5".split(" ")) == dict(i=5) + assert tyro.cli( + ManyTypesTypedDict, args="--i 5".split(" "), default={"s": "5"} + ) == dict(i=5, s="5") + assert tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) == dict(s="5") + + +def test_total_false_nested_typeddict() -> None: + class ChildTypedDict(TypedDict, total=False): + i: int + s: str + + class ParentTypedDict(TypedDict, total=False): + child: ChildTypedDict + + assert tyro.cli( + ParentTypedDict, + args="--child.i 5 --child.s 5".split(" "), + ) == {"child": {"i": 5, "s": "5"}} + + # total=False is ~ignored on the parent. + assert tyro.cli( + ParentTypedDict, + args=[], + ) == {"child": {}} + + +def test_total_false_typeddict_with_nested() -> None: + @dataclasses.dataclass + class Inner: + j: float + + class ManyTypesTypedDict(TypedDict, total=False): + i: int + s: Inner + + # --s.j is (unfortunately) still required. + with pytest.raises(SystemExit): + tyro.cli( + ManyTypesTypedDict, + args="".split(" "), + ) + + assert tyro.cli( + ManyTypesTypedDict, + args="--s.j 5".split(" "), + ) == {"s": Inner(5.0)} + + +def test_total_false_typeddict_with_tuple() -> None: + class ManyTypesTypedDict(TypedDict, total=False): + i: int + s: Tuple[str, str] + + assert ( + tyro.cli( + ManyTypesTypedDict, + args=[], + ) + == dict() + ) + + assert tyro.cli( + ManyTypesTypedDict, + args="--i 5 --s 5 5".split(" "), + ) == dict(i=5, s=("5", "5")) + + +def test_nested_typeddict() -> None: + class ChildTypedDict(TypedDict): + y: int + + class NestedTypedDict(TypedDict): + x: int + b: ChildTypedDict + + assert tyro.cli(NestedTypedDict, args=["--x", "1", "--b.y", "3"]) == dict( + x=1, b=dict(y=3) + ) + with pytest.raises(SystemExit): + tyro.cli(NestedTypedDict, args=["--x", "1"]) + + +def test_helptext_and_default_typeddict() -> None: + class HelptextTypedDict(TypedDict): + """This docstring should be printed as a description.""" + + x: int # Documentation 1 + + # Documentation 2 + y: int + + z: int + """Documentation 3""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(HelptextTypedDict, default={"z": 3}, args=["--help"]) + helptext = tyro._strings.strip_ansi_sequences(f.getvalue()) + assert cast(str, HelptextTypedDict.__doc__) in helptext + assert "--x INT" in helptext + assert "--y INT" in helptext + assert "--z INT" in helptext + assert "Documentation 1 (required)" in helptext + assert "Documentation 2 (required)" in helptext + assert "Documentation 3 (default: 3)" in helptext + + +def test_basic_namedtuple() -> None: + class ManyTypesNamedTuple(NamedTuple): + i: int + s: str + f: float + p: pathlib.Path + + assert tyro.cli( + ManyTypesNamedTuple, + args=[ + "--i", + "5", + "--s", + "5", + "--f", + "5", + "--p", + "~", + ], + ) == ManyTypesNamedTuple(i=5, s="5", f=5.0, p=pathlib.Path("~")) + + +def test_nested_namedtuple() -> None: + class ChildNamedTuple(NamedTuple): + y: int + + class NestedNamedTuple(NamedTuple): + x: int + b: ChildNamedTuple + + assert tyro.cli( + NestedNamedTuple, args=["--x", "1", "--b.y", "3"] + ) == NestedNamedTuple(x=1, b=ChildNamedTuple(y=3)) + with pytest.raises(SystemExit): + tyro.cli(NestedNamedTuple, args=["--x", "1"]) + + +def test_helptext_and_default_namedtuple() -> None: + class HelptextNamedTupleDefault(NamedTuple): + """This docstring should be printed as a description.""" + + x: int # Documentation 1 + + # Documentation 2 + y: int + + z: int = 3 + """Documentation 3""" + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(HelptextNamedTupleDefault, args=["--help"]) + helptext = tyro._strings.strip_ansi_sequences(f.getvalue()) + assert cast(str, HelptextNamedTupleDefault.__doc__) in helptext + assert "--x INT" in helptext + assert "--y INT" in helptext + assert "--z INT" in helptext + assert "Documentation 1 (required)" in helptext + assert "Documentation 2 (required)" in helptext + assert "Documentation 3 (default: 3)" in helptext + + +def test_helptext_and_default_namedtuple_alternate() -> None: + class HelptextNamedTuple(NamedTuple): + """This docstring should be printed as a description.""" + + x: int # Documentation 1 + + # Documentation 2 + y: int + + z: int + """Documentation 3""" + + with pytest.raises(SystemExit): + tyro.cli( + HelptextNamedTuple, + default=tyro.MISSING, + args=[], + ) + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli( + HelptextNamedTuple, + default=HelptextNamedTuple( + x=tyro.MISSING, + y=tyro.MISSING, + z=3, + ), + args=["--help"], + ) + helptext = tyro._strings.strip_ansi_sequences(f.getvalue()) + assert cast(str, HelptextNamedTuple.__doc__) in helptext + assert "Documentation 1 (required)" in helptext + assert "Documentation 2 (required)" in helptext + assert "Documentation 3" in helptext + assert "(default: 3)" in helptext + + +def test_nested_dict() -> None: + loaded_config = { + "batch_size": 32, + "optimizer": { + "learning_rate": 1e-4, + "epsilon": 1e-8, + "scheduler": {"schedule_type": "constant"}, + }, + } + backup_config = copy.deepcopy(loaded_config) + overrided_config = tyro.cli( + dict, + default=loaded_config, + args=[ + "--batch-size", + "16", + "--optimizer.scheduler.schedule_type", + "exponential", + ], + ) + + # Overridden config should be different from loaded config. + assert overrided_config != loaded_config + assert overrided_config["batch_size"] == 16 + assert overrided_config["optimizer"]["scheduler"]["schedule_type"] == "exponential" + + # Original loaded config should not be mutated. + assert loaded_config == backup_config + + +def test_nested_dict_use_underscores() -> None: + loaded_config = { + "batch_size": 32, + "optimizer": { + "learning_rate": 1e-4, + "epsilon": 1e-8, + "scheduler": {"schedule_type": "constant"}, + }, + } + backup_config = copy.deepcopy(loaded_config) + overrided_config = tyro.cli( + dict, + default=loaded_config, + args=[ + "--batch-size", + "16", + "--optimizer.scheduler.schedule-type", + "exponential", + ], + use_underscores=True, + ) + + # Overridden config should be different from loaded config. + assert overrided_config != loaded_config + assert overrided_config["batch_size"] == 16 + assert overrided_config["optimizer"]["scheduler"]["schedule_type"] == "exponential" + + # Original loaded config should not be mutated. + assert loaded_config == backup_config + + +def test_nested_dict_hyphen() -> None: + # We do a lot of underscore <=> conversion in the code; this is just to make sure it + # doesn't break anything! + loaded_config = { + "batch-size": 32, + "optimizer": { + "learning-rate": 1e-4, + "epsilon": 1e-8, + "scheduler": {"schedule-type": "constant"}, + }, + } + backup_config = copy.deepcopy(loaded_config) + overrided_config = tyro.cli( + dict, + default=loaded_config, + args=[ + "--batch-size", + "16", + "--optimizer.scheduler.schedule-type", + "exponential", + ], + ) + + # Overridden config should be different from loaded config. + assert overrided_config != loaded_config + assert overrided_config["batch-size"] == 16 + assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" + + # Original loaded config should not be mutated. + assert loaded_config == backup_config + + +def test_nested_dict_hyphen_use_underscores() -> None: + # We do a lot of underscore <=> conversion in the code; this is just to make sure it + # doesn't break anything! + loaded_config = { + "batch-size": 32, + "optimizer": { + "learning-rate": 1e-4, + "epsilon": 1e-8, + "scheduler": {"schedule-type": "constant"}, + }, + } + backup_config = copy.deepcopy(loaded_config) + overrided_config = tyro.cli( + dict, + default=loaded_config, + args=[ + "--batch-size", + "16", + "--optimizer.scheduler.schedule-type", + "exponential", + ], + use_underscores=True, + ) + + # Overridden config should be different from loaded config. + assert overrided_config != loaded_config + assert overrided_config["batch-size"] == 16 + assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" + + # Original loaded config should not be mutated. + assert loaded_config == backup_config + + overrided_config = tyro.cli( + dict, + default=loaded_config, + args=[ + "--batch_size", + "16", + "--optimizer.scheduler.schedule_type", + "exponential", + ], + use_underscores=True, + ) + + # Overridden config should be different from loaded config. + assert overrided_config != loaded_config + assert overrided_config["batch-size"] == 16 + assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" + + # Original loaded config should not be mutated. + assert loaded_config == backup_config + + +def test_nested_dict_annotations() -> None: + loaded_config = { + "optimizer": { + "scheduler": {"schedule-type": "constant"}, + }, + } + + overrided_config = tyro.cli( + dict, + default=loaded_config, + args=[ + "--optimizer.scheduler.schedule-type", + "exponential", + ], + ) + assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" + del overrided_config + + overrided_config = tyro.cli( + Dict[str, Dict], + default=loaded_config, + args=[ + "--optimizer.scheduler.schedule-type", + "exponential", + ], + ) + assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" + del overrided_config + + overrided_config = tyro.cli( + Dict[str, Dict[str, Dict]], + default=loaded_config, + args=[ + "--optimizer.scheduler.schedule-type", + "exponential", + ], + ) + assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" + del overrided_config diff --git a/tests/test_py311_generated/test_dynamic_dataclasses_generated.py b/tests/test_py311_generated/test_dynamic_dataclasses_generated.py new file mode 100644 index 000000000..b0f30091a --- /dev/null +++ b/tests/test_py311_generated/test_dynamic_dataclasses_generated.py @@ -0,0 +1,14 @@ +from dataclasses import field, make_dataclass + +import pytest + +import tyro + + +def test_dynamic(): + B = make_dataclass("B", [("c", int, field())]) + A = make_dataclass("A", [("b", B, field())]) + + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + assert tyro.cli(A, args=["--b.c", "5"]) == A(b=B(c=5)) diff --git a/tests/test_py311_generated/test_errors_generated.py b/tests/test_py311_generated/test_errors_generated.py new file mode 100644 index 000000000..46c8c88de --- /dev/null +++ b/tests/test_py311_generated/test_errors_generated.py @@ -0,0 +1,471 @@ +import contextlib +import dataclasses +import io +from typing import List, Tuple, TypeVar, Union + +import pytest + +import tyro + + +def test_ambiguous_collection_0() -> None: + @dataclasses.dataclass + class A: + x: Tuple[Tuple[int, ...], ...] + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(A, args=["--x", "0", "1"]) + + +def test_ambiguous_collection_1() -> None: + @dataclasses.dataclass + class A: + x: List[List[int]] + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(A, args=["--x", "0", "1"]) + + +def test_ambiguous_collection_2() -> None: + def main(x: Tuple[List[str], List[str]]) -> None: + pass + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) + + +def test_ambiguous_collection_3() -> None: + def main(x: List[Union[Tuple[int, int], Tuple[int, int, int]]]) -> None: + pass + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) + + +# Must be global. +@dataclasses.dataclass +class _CycleDataclass: + x: "_CycleDataclass" + + +def test_cycle() -> None: + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(_CycleDataclass, args=[]) + + +def test_uncallable_annotation() -> None: + def main(arg: 5) -> None: # type: ignore + pass + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=[]) + + +def test_nested_annotation() -> None: + @dataclasses.dataclass + class OneIntArg: + x: int + + def main(arg: List[OneIntArg]) -> List[OneIntArg]: # type: ignore + return arg + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=[]) + + @dataclasses.dataclass + class OneStringArg: + x: str + + def main(arg: List[OneStringArg]) -> List[OneStringArg]: # type: ignore + return arg + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--arg", "0", "1", "2"]) + + @dataclasses.dataclass + class TwoStringArg: + x: str + y: str + + def main2(arg: List[TwoStringArg]) -> None: + pass + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main2, args=[]) + + +def test_missing_annotation_1() -> None: + def main(a, b) -> None: + pass + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) + + +def test_missing_annotation_2() -> None: + def main(*, a) -> None: + pass + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) + + +def test_tuple_needs_default() -> None: + def main(arg: tuple) -> None: # type: ignore + pass + + # This formerly raised an error, but now defaults to Tuple[str, ...]. + # with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(SystemExit): + tyro.cli(main, args=["--help"]) + + +def test_unbound_typevar() -> None: + T = TypeVar("T") + + def main(arg: T) -> None: # type: ignore + pass + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) + + +def test_missing_default_fixed() -> None: + def main(value: tyro.conf.SuppressFixed[tyro.conf.Fixed[int]]) -> int: + return value + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) + + +def test_missing_default_suppressed() -> None: + def main(value: tyro.conf.Suppress[int]) -> int: + return value + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=["--help"]) + + +def test_ambiguous_sequence() -> None: + def main(value: list) -> None: + return None + + # This formerly raised an error, but now defaults to List[str]. + # with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(SystemExit): + tyro.cli(main, args=["--help"]) + + +def test_similar_arguments_basic() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + + @dataclasses.dataclass + class Class: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Class, args="--reward.trac".split(" ")) + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Perhaps you meant:" in error + + # --reward.track should appear in both the usage string and as a similar argument. + assert error.count("--reward.track") == 1 + assert error.count("--help") == 1 + + +def test_similar_arguments_subcommands() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Union[ClassA, ClassB], args="--reward.trac".split(" ")) # type: ignore + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Perhaps you meant:" in error + assert error.count("--reward.track") == 1 + assert error.count("--help") == 3 + + +def test_similar_arguments_subcommands_multiple() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + trace: int + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Union[ClassA, ClassB], args="--fjdkslaj --reward.trac".split(" ")) # type: ignore + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Arguments similar to --reward.trac" in error + assert error.count("--reward.track {True,False}") == 1 + assert error.count("--reward.trace INT") == 1 + assert error.count("--help") == 5 + + +def test_similar_arguments_subcommands_multiple_contains_match() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + trace: int + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Union[ClassA, ClassB], args="--rd.trac".split(" ")) # type: ignore + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Perhaps you meant:" in error + assert error.count("--reward.track {True,False}") == 1 + assert error.count("--reward.trace INT") == 1 + assert error.count("--help") == 5 # 2 subcommands * 2 arguments + usage hint. + + +def test_similar_arguments_subcommands_multiple_contains_match_alt() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + trace: int + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Union[ClassA, ClassB], args="--track".split(" ")) # type: ignore + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Perhaps you meant:" in error + assert error.count("--reward.track {True,False}") == 1 + assert ( + error.count("--help") == 3 + ) # Should show two possible subcommands + usage hint. + + +def test_similar_arguments_subcommands_overflow_different() -> None: + @dataclasses.dataclass + class RewardConfig: + track0: bool + track1: bool + track2: bool + track3: bool + track4: bool + track5: bool + track6: bool + track7: bool + track8: bool + track9: bool + track10: bool + track11: bool + track12: bool + track13: bool + track14: bool + track15: bool + track16: bool + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Union[ClassA, ClassB], args="--track".split(" ")) # type: ignore + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Perhaps you meant:" in error + assert error.count("--reward.track") == 10 + assert "[...]" not in error + assert error.count("--help") == 21 + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + # --tracked is intentionally between 0.8 ~ 0.9 similarity to track{i} for test + # coverage. + tyro.cli(RewardConfig, args="--tracked".split(" ")) # type: ignore + + # Usage print should be clipped. + error = target.getvalue() + assert "For full helptext, run" in error + + +def test_similar_arguments_subcommands_overflow_same() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + @dataclasses.dataclass + class ClassC: + reward: RewardConfig + + @dataclasses.dataclass + class ClassD: + reward: RewardConfig + + @dataclasses.dataclass + class ClassE: + reward: RewardConfig + + @dataclasses.dataclass + class ClassF: + reward: RewardConfig + + @dataclasses.dataclass + class ClassG: + reward: RewardConfig + + @dataclasses.dataclass + class ClassH: + reward: RewardConfig + + @dataclasses.dataclass + class ClassI: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli( # type: ignore + Union[ + ClassA, ClassB, ClassC, ClassD, ClassE, ClassF, ClassG, ClassH, ClassI + ], + args="--track".split(" "), + ) + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Perhaps you meant:" in error + assert error.count("--reward.track") == 1 + assert "[...]" in error + assert error.count("--help") == 5 + + +def test_similar_arguments_subcommands_overflow_same_startswith_multiple() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + + @dataclasses.dataclass + class ClassA: + reward: RewardConfig + + @dataclasses.dataclass + class ClassB: + reward: RewardConfig + + @dataclasses.dataclass + class ClassC: + reward: RewardConfig + + @dataclasses.dataclass + class ClassD: + reward: RewardConfig + + @dataclasses.dataclass + class ClassE: + reward: RewardConfig + rewarde: tyro.conf.Suppress[int] = 10 + + @dataclasses.dataclass + class ClassF: + reward: RewardConfig + + @dataclasses.dataclass + class ClassG: + reward: RewardConfig + + @dataclasses.dataclass + class ClassH: + reward: RewardConfig + + @dataclasses.dataclass + class ClassI: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli( # type: ignore + Union[ + ClassA, ClassB, ClassC, ClassD, ClassE, ClassF, ClassG, ClassH, ClassI + ], + args="--track --ffff".split(" "), + ) + + error = target.getvalue() + assert "Unrecognized argument" in error + assert "Arguments similar to --track" in error + assert error.count("--rewar") == 1 + assert "rewarde" not in error + assert "[...]" in error + assert error.count("--help") == 5 + + +def test_similar_flag() -> None: + @dataclasses.dataclass + class Args: + flag: bool = False + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli( + Args, + args="--lag".split(" "), + ) + + error = target.getvalue() + + # We don't print usage text anymore. + assert error.count("--flag | --no-flag") == 0 + + # Printed in the similar argument list. + assert error.count("--flag, --no-flag") == 1 diff --git a/tests/test_py311_generated/test_flax_min_py38_generated.py b/tests/test_py311_generated/test_flax_min_py38_generated.py new file mode 100644 index 000000000..4448c556b --- /dev/null +++ b/tests/test_py311_generated/test_flax_min_py38_generated.py @@ -0,0 +1,65 @@ +"""Tests initializing flax modules directly via tyro.""" +import jax +import pytest +from flax import linen as nn +from helptext_utils import get_helptext +from jax import numpy as jnp + +import tyro + + +class Classifier(nn.Module): + layers: int + units: int + output_dim: int + + @nn.compact + def __call__(self, x: jnp.ndarray) -> jnp.ndarray: # type: ignore + for i in range(self.layers - 1): + x = nn.Dense( + self.units, + kernel_init=nn.initializers.kaiming_normal(), + )(x) + x = nn.relu(x) + + x = nn.Dense( + self.output_dim, + kernel_init=nn.initializers.xavier_normal(), + )(x) + x = nn.sigmoid(x) + return x + + +def test_ok(): + network = tyro.cli( + Classifier, + args=[ + "--layers", + "3", + "--units", + "8", + "--output-dim", + "3", + ], + ) + + x = jnp.zeros((10, 4)) + params = network.init(jax.random.PRNGKey(0), x) + assert network.apply(params, x).shape == (10, 3) + + helptext = get_helptext(Classifier) + assert "parent" not in helptext + assert "name" not in helptext + + +def test_missing(): + with pytest.raises(SystemExit): + tyro.cli( + Classifier, + args=[ + "--layers", + "3", + "--units", + "8", + ], + ) diff --git a/tests/test_py311_generated/test_forward_ref_generated.py b/tests/test_py311_generated/test_forward_ref_generated.py new file mode 100644 index 000000000..ccd29863e --- /dev/null +++ b/tests/test_py311_generated/test_forward_ref_generated.py @@ -0,0 +1,48 @@ +import dataclasses +from typing import Union + +import pytest + +import tyro + + +@dataclasses.dataclass +class A1: + x: int + bc: "Union[B, C]" + + +@dataclasses.dataclass +class A2: + x: int + bc: Union["B", "C"] + + +@dataclasses.dataclass +class B: + y: "int" + + +@dataclasses.dataclass +class C: + z: int + + +def test_forward_ref_1(): + assert tyro.cli(A1, args=["--x", "1", "bc:b", "--bc.y", "3"]) == A1(x=1, bc=B(y=3)) + assert tyro.cli(A1, args=["--x", "1", "bc:c", "--bc.z", "3"]) == A1(x=1, bc=C(z=3)) + + with pytest.raises(SystemExit): + tyro.cli(A1, args=["--x", "1", "bc:b", "--bc.z", "3"]) + with pytest.raises(SystemExit): + tyro.cli(A1, args=["--x", "1", "bc:c", "--bc.y", "3"]) + + +def test_forward_ref_2(): + assert tyro.cli(A2, args=["--x", "1", "bc:b", "--bc.y", "3"]) == A2(x=1, bc=B(y=3)) + assert tyro.cli(A2, args=["--x", "1", "bc:c", "--bc.z", "3"]) == A2(x=1, bc=C(z=3)) + + with pytest.raises(SystemExit): + tyro.cli(A2, args=["--x", "1", "bc:b", "--bc.z", "3"]) + with pytest.raises(SystemExit): + tyro.cli(A2, args=["--x", "1", "bc:c", "--bc.y", "3"]) diff --git a/tests/test_py311_generated/test_functools_generated.py b/tests/test_py311_generated/test_functools_generated.py new file mode 100644 index 000000000..684534096 --- /dev/null +++ b/tests/test_py311_generated/test_functools_generated.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import dataclasses +import functools + +import pytest +from helptext_utils import get_helptext + +import tyro + + +def test_partial_func() -> None: + def main(a: int, b: str) -> str: + return b * a + + assert tyro.cli(functools.partial(main, a=3), args=["--b", "hi"]) == "hihihi" + + +def test_partial_class() -> None: + class Main: + def __init__(self, a: int, b: str) -> None: + self.inner = b * a + + assert tyro.cli(functools.partial(Main, a=3), args=["--b", "hi"]).inner == "hihihi" + + +def test_partial_helptext_func() -> None: + def main(a: int, b: str) -> str: + """Hello!""" + return b * a + + helptext = get_helptext(functools.partial(main, b="hello world")) + assert "partial" not in helptext + assert "Hello!" in helptext + assert "hello world" in helptext + + +def test_partial_helptext_class() -> None: + class Main: + """Hello!""" + + def __init__(self, a: int, b: str) -> None: + self.inner = b * a + + helptext = get_helptext(functools.partial(Main, b=3)) + assert "partial" not in helptext + assert "Hello!" in helptext + + +def test_wraps_func() -> None: + def main(a: int, b: str) -> str: + return b * a + + @functools.wraps(main) + def wrapper(*args, **kwargs) -> int: + return kwargs["a"] + + assert tyro.cli(wrapper, args=["--a", "3", "--b", "hi"]) == 3 + with pytest.raises(SystemExit): + tyro.cli(wrapper, args=["--a", "3"]) + + +def test_wraps_partial_func_helptext() -> None: + def main(a: int, b: str) -> str: + """Hello! + + Args: + a: Argument. + b: Argument. + """ + return b * a + + @functools.wraps(main) + def wrapper(*args, **kwargs) -> int: + return kwargs["a"] + + assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == 3 + + helptext = get_helptext(functools.partial(wrapper, b=3)) + assert "wraps" not in helptext + assert "Hello!" in helptext + assert "Argument." in helptext + + +def test_wraps_partial_class_helptext() -> None: + class Main: + """Hello!""" + + def __init__(self, a: int, b: str) -> None: + self.inner = b * a + + @functools.wraps(Main) + def wrapper(*args, **kwargs) -> int: + return kwargs["a"] + + assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == 3 + + helptext = get_helptext(functools.partial(wrapper, b=3)) + assert "wraps" not in helptext + assert "Hello!" in helptext + + +@dataclasses.dataclass +class WrappedDataclass: + """Hello!""" + + a: int + b: str + """Second field.""" + + +def test_wraps_partial_dataclass() -> None: + @functools.wraps(WrappedDataclass) + def wrapper(*args, **kwargs) -> str: + return kwargs["a"] * kwargs["b"] + + assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == "hihihi" + assert ( + tyro.cli( + functools.partial(functools.partial(wrapper, a=3), b="hello"), + args=["--b", "hi"], + ) + == "hihihi" + ) + assert ( + tyro.cli(functools.partial(functools.partial(wrapper, a=3), b="hello"), args=[]) + == "hellohellohello" + ) + + helptext = get_helptext(functools.partial(wrapper, b=3)) + assert "wraps" not in helptext + assert "Hello!" in helptext + assert "Second field." in helptext diff --git a/tests/test_py311_generated/test_generics_and_serialization_generated.py b/tests/test_py311_generated/test_generics_and_serialization_generated.py new file mode 100644 index 000000000..54c4633e5 --- /dev/null +++ b/tests/test_py311_generated/test_generics_and_serialization_generated.py @@ -0,0 +1,422 @@ +import contextlib +import dataclasses +import enum +import io +from typing import Annotated, Generic, List, Tuple, Type, TypeVar, Union + +import pytest +import yaml + +import tyro + +T = TypeVar("T") + + +def _check_serialization_identity(cls: Type[T], instance: T) -> None: + assert tyro.extras.from_yaml(cls, tyro.extras.to_yaml(instance)) == instance + + +ScalarType = TypeVar("ScalarType") + + +def test_tuple_generic_variable() -> None: + @dataclasses.dataclass + class TupleGenericVariable(Generic[ScalarType]): + xyz: Tuple[ScalarType, ...] + + assert tyro.cli( + TupleGenericVariable[int], args=["--xyz", "1", "2", "3"] + ) == TupleGenericVariable((1, 2, 3)) + + +def test_tuple_generic_helptext() -> None: + @dataclasses.dataclass + class TupleGenericVariableHelptext(Generic[ScalarType]): + """Helptext!""" + + xyz: Tuple[ScalarType, ...] + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(TupleGenericVariableHelptext[int], args=["--help"]) + helptext = f.getvalue() + assert "Helptext!" in helptext + + +def test_tuple_generic_no_helptext() -> None: + @dataclasses.dataclass + class TupleGenericVariableNoHelptext(Generic[ScalarType]): + xyz: Tuple[ScalarType, ...] + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(TupleGenericVariableNoHelptext[int], args=["--help"]) + helptext = f.getvalue() + assert "Helptext!" not in helptext + + # Check that we don't accidentally grab docstrings from the generic alias! + assert "The central part of internal API" not in helptext + + +def test_tuple_generic_fixed() -> None: + @dataclasses.dataclass + class TupleGenericFixed(Generic[ScalarType]): + xyz: Tuple[ScalarType, ScalarType, ScalarType] + + assert tyro.cli( + TupleGenericFixed[int], args=["--xyz", "1", "2", "3"] + ) == TupleGenericFixed((1, 2, 3)) + + +class CoordinateFrame(enum.Enum): + WORLD = enum.auto() + CAMERA = enum.auto() + + +@dataclasses.dataclass +class Point3(Generic[ScalarType]): + x: ScalarType + y: ScalarType + z: ScalarType + frame: CoordinateFrame + + +def test_simple_generic() -> None: + @dataclasses.dataclass + class SimpleGeneric: + point_continuous: Point3[float] + point_discrete: Point3[int] + + parsed_instance = tyro.cli( + SimpleGeneric, + args=[ + "--point-continuous.x", + "1.2", + "--point-continuous.y", + "2.2", + "--point-continuous.z", + "3.2", + "--point-continuous.frame", + "WORLD", + "--point-discrete.x", + "1", + "--point-discrete.y", + "2", + "--point-discrete.z", + "3", + "--point-discrete.frame", + "WORLD", + ], + ) + assert parsed_instance == SimpleGeneric( + Point3(1.2, 2.2, 3.2, CoordinateFrame.WORLD), + Point3(1, 2, 3, CoordinateFrame.WORLD), + ) + _check_serialization_identity(SimpleGeneric, parsed_instance) + + with pytest.raises(SystemExit): + # Accidentally pass in floats instead of ints for discrete + tyro.cli( + SimpleGeneric, + args=[ + "--point-continuous.x", + "1.2", + "--point-continuous.y", + "2.2", + "--point-continuous.z", + "3.2", + "--point-continuous.frame", + "WORLD", + "--point-discrete.x", + "1.5", + "--point-discrete.y", + "2.5", + "--point-discrete.z", + "3.5", + "--point-discrete.frame", + "WORLD", + ], + ) + + +def test_multilevel_generic() -> None: + @dataclasses.dataclass + class Triangle(Generic[ScalarType]): + a: Point3[ScalarType] + b: Point3[ScalarType] + c: Point3[ScalarType] + + parsed_instance = tyro.cli( + Triangle[float], + args=[ + "--a.x", + "1.0", + "--a.y", + "1.2", + "--a.z", + "1.3", + "--a.frame", + "WORLD", + "--b.x", + "1.0", + "--b.y", + "1.2", + "--b.z", + "1.3", + "--b.frame", + "WORLD", + "--c.x", + "1.0", + "--c.y", + "1.2", + "--c.z", + "1.3", + "--c.frame", + "WORLD", + ], + ) + assert parsed_instance == Triangle( + Point3(1.0, 1.2, 1.3, CoordinateFrame.WORLD), + Point3(1.0, 1.2, 1.3, CoordinateFrame.WORLD), + Point3(1.0, 1.2, 1.3, CoordinateFrame.WORLD), + ) + _check_serialization_identity(Triangle[float], parsed_instance) + + +def test_multilevel_generic_no_helptext() -> None: + @dataclasses.dataclass + class LineSegment(Generic[ScalarType]): + a: Point3[ScalarType] + b: Point3[ScalarType] + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(LineSegment[int], args=["--help"]) + helptext = f.getvalue() + + # Check that we don't accidentally grab docstrings from the generic alias! + assert "The central part of internal API" not in helptext + + +def test_generic_nested_dataclass() -> None: + @dataclasses.dataclass + class Child: + a: int + b: int + + T = TypeVar("T") + + @dataclasses.dataclass + class DataclassGeneric(Generic[T]): + child: T + + parsed_instance = tyro.cli( + DataclassGeneric[Child], args=["--child.a", "5", "--child.b", "7"] + ) + assert parsed_instance == DataclassGeneric(Child(5, 7)) + + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity(DataclassGeneric[Child], parsed_instance) + + +def test_generic_nested_dataclass_helptext() -> None: + @dataclasses.dataclass + class Child: + a: int + b: int + + T = TypeVar("T") + + @dataclasses.dataclass + class DataclassGeneric(Generic[T]): + child: T + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(DataclassGeneric[Child], args=["--help"]) + helptext = f.getvalue() + + # Check that we don't accidentally grab docstrings from the generic alias! + assert "The central part of internal API" not in helptext + + +def test_generic_subparsers() -> None: + @dataclasses.dataclass + class CommandOne: + a: int + + @dataclasses.dataclass + class CommandTwo: + b: int + + T1 = TypeVar("T1") + T2 = TypeVar("T2") + + @dataclasses.dataclass + class Subparser(Generic[T1, T2]): + command: Union[T1, T2] + + parsed_instance = tyro.cli( + Subparser[CommandOne, CommandTwo], + args="command:command-one --command.a 5".split(" "), + ) + # Not supported in mypy. + assert parsed_instance == Subparser(CommandOne(5)) # type: ignore + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity( + Subparser[CommandOne, CommandTwo], parsed_instance + ) + + parsed_instance = tyro.cli( + Subparser[CommandOne, CommandTwo], + args="command:command-two --command.b 7".split(" "), + ) + # Not supported in mypy. + assert parsed_instance == Subparser(CommandTwo(7)) # type: ignore + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity( + Subparser[CommandOne, CommandTwo], parsed_instance + ) + + +def test_generic_subparsers_in_container() -> None: + T = TypeVar("T") + + @dataclasses.dataclass + class Command(Generic[T]): + a: List[T] + + T1 = TypeVar("T1") + T2 = TypeVar("T2") + + @dataclasses.dataclass + class Subparser(Generic[T1, T2]): + command: Union[T1, T2] + + parsed_instance = tyro.cli( + Subparser[Command[int], Command[float]], + args="command:command-int --command.a 5 3".split(" "), + ) + # Not supported in mypy. + assert parsed_instance == Subparser(Command([5, 3])) and isinstance( # type: ignore + parsed_instance.command.a[0], int + ) + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity( + Subparser[Command[int], Command[float]], parsed_instance + ) + + parsed_instance = tyro.cli( + Subparser[Command[int], Command[float]], + args="command:command-float --command.a 7 2".split(" "), + ) + # Not supported in mypy. + assert parsed_instance == Subparser(Command([7.0, 2.0])) and isinstance( # type: ignore + parsed_instance.command.a[0], float + ) + # Local generics will break. + with pytest.raises(yaml.constructor.ConstructorError): + _check_serialization_identity( + Subparser[Command[int], Command[float]], parsed_instance + ) + + +def test_serialize_missing() -> None: + @dataclasses.dataclass + class TupleGenericVariableMissing(Generic[ScalarType]): + xyz: Tuple[ScalarType, ...] + + x = TupleGenericVariableMissing[int](xyz=(tyro.MISSING, tyro.MISSING)) + assert tyro.extras.to_yaml(x).count("!missing") == 2 + _check_serialization_identity(TupleGenericVariableMissing[int], x) + assert ( + tyro.extras.from_yaml( + TupleGenericVariableMissing[int], tyro.extras.to_yaml(x) + ).xyz[0] + is tyro.MISSING + ) + + +def test_generic_inherited_type_narrowing() -> None: + T = TypeVar("T") + + @dataclasses.dataclass + class ActualParentClass(Generic[T]): + x: T # Documentation 1 + + # Documentation 2 + y: T + + z: T = 3 # type: ignore + """Documentation 3""" + + @dataclasses.dataclass + class ChildClass(ActualParentClass[int]): + a: int = 7 + + def main(x: ActualParentClass[int] = ChildClass(5, 5)) -> ActualParentClass: + return x + + assert tyro.cli(main, args="--x.x 3".split(" ")) == ChildClass(3, 5, 3) + + +def test_pculbertson() -> None: + # https://github.com/brentyi/tyro/issues/7 + from typing import Union + + @dataclasses.dataclass(frozen=True) + class TypeA: + data: int + + @dataclasses.dataclass + class TypeB: + data: int + + @dataclasses.dataclass + class Wrapper: + subclass: Union[TypeA, TypeB] = TypeA(1) + + wrapper1 = Wrapper() # Create Wrapper object. + assert wrapper1 == tyro.extras.from_yaml(Wrapper, tyro.extras.to_yaml(wrapper1)) + + +def test_annotated() -> None: + # https://github.com/brentyi/tyro/issues/7 + + @dataclasses.dataclass(frozen=True) + class TypeA: + data: int + + @dataclasses.dataclass + class Wrapper: + subclass: Annotated[TypeA, int] = TypeA(1) + + wrapper1 = Wrapper() # Create Wrapper object. + assert wrapper1 == tyro.extras.from_yaml(Wrapper, tyro.extras.to_yaml(wrapper1)) + + +def test_superclass() -> None: + # https://github.com/brentyi/tyro/issues/7 + + @dataclasses.dataclass + class TypeA: + data: int + + @dataclasses.dataclass + class TypeASubclass(TypeA): + pass + + @dataclasses.dataclass + class Wrapper: + subclass: TypeA + + wrapper1 = Wrapper(TypeASubclass(3)) # Create Wrapper object. + assert wrapper1 == tyro.extras.from_yaml(Wrapper, tyro.extras.to_yaml(wrapper1)) diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py new file mode 100644 index 000000000..2591f18f9 --- /dev/null +++ b/tests/test_py311_generated/test_helptext_generated.py @@ -0,0 +1,698 @@ +import dataclasses +import enum +import os +import pathlib +from collections.abc import Callable +from typing import ( + Annotated, + Any, + Dict, + Generic, + List, + Literal, + Optional, + Tuple, + TypeVar, + Union, + cast, +) + +import torch.nn as nn +from helptext_utils import get_helptext + + +def test_helptext() -> None: + @dataclasses.dataclass + class Helptext: + """This docstring should be printed as a description.""" + + x: int # Documentation 1 + + # Documentation 2 + y: Annotated[int, "ignored"] + + z: int = 3 + """Documentation 3""" + + helptext = get_helptext(Helptext) + assert cast(str, helptext) in helptext + assert "x INT" in helptext + assert "y INT" in helptext + assert "z INT" in helptext + assert "Documentation 1 (required)" in helptext + assert "Documentation 2 (required)" in helptext + assert "Documentation 3 (default: 3)" in helptext + + +def test_helptext_from_class_docstring() -> None: + @dataclasses.dataclass + class Helptext2: + """This docstring should be printed as a description. + + Attributes: + x: Documentation 1 + y: Documentation 2 + z: Documentation 3 + """ + + x: int + y: Annotated[int, "ignored"] + z: int = 3 + + helptext = get_helptext(Helptext2) + assert "This docstring should be printed as a description" in helptext + assert "Attributes" not in helptext + assert "x INT" in helptext + assert "y INT" in helptext + assert "z INT" in helptext + assert "Documentation 1 (required)" in helptext + assert "Documentation 2 (required)" in helptext + assert "Documentation 3 (default: 3)" in helptext + + +def test_helptext_from_class_docstring_args() -> None: + @dataclasses.dataclass + class Helptext3: + """This docstring should be printed as a description. + + Args: + x: Documentation 1 + y: Documentation 2 + z: Documentation 3 + """ + + x: int + y: Annotated[int, "ignored"] + z: int = 3 + + helptext = get_helptext(Helptext3) + assert "This docstring should be printed as a description" in helptext + assert "Args" not in helptext + assert "x INT" in helptext + assert "y INT" in helptext + assert "z INT" in helptext + assert "Documentation 1 (required)" in helptext + assert "Documentation 2 (required)" in helptext + assert "Documentation 3 (default: 3)" in helptext + + +def test_helptext_inherited() -> None: + class UnrelatedParentClass: + pass + + @dataclasses.dataclass + class ActualParentClass: + """This docstring should be printed as a description.""" + + x: int # Documentation 1 + + # Documentation 2 + y: int + + # fmt: off + + z: int = 3 + def some_method(self) -> None: # noqa + """Coverage stress test.""" + # fmt: on + + @dataclasses.dataclass + class ChildClass(UnrelatedParentClass, ActualParentClass): + pass + + helptext = get_helptext(ChildClass) + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + + +def test_helptext_inherited_default_override() -> None: + @dataclasses.dataclass + class ParentClass: + """This docstring should __not__ be printed as a description.""" + + x: int # Documentation 1 + + # Documentation 2 + y: int + + # fmt: off + + z: int = 3 + def some_method(self) -> None: # noqa + """Coverage stress test.""" + # fmt: on + + @dataclasses.dataclass + class ChildClass(ParentClass): + """This docstring should be printed as a description.""" + + def main(x: ParentClass = ChildClass(x=5, y=5)) -> Any: + return x + + helptext = get_helptext(main) + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + assert "__not__" not in helptext + assert "should be printed" in helptext + + +def test_helptext_nested() -> None: + """For nested classes, we should pull helptext from the outermost docstring if + possible. The class docstring can be used as a fallback.""" + + class Inner: + def __init__(self, a: int): + """Something + + Args: + a (int): Hello world! + """ + + def main_with_docstring(a: Inner) -> None: + """main_with_docstring. + + Args: + a: Documented in function. + """ + + def main_no_docstring(a: Inner) -> None: + """main_no_docstring.""" + + helptext = get_helptext(main_with_docstring) + assert "Documented in function" in helptext and str(Inner.__doc__) not in helptext + assert "Args:" not in helptext + assert "Hello world!" in helptext + + helptext = get_helptext(main_no_docstring) + assert "Something" in helptext + assert "Args:" not in helptext + assert "Hello world!" in helptext + + +def test_helptext_defaults() -> None: + class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + @dataclasses.dataclass + class HelptextWithVariousDefaults: + x: pathlib.Path = pathlib.Path("/some/path/to/a/file") + y: Color = Color.RED + z: str = "%" + + helptext = get_helptext(HelptextWithVariousDefaults) + assert "show this help message and exit" in helptext + assert "--x PATH" in helptext + assert "(default: /some/path/to/a/file)" in helptext + assert "--y {RED,GREEN,BLUE}" in helptext + assert "(default: RED)" in helptext + assert "--z STR" in helptext + assert "(default: %)" in helptext + + +def test_multiline_helptext() -> None: + @dataclasses.dataclass + class HelptextMultiline: + x: int # Documentation 1 + + # This comment should be ignored! + + # Documentation 2 + # Next line of documentation 2 + y: int + + z: int = 3 + """Documentation 3 + Next line of documentation 3""" + + helptext = get_helptext(HelptextMultiline) + assert "Documentation 1 (required)" in helptext + assert "Documentation 2" in helptext + assert "documentation 2" in helptext + assert "Documentation 3" in helptext + assert "documentation 3" in helptext + + +def test_grouped_helptext() -> None: + @dataclasses.dataclass + class HelptextGrouped: + x: int # Documentation 1 + # Description of both y and z. + y: int + z: int = 3 + + helptext = get_helptext(HelptextGrouped) + assert "Documentation 1 (required)" in helptext + assert "Description of both y and z. (required)" in helptext + assert "Description of both y and z. (default: 3)" in helptext + + +def test_none_default_value_helptext() -> None: + @dataclasses.dataclass + class Config: + x: Optional[int] = None + """An optional variable.""" + + helptext = get_helptext(Config) + assert "--x {None}|INT" in helptext + assert "An optional variable. (default: None)" in helptext + + +def test_helptext_hard_bool() -> None: + @dataclasses.dataclass + class HelptextHardString: + # fmt: off + x: bool = ( + False + ) + """Helptext. 2% milk.""" + # fmt: on + + # Note that the percent symbol needs some extra handling in argparse. + # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string + + helptext = get_helptext(HelptextHardString) + assert "--x" in helptext + assert "2% milk." in helptext + + +def test_helptext_with_inheritance() -> None: + @dataclasses.dataclass + class Parent: + # fmt: off + x: str = ( + "This docstring may be tougher to parse!" + ) + """Helptext.""" + # fmt: on + + @dataclasses.dataclass + class Child(Parent): + pass + + helptext = get_helptext(Child) + assert "--x STR" in helptext + assert "Helptext." in helptext + assert "(default: 'This docstring" in helptext + + +def test_helptext_with_inheritance_overriden() -> None: + @dataclasses.dataclass + class Parent2: + # fmt: off + x: str = ( + "This docstring may be tougher to parse!" + ) + """Helptext.""" + # fmt: on + + @dataclasses.dataclass + class Child2(Parent2): + # fmt: off + x: str = ( + "This docstring may be tougher to parse?" + ) + """Helptext!""" + # fmt: on + + helptext = get_helptext(Child2) + assert "--x STR" in helptext + assert "Helptext! (default: 'This" in helptext + + +def test_tuple_helptext() -> None: + @dataclasses.dataclass + class TupleHelptext: + x: Tuple[int, str, float] + + helptext = get_helptext(TupleHelptext) + assert "--x INT STR FLOAT" in helptext + + +def test_tuple_helptext_defaults() -> None: + @dataclasses.dataclass + class TupleHelptextDefaults: + x: Tuple[int, str, str] = (5, "hello world", "hello") + + helptext = get_helptext(TupleHelptextDefaults) + assert "--x INT STR STR" in helptext + assert "(default: 5 'hello world' hello)" in helptext + + +def test_generic_helptext() -> None: + T = TypeVar("T") + + @dataclasses.dataclass + class GenericTupleHelptext(Generic[T]): + x: T + + helptext = get_helptext(GenericTupleHelptext[int]) + assert "--x INT" in helptext + + +def test_generic_tuple_helptext() -> None: + T = TypeVar("T") + + @dataclasses.dataclass + class GenericTupleHelptext(Generic[T]): + x: Tuple[T, T, T] + + helptext = get_helptext(GenericTupleHelptext[int]) + assert "--x INT INT INT" in helptext + + +def test_generic_list_helptext() -> None: + T = TypeVar("T") + + @dataclasses.dataclass + class GenericTupleHelptext(Generic[T]): + x: List[T] + + helptext = get_helptext(GenericTupleHelptext[int]) + assert "--x [INT [INT ...]]" in helptext + + +def test_literal_helptext() -> None: + @dataclasses.dataclass + class LiteralHelptext: + x: Literal[1, 2, 3] + """A number.""" + + helptext = get_helptext(LiteralHelptext) + assert "--x {1,2,3}" in helptext + assert "A number. (required)" in helptext + + +def test_optional_literal_helptext() -> None: + @dataclasses.dataclass + class OptionalLiteralHelptext: + x: Optional[Literal[1, 2, 3]] = None + """A number.""" + + helptext = get_helptext(OptionalLiteralHelptext) + assert "--x {None,1,2,3}" in helptext + assert "A number. (default: None)" in helptext + + +def test_multiple_subparsers_helptext() -> None: + @dataclasses.dataclass + class Subcommand1: + """2% milk.""" # % symbol is prone to bugs in argparse. + + x: int = 0 + + @dataclasses.dataclass + class Subcommand2: + y: int = 1 + + @dataclasses.dataclass + class Subcommand3: + z: int = 2 + + @dataclasses.dataclass + class MultipleSubparsers: + # Field a description. + a: Union[Subcommand1, Subcommand2, Subcommand3] + # Field b description. + b: Union[Subcommand1, Subcommand2, Subcommand3] + # Field c description. + c: Union[Subcommand1, Subcommand2, Subcommand3] = dataclasses.field( + default_factory=Subcommand3 + ) + + helptext = get_helptext(MultipleSubparsers) + + assert "2% milk." in helptext + assert "Field a description." in helptext + assert "Field b description." not in helptext + assert "Field c description." not in helptext + + helptext = get_helptext( + MultipleSubparsers, args=["a:subcommand1", "b:subcommand1", "--help"] + ) + + assert "2% milk." in helptext + assert "Field a description." not in helptext + assert "Field b description." not in helptext + assert "Field c description." in helptext + assert "(default: c:subcommand3)" in helptext + + +def test_optional_helptext() -> None: + @dataclasses.dataclass + class OptionalHelptext: + """This docstring should be printed as a description. 2% milk.""" + + x: Optional[int] # Documentation 1 + + # Documentation 2 + y: List[Optional[int]] + + z: Optional[int] = 3 + """Documentation 3""" + + helptext = get_helptext(OptionalHelptext) + assert cast(str, cast(str, OptionalHelptext.__doc__)[:20]) in helptext + assert "2% milk" in helptext + assert "--x {None}|INT" in helptext + assert "--y [{None}|INT [{None}|INT ...]]" in helptext + assert "[--z {None}|INT]" in helptext + + +def test_metavar_0() -> None: + def main(x: Union[Literal[0, 1, 2, 3], Tuple[int, int]]) -> None: + pass + + helptext = get_helptext(main) + assert "--x {0,1,2,3}|{INT INT}" in helptext + + +def test_metavar_1() -> None: + def main( + x: Union[ + Literal[0, 1, 2, 3], + Literal["hey,there", "hello"], + List[int], + ] + ) -> None: + pass + + # The comma formatting is unfortunate, but matches argparse's default behavior. + helptext = get_helptext(main) + assert "--x {0,1,2,3,hey,there,hello}|{[INT [INT ...]]}" in helptext + + +def test_metavar_2() -> None: + def main( + x: Tuple[ + Literal[0, 1, 2, 3], + Union[int, str], + ] + ) -> None: + pass + + helptext = get_helptext(main) + assert "--x {0,1,2,3} INT|STR" in helptext + + +def test_metavar_3() -> None: + def main( + x: Union[ + Literal[0, 1, 2, 3], + Union[Tuple[int, int], Tuple[str]], + ] + ) -> None: + pass + + helptext = get_helptext(main) + assert "--x {0,1,2,3}|{INT INT}|STR" in helptext + + +def test_metavar_4() -> None: + def main( + x: Union[ + Literal[0, 1, 2, 3], + Union[Tuple[int, int], Tuple[str, str, str]], + Literal[True], + ] + ) -> None: + pass + + helptext = get_helptext(main) + assert "--x {0,1,2,3}|{INT INT}|{STR STR STR}|{True}" in helptext + + +def test_metavar_5() -> None: + def main( + x: List[Union[Tuple[int, int], Tuple[str, str]]] = [(1, 1), (2, 2)] + ) -> None: + pass + + helptext = get_helptext(main) + assert "[--x [{INT INT}|{STR STR} [{INT INT}|{STR STR} ...]]]" in helptext + + +def test_metavar_6() -> None: + def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> dict: + return x + + helptext = get_helptext(main) + assert ( + "--x [{INT INT}|{STR STR} INT INT [{INT INT}|{STR STR} INT INT ...]]" + in helptext + ) + + +def test_comment_in_subclass_list() -> None: + @dataclasses.dataclass + class Something( + # This text should not show up in the helptext anywhere. + object, + ): + a: int + + # But this text should! + b: int + + helptext = get_helptext(Something) + assert "This text should not" not in helptext + assert "But this text should!" in helptext + + +def test_unparsable() -> None: + class Struct: + a: int = 5 + b: str = "7" + + def main(x: Any = Struct()): + pass + + helptext = get_helptext(main) + assert "--x {fixed}" in helptext + + def main2(x: Callable = nn.ReLU): + pass + + helptext = get_helptext(main2) + assert "--x {fixed}" in helptext + assert "(fixed to:" in helptext + assert "torch" in helptext + + +def test_pathlike() -> None: + def main(x: os.PathLike) -> None: + pass + + helptext = get_helptext(main) + assert "--x PATH " in helptext + + +def test_nested_bool() -> None: + @dataclasses.dataclass + class Child: + x: bool = False + + def main(child: Child) -> None: + pass + + helptext = get_helptext(main) + assert "--child.x | --child.no-x" in helptext + + +def test_multiple_subparsers_helptext_hyphens() -> None: + @dataclasses.dataclass + class SubcommandOne: + """2% milk.""" # % symbol is prone to bugs in argparse. + + arg_x: int = 0 + arg_flag: bool = False + + @dataclasses.dataclass + class SubcommandTwo: + arg_y: int = 1 + + @dataclasses.dataclass + class SubcommandThree: + arg_z: int = 2 + + @dataclasses.dataclass + class MultipleSubparsers: + # Field a description. + a: Union[SubcommandOne, SubcommandTwo, SubcommandThree] + # Field b description. + b: Union[SubcommandOne, SubcommandTwo, SubcommandThree] + # Field c description. + c: Union[SubcommandOne, SubcommandTwo, SubcommandThree] = dataclasses.field( + default_factory=SubcommandThree + ) + + helptext = get_helptext(MultipleSubparsers) + + assert "2% milk." in helptext + assert "Field a description." in helptext + assert "Field b description." not in helptext + assert "Field c description." not in helptext + + helptext = get_helptext( + MultipleSubparsers, args=["a:subcommand-one", "b:subcommand-one", "--help"] + ) + + assert "2% milk." in helptext + assert "Field a description." not in helptext + assert "Field b description." not in helptext + assert "Field c description." in helptext + assert "(default: c:subcommand-three)" in helptext + assert "--b.arg-x" in helptext + assert "--b.no-arg-flag" in helptext + assert "--b.arg-flag" in helptext + + +def test_multiple_subparsers_helptext_underscores() -> None: + @dataclasses.dataclass + class SubcommandOne: + """2% milk.""" # % symbol is prone to bugs in argparse. + + arg_x: int = 0 + arg_flag: bool = False + + @dataclasses.dataclass + class SubcommandTwo: + arg_y: int = 1 + + @dataclasses.dataclass + class SubcommandThree: + arg_z: int = 2 + + @dataclasses.dataclass + class MultipleSubparsers: + # Field a description. + a: Union[SubcommandOne, SubcommandTwo, SubcommandThree] + # Field b description. + b: Union[SubcommandOne, SubcommandTwo, SubcommandThree] + # Field c description. + c: Union[SubcommandOne, SubcommandTwo, SubcommandThree] = dataclasses.field( + default_factory=SubcommandThree + ) + + helptext = get_helptext(MultipleSubparsers, use_underscores=True) + + assert "2% milk." in helptext + assert "Field a description." in helptext + assert "Field b description." not in helptext + assert "Field c description." not in helptext + + helptext = get_helptext( + MultipleSubparsers, + args=["a:subcommand_one", "b:subcommand_one", "--help"], + use_underscores=True, + ) + + assert "2% milk." in helptext + assert "Field a description." not in helptext + assert "Field b description." not in helptext + assert "Field c description." in helptext + assert "(default: c:subcommand_three)" in helptext + assert "--b.arg_x" in helptext + assert "--b.no_arg_flag" in helptext + assert "--b.arg_flag" in helptext diff --git a/tests/test_py311_generated/test_initvar_min_py38_generated.py b/tests/test_py311_generated/test_initvar_min_py38_generated.py new file mode 100644 index 000000000..721462716 --- /dev/null +++ b/tests/test_py311_generated/test_initvar_min_py38_generated.py @@ -0,0 +1,27 @@ +import dataclasses + +import tyro + + +def test_dataclass_init_var() -> None: + @dataclasses.dataclass + class DataclassWithInitVar: + i: dataclasses.InitVar[int] + x: str + + def __post_init__(self, i: int) -> None: + self.x += str(i) + + # We can directly pass a dataclass to `tyro.cli()`: + assert ( + tyro.cli( + DataclassWithInitVar, + args=[ + "--i", + "5", + "--x", + "5", + ], + ).x + == "55" + ) diff --git a/tests/test_py311_generated/test_is_nested_type_generated.py b/tests/test_py311_generated/test_is_nested_type_generated.py new file mode 100644 index 000000000..a586756f4 --- /dev/null +++ b/tests/test_py311_generated/test_is_nested_type_generated.py @@ -0,0 +1,52 @@ +import dataclasses +import pathlib +from typing import Any, Dict, List, Tuple + +from tyro._fields import MISSING_NONPROP, is_nested_type + + +def test_is_nested_type_simple(): + assert not is_nested_type(int, MISSING_NONPROP) + assert not is_nested_type(bool, MISSING_NONPROP) + assert not is_nested_type(str, MISSING_NONPROP) + assert not is_nested_type(pathlib.Path, MISSING_NONPROP) + + +def test_is_nested_type_containers(): + assert not is_nested_type(List[int], MISSING_NONPROP) + assert not is_nested_type(List[bool], MISSING_NONPROP) + assert not is_nested_type(List[str], MISSING_NONPROP) + assert not is_nested_type(List[pathlib.Path], MISSING_NONPROP) + + +@dataclasses.dataclass +class Color: + r: int + g: int + b: int + + +def test_is_nested_type_actually_nested(): + assert is_nested_type(Color, Color(255, 0, 0)) + + +def test_is_nested_type_actually_nested_narrowing(): + assert is_nested_type(Any, Color(255, 0, 0)) + assert is_nested_type(object, Color(255, 0, 0)) + assert not is_nested_type(int, Color(255, 0, 0)) + + +def test_is_nested_type_actually_nested_in_container(): + assert is_nested_type(Tuple[Color, Color], MISSING_NONPROP) + assert is_nested_type(Tuple[object, ...], (Color(255, 0, 0),)) + assert is_nested_type(Tuple[Any, ...], (Color(255, 0, 0),)) + assert is_nested_type(tuple, (Color(255, 0, 0),)) + assert not is_nested_type(tuple, (1, 2, 3)) + assert is_nested_type(tuple, (1, Color(255, 0, 0), 3)) + assert is_nested_type(List[Any], [Color(255, 0, 0)]) + + +def test_nested_dict(): + assert is_nested_type(Dict[str, int], {"x": 5}) + assert is_nested_type(dict, {"x": 5}) + assert is_nested_type(Any, {"x": 5}) diff --git a/tests/test_py311_generated/test_missing_generated.py b/tests/test_py311_generated/test_missing_generated.py new file mode 100644 index 000000000..3b8b8a47c --- /dev/null +++ b/tests/test_py311_generated/test_missing_generated.py @@ -0,0 +1,73 @@ +"""Tests for tyro.MISSING.""" + +import dataclasses +from typing import Tuple + +import pytest + +import tyro + + +def test_missing() -> None: + def main(a: int = 5, b: int = tyro.MISSING, c: int = 3) -> Tuple[int, int, int]: + return a, b, c + + with pytest.raises(SystemExit): + tyro.cli(main, args=[]) + assert tyro.cli(main, args=["--b", "7"]) == (5, 7, 3) + + +def test_missing_dataclass() -> None: + @dataclasses.dataclass + class Args2: + a: int = 5 + b: int = tyro.MISSING + c: int = 3 + + with pytest.raises(SystemExit): + tyro.cli(Args2, args=[]) + assert tyro.cli(Args2, args=["--b", "7"]) == Args2(5, 7, 3) + + +def test_missing_default() -> None: + @dataclasses.dataclass + class Args2: + a: int + b: int + c: int + + with pytest.raises(SystemExit): + tyro.cli( + Args2, + args=[], + default=Args2(5, tyro.MISSING, 3), + ) + assert tyro.cli( + Args2, + args=["--b", "7"], + default=Args2(5, tyro.MISSING, 3), + ) == Args2(5, 7, 3) + + +def test_missing_nested_default() -> None: + @dataclasses.dataclass + class Child: + a: int = 5 + b: int = 3 + c: int = 7 + + @dataclasses.dataclass + class Parent: + child: Child + + with pytest.raises(SystemExit): + tyro.cli( + Parent, + args=[], + default=Parent(child=tyro.MISSING), + ) + assert tyro.cli( + Parent, + args=["--child.a", "5", "--child.b", "7", "--child.c", "3"], + default=Parent(child=tyro.MISSING), + ) == Parent(Child(5, 7, 3)) diff --git a/tests/test_py311_generated/test_missing_optional_packages_generated.py b/tests/test_py311_generated/test_missing_optional_packages_generated.py new file mode 100644 index 000000000..a093baba0 --- /dev/null +++ b/tests/test_py311_generated/test_missing_optional_packages_generated.py @@ -0,0 +1,25 @@ +import builtins + +import pytest + + +# https://stackoverflow.com/questions/60227582/making-a-python-test-think-an-installed-package-is-not-available +@pytest.fixture +def hide_optional_packages(monkeypatch): + import_orig = builtins.__import__ + + def mocked_import(name, *args, **kwargs): + if name in ("attr", "pydantic", "flax", "omegaconf"): + raise ImportError() + return import_orig(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", mocked_import) + + +def test_missing_optional_packages(hide_optional_packages): + def main(x: int) -> int: + return x + + import tyro + + assert tyro.cli(main, args=["--x", "5"]) == 5 diff --git a/tests/test_py311_generated/test_mixed_unions_generated.py b/tests/test_py311_generated/test_mixed_unions_generated.py new file mode 100644 index 000000000..47fb35833 --- /dev/null +++ b/tests/test_py311_generated/test_mixed_unions_generated.py @@ -0,0 +1,87 @@ +"""Tests for unsupported union types. + +Unions like `int | str` or `SomeDataclassA | SomeDataclassB` are generally OK (note that +the latter will produce a pair of subcommands), but when we write things like +`int | SomeDataclassA` handling gets more complicated; see docstring for +`narrow_union_type()` in _resolvers.py. + +Hopefully we can fix/improve this in the future! +""" + + +import dataclasses +from typing import Union + +import pytest + +import tyro + + +def test_subparser_strip_non_nested() -> None: + @dataclasses.dataclass + class DefaultHTTPServer: + y: int + + @dataclasses.dataclass + class DefaultSMTPServer: + z: int + + @dataclasses.dataclass + class DefaultSubparser: + x: int + # Note that we add [int, str] to the annotation here... this should be ignored. + bc: Union[int, str, DefaultHTTPServer, DefaultSMTPServer] = dataclasses.field( + default_factory=lambda: DefaultHTTPServer(5) + ) + + assert ( + tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "5"] + ) + == tyro.cli(DefaultSubparser, args=["--x", "1"]) + == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=5)) + ) + assert tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-smtp-server", "--bc.z", "3"] + ) == DefaultSubparser(x=1, bc=DefaultSMTPServer(z=3)) + assert ( + tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "8"] + ) + == tyro.cli( + DefaultSubparser, + args=[], + default=DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)), + ) + == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)) + ) + + with pytest.raises(SystemExit): + tyro.cli(DefaultSubparser, args=["--x", "1", "b", "--bc.z", "3"]) + with pytest.raises(SystemExit): + tyro.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) + + +def test_subparser_strip_nested() -> None: + @dataclasses.dataclass + class DefaultHTTPServer: + y: int + + @dataclasses.dataclass + class DefaultSMTPServer: + z: int + + @dataclasses.dataclass + class DefaultSubparser: + x: int + # Note that we add [int, str] to the annotation here... this should be ignored. + bc: Union[int, str, DefaultHTTPServer, DefaultSMTPServer] = 5 + + assert ( + tyro.cli(DefaultSubparser, args=["--x", "1", "--bc", "5"]) + == tyro.cli(DefaultSubparser, args=["--x", "1"]) + == DefaultSubparser(x=1, bc=5) + ) + assert tyro.cli( + DefaultSubparser, args=["--x", "1", "--bc", "five"] + ) == DefaultSubparser(x=1, bc="five") diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py new file mode 100644 index 000000000..79e94a6b1 --- /dev/null +++ b/tests/test_py311_generated/test_nested_generated.py @@ -0,0 +1,1028 @@ +import dataclasses +from typing import ( + Annotated, + Any, + Generic, + Literal, + Mapping, + Optional, + Tuple, + TypeVar, + Union, +) + +import pytest +from frozendict import frozendict # type: ignore + +import tyro + + +def test_nested() -> None: + @dataclasses.dataclass + class B: + y: int + + @dataclasses.dataclass + class Nested: + x: int + b: B + + assert tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) + with pytest.raises(SystemExit): + tyro.cli(Nested, args=["--x", "1"]) + + +def test_nested_annotated() -> None: + @dataclasses.dataclass + class B: + y: int + + @dataclasses.dataclass + class Nested: + x: int + b: Annotated[B, "this should be ignored"] + + assert tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) + with pytest.raises(SystemExit): + tyro.cli(Nested, args=["--x", "1"]) + + +def test_nested_accidental_underscores() -> None: + @dataclasses.dataclass + class B: + arg_name: str + + @dataclasses.dataclass + class Nested: + x: int + child_struct: B + + assert ( + tyro.cli(Nested, args=["--x", "1", "--child-struct.arg-name", "three_five"]) + == tyro.cli(Nested, args=["--x", "1", "--child_struct.arg_name", "three_five"]) + == tyro.cli(Nested, args=["--x", "1", "--child_struct.arg-name", "three_five"]) + == tyro.cli(Nested, args=["--x", "1", "--child_struct.arg_name=three_five"]) + == Nested(x=1, child_struct=B(arg_name="three_five")) + ) + with pytest.raises(SystemExit): + tyro.cli(Nested, args=["--x", "1"]) + + +def test_nested_default() -> None: + @dataclasses.dataclass(frozen=True) + class B: + y: int = 1 + + @dataclasses.dataclass + class Nested: + x: int = 2 + b: B = B() + + assert tyro.cli(Nested, args=[], default=Nested(x=1, b=B(y=2))) == Nested( + x=1, b=B(y=2) + ) + + +def test_nested_default_alternate() -> None: + @dataclasses.dataclass + class B: + y: int = 3 + + @dataclasses.dataclass + class Nested: + x: int + b: B + + assert ( + Nested(x=1, b=B(y=3)) + == tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) + == tyro.cli(Nested, args=[], default=Nested(x=1, b=B(y=3))) + ) + assert tyro.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=3)) + + +def test_default_nested() -> None: + @dataclasses.dataclass(frozen=True) + class B: + y: int = 3 + + @dataclasses.dataclass(frozen=True) + class Nested: + x: int + b: B = B(y=5) + + assert tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) + assert tyro.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) + + +def test_double_default_nested() -> None: + @dataclasses.dataclass(frozen=True) + class Child: + y: int + + @dataclasses.dataclass(frozen=True) + class Parent: + c: Child + + @dataclasses.dataclass(frozen=True) + class Grandparent: + x: int + b: Parent = Parent(Child(y=5)) + + assert tyro.cli(Grandparent, args=["--x", "1", "--b.c.y", "3"]) == Grandparent( + x=1, b=Parent(Child(y=3)) + ) + assert tyro.cli(Grandparent, args=["--x", "1"]) == Grandparent( + x=1, b=Parent(Child(y=5)) + ) + + +def test_default_factory_nested() -> None: + @dataclasses.dataclass + class B: + y: int = 3 + + @dataclasses.dataclass + class Nested: + x: int + b: B = dataclasses.field(default_factory=lambda: B(y=5)) + + assert tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) + assert tyro.cli(Nested, args=["--x", "1"]) == Nested(x=1, b=B(y=5)) + + +def test_optional_nested() -> None: + @dataclasses.dataclass + class OptionalNestedChild: + y: int + z: int + + @dataclasses.dataclass + class OptionalNested: + x: int + b: Optional[OptionalNestedChild] = None + + assert tyro.cli(OptionalNested, args=["--x", "1"]) == OptionalNested(x=1, b=None) + with pytest.raises(SystemExit): + tyro.cli( + OptionalNested, args=["--x", "1", "b:optional-nested-child", "--b.y", "3"] + ) + with pytest.raises(SystemExit): + tyro.cli( + OptionalNested, args=["--x", "1", "b:optional-nested-child", "--b.z", "3"] + ) + + assert tyro.cli( + OptionalNested, + args=["--x", "1", "b:optional-nested-child", "--b.y", "2", "--b.z", "3"], + ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) + + +def test_optional_nested_multiple() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/60""" + + @dataclasses.dataclass(frozen=True) + class OutputHeadSettings: + number_of_outputs: int = 1 + + @dataclasses.dataclass(frozen=True) + class OptimizerSettings: + name: str = "adam" + + @dataclasses.dataclass(frozen=True) + class ModelSettings: + output_head_settings: Optional[OutputHeadSettings] = None + optimizer_settings: Optional[OptimizerSettings] = None + + assert tyro.cli( + ModelSettings, + args="output-head-settings:None optimizer-settings:None".split(" "), + ) == ModelSettings(None, None) + + with pytest.raises(SystemExit): + # Order cannot be flipped, unfortunately. + tyro.cli( + ModelSettings, + args="optimizer-settings:None output-head-settings:None".split(" "), + ) + + assert tyro.cli( + ModelSettings, + args="output-head-settings:output-head-settings optimizer-settings:None".split( + " " + ), + ) == ModelSettings(OutputHeadSettings(1), None) + + assert tyro.cli( + ModelSettings, + args=( + "output-head-settings:output-head-settings" + " --output-head-settings.number-of-outputs 5 optimizer-settings:None".split( + " " + ) + ), + ) == ModelSettings(OutputHeadSettings(5), None) + + assert tyro.cli( + tyro.conf.OmitSubcommandPrefixes[ + tyro.conf.ConsolidateSubcommandArgs[ModelSettings] + ], + args=( + "output-head-settings:output-head-settings optimizer-settings:None" + " --number-of-outputs 5".split(" ") + ), + ) == ModelSettings(OutputHeadSettings(5), None) + + assert tyro.cli( + tyro.conf.OmitSubcommandPrefixes[ + tyro.conf.ConsolidateSubcommandArgs[ModelSettings] + ], + args=( + "output-head-settings:output-head-settings" + " optimizer-settings:optimizer-settings --name sgd --number-of-outputs 5".split( + " " + ) + ), + ) == ModelSettings(OutputHeadSettings(5), OptimizerSettings("sgd")) + + +def test_subparser() -> None: + @dataclasses.dataclass + class HTTPServer: + y: int + + @dataclasses.dataclass + class SMTPServer: + z: int + + @dataclasses.dataclass + class Subparser: + x: int + bc: Union[HTTPServer, SMTPServer] + + assert tyro.cli( + Subparser, args=["--x", "1", "bc:http-server", "--bc.y", "3"] + ) == Subparser(x=1, bc=HTTPServer(y=3)) + assert tyro.cli( + Subparser, args=["--x", "1", "bc:smtp-server", "--bc.z", "3"] + ) == Subparser(x=1, bc=SMTPServer(z=3)) + + with pytest.raises(SystemExit): + # Missing subcommand. + tyro.cli(Subparser, args=["--x", "1"]) + with pytest.raises(SystemExit): + # Wrong field. + tyro.cli(Subparser, args=["--x", "1", "bc:http-server", "--bc.z", "3"]) + with pytest.raises(SystemExit): + # Wrong field. + tyro.cli(Subparser, args=["--x", "1", "bc:smtp-server", "--bc.y", "3"]) + + +def test_subparser_root() -> None: + @dataclasses.dataclass + class HTTPServer: + y: int + + @dataclasses.dataclass + class SMTPServer: + z: int + + @dataclasses.dataclass + class Subparser: + x: int + bc: Union[HTTPServer, SMTPServer] + + assert tyro.cli( + Union[HTTPServer, SMTPServer], args=["http-server", "--y", "3"] # type: ignore + ) == HTTPServer(y=3) + + +def test_subparser_with_default() -> None: + @dataclasses.dataclass + class DefaultHTTPServer: + y: int + + @dataclasses.dataclass + class DefaultSMTPServer: + z: int + + @dataclasses.dataclass + class DefaultSubparser: + x: int + bc: Union[DefaultHTTPServer, DefaultSMTPServer] = dataclasses.field( + default_factory=lambda: DefaultHTTPServer(5) + ) + + assert ( + tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "5"] + ) + == tyro.cli(DefaultSubparser, args=["--x", "1"]) + == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=5)) + ) + assert tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-smtp-server", "--bc.z", "3"] + ) == DefaultSubparser(x=1, bc=DefaultSMTPServer(z=3)) + assert ( + tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "8"] + ) + == tyro.cli( + DefaultSubparser, + args=[], + default=DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)), + ) + == DefaultSubparser(x=1, bc=DefaultHTTPServer(y=8)) + ) + + with pytest.raises(SystemExit): + tyro.cli(DefaultSubparser, args=["--x", "1", "b", "--bc.z", "3"]) + with pytest.raises(SystemExit): + tyro.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) + + +def test_subparser_with_default_alternate() -> None: + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + + assert ( + tyro.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], + ) + == tyro.cli( + DefaultInstanceSubparser, + args=[], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), + ) + == tyro.cli( + DefaultInstanceSubparser, + args=["bc:default-instance-http-server"], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) + ) + assert tyro.cli( + DefaultInstanceSubparser, + args=["bc:default-instance-smtp-server", "--bc.z", "3"], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)), + ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceSMTPServer(z=3)) + assert ( + tyro.cli( + DefaultInstanceSubparser, + args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], + ) + == tyro.cli( + DefaultInstanceSubparser, + args=[], + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)), + ) + == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) + ) + + with pytest.raises(SystemExit): + tyro.cli(DefaultInstanceSubparser, args=["--x", "1", "b", "--bc.z", "3"]) + with pytest.raises(SystemExit): + tyro.cli(DefaultInstanceSubparser, args=["--x", "1", "c", "--bc.y", "3"]) + + +def test_subparser_with_default_bad() -> None: + @dataclasses.dataclass + class DefaultHTTPServer: + y: int + + @dataclasses.dataclass + class DefaultSMTPServer: + z: int + + @dataclasses.dataclass + class DefaultSubparser: + x: int + bc: Union[DefaultHTTPServer, DefaultSMTPServer] = dataclasses.field( + default_factory=lambda: 5 # type: ignore + ) + + # Tolerate bad static types: https://github.com/brentyi/tyro/issues/20 + # Should give us a bunch of warnings! + with pytest.warns(UserWarning): + assert tyro.cli( + DefaultSubparser, args=["--x", "1", "--bc", "3"] + ) == DefaultSubparser( + 1, 3 # type: ignore + ) + + +def test_subparser_with_default_bad_alt() -> None: + @dataclasses.dataclass + class A: + a: int + + @tyro.conf.configure(tyro.conf.subcommand(name="bbbb")) + @dataclasses.dataclass + class B: + b: int + + @dataclasses.dataclass + class C: + c: int + + with pytest.warns(UserWarning): + assert tyro.cli(Union[A, B], default=C(3), args=["--c", "2"]) == C(2) # type: ignore + + +def test_optional_subparser() -> None: + @dataclasses.dataclass + class OptionalHTTPServer: + y: int + + @dataclasses.dataclass + class OptionalSMTPServer: + z: int + + @dataclasses.dataclass + class OptionalSubparser: + x: int + bc: Optional[Union[OptionalHTTPServer, OptionalSMTPServer]] + + assert tyro.cli( + OptionalSubparser, args=["--x", "1", "bc:optional-http-server", "--bc.y", "3"] + ) == OptionalSubparser(x=1, bc=OptionalHTTPServer(y=3)) + assert tyro.cli( + OptionalSubparser, args=["--x", "1", "bc:optional-smtp-server", "--bc.z", "3"] + ) == OptionalSubparser(x=1, bc=OptionalSMTPServer(z=3)) + assert tyro.cli( + OptionalSubparser, args=["--x", "1", "bc:None"] + ) == OptionalSubparser(x=1, bc=None) + + with pytest.raises(SystemExit): + # Wrong field. + tyro.cli( + OptionalSubparser, + args=["--x", "1", "bc:optional-http-server", "--bc.z", "3"], + ) + with pytest.raises(SystemExit): + # Wrong field. + tyro.cli( + OptionalSubparser, + args=["--x", "1", "bc:optional-smtp-server", "--bc.y", "3"], + ) + + +def test_post_init_default() -> None: + @dataclasses.dataclass + class DataclassWithDynamicDefault: + x: int = 3 + y: Optional[int] = None + + def __post_init__(self): + # If unspecified, set y = x. + if self.y is None: + self.y = self.x + + @dataclasses.dataclass + class NoDefaultPostInitArgs: + inner: DataclassWithDynamicDefault + + @dataclasses.dataclass + class DefaultFactoryPostInitArgs: + inner: DataclassWithDynamicDefault = dataclasses.field( + default_factory=DataclassWithDynamicDefault + ) + + assert ( + tyro.cli(NoDefaultPostInitArgs, args=["--inner.x", "5"]).inner + == tyro.cli(DefaultFactoryPostInitArgs, args=["--inner.x", "5"]).inner + == DataclassWithDynamicDefault(x=5, y=5) + ) + + +def test_multiple_subparsers() -> None: + @dataclasses.dataclass + class Subcommand1: + x: int = 0 + + @dataclasses.dataclass + class Subcommand2: + y: int = 1 + + @dataclasses.dataclass + class Subcommand3: + z: int = 2 + + @dataclasses.dataclass + class MultipleSubparsers: + a: Union[Subcommand1, Subcommand2, Subcommand3] + b: Union[Subcommand1, Subcommand2, Subcommand3] + c: Union[Subcommand1, Subcommand2, Subcommand3] + + with pytest.raises(SystemExit): + tyro.cli(MultipleSubparsers, args=[]) + + assert tyro.cli( + MultipleSubparsers, args="a:subcommand1 b:subcommand2 c:subcommand3".split(" ") + ) == MultipleSubparsers(Subcommand1(), Subcommand2(), Subcommand3()) + + assert tyro.cli( + MultipleSubparsers, + args="a:subcommand1 --a.x 5 b:subcommand2 --b.y 7 c:subcommand3 --c.z 3".split( + " " + ), + ) == MultipleSubparsers(Subcommand1(x=5), Subcommand2(y=7), Subcommand3(z=3)) + + assert tyro.cli( + MultipleSubparsers, + args="a:subcommand2 --a.y 5 b:subcommand1 --b.x 7 c:subcommand3 --c.z 3".split( + " " + ), + ) == MultipleSubparsers(Subcommand2(y=5), Subcommand1(x=7), Subcommand3(z=3)) + + assert tyro.cli( + MultipleSubparsers, + args="a:subcommand3 --a.z 5 b:subcommand1 --b.x 7 c:subcommand3 --c.z 3".split( + " " + ), + ) == MultipleSubparsers(Subcommand3(z=5), Subcommand1(x=7), Subcommand3(z=3)) + + +def test_multiple_subparsers_with_default() -> None: + @dataclasses.dataclass(frozen=True) + class Subcommand1: + x: int = 0 + + @dataclasses.dataclass(frozen=True) + class Subcommand2: + y: int = 1 + + @dataclasses.dataclass(frozen=True) + class Subcommand3: + z: int = 2 + + @dataclasses.dataclass + class MultipleSubparsers: + a: Union[Subcommand1, Subcommand2, Subcommand3] = Subcommand1(tyro.MISSING) + b: Union[Subcommand1, Subcommand2, Subcommand3] = Subcommand2(7) + c: Union[Subcommand1, Subcommand2, Subcommand3] = Subcommand3(3) + + with pytest.raises(SystemExit): + tyro.cli( + MultipleSubparsers, + args=[], + ) + + assert tyro.cli( + MultipleSubparsers, + args=["a:subcommand1", "--a.x", "5"], + ) == MultipleSubparsers(Subcommand1(x=5), Subcommand2(y=7), Subcommand3(z=3)) + + assert tyro.cli( + MultipleSubparsers, + args="a:subcommand1 --a.x 3".split(" "), + ) == MultipleSubparsers(Subcommand1(x=3), Subcommand2(y=7), Subcommand3(z=3)) + + with pytest.raises(SystemExit): + tyro.cli( + MultipleSubparsers, + args=[], + default=MultipleSubparsers( + Subcommand1(), + Subcommand2(), + Subcommand3(tyro.MISSING), + ), + ) + with pytest.raises(SystemExit): + tyro.cli( + MultipleSubparsers, + args=[ + "a:subcommand1", + ], + default=MultipleSubparsers( + Subcommand1(), + Subcommand2(), + Subcommand3(tyro.MISSING), + ), + ) + with pytest.raises(SystemExit): + tyro.cli( + MultipleSubparsers, + args=["a:subcommand1", "b:subcommand2"], + default=MultipleSubparsers( + Subcommand1(), + Subcommand2(), + Subcommand3(tyro.MISSING), + ), + ) + with pytest.raises(SystemExit): + tyro.cli( + MultipleSubparsers, + args=["a:subcommand1", "b:subcommand2", "c:subcommand3"], + default=MultipleSubparsers( + Subcommand1(), + Subcommand2(), + Subcommand3(tyro.MISSING), + ), + ) + assert tyro.cli( + MultipleSubparsers, + args=["a:subcommand1", "b:subcommand2", "c:subcommand3", "--c.z", "3"], + default=MultipleSubparsers( + Subcommand1(), + Subcommand2(), + Subcommand3(tyro.MISSING), + ), + ) == MultipleSubparsers(Subcommand1(x=0), Subcommand2(y=1), Subcommand3(z=3)) + assert tyro.cli( + MultipleSubparsers, + args=["a:subcommand1", "b:subcommand2", "c:subcommand2"], + default=MultipleSubparsers( + Subcommand1(), + Subcommand2(), + Subcommand3(tyro.MISSING), + ), + ) == MultipleSubparsers(Subcommand1(x=0), Subcommand2(y=1), Subcommand2(y=1)) + + +def test_nested_subparsers_with_default() -> None: + @dataclasses.dataclass(frozen=True) + class Subcommand1: + x: int = 0 + + @dataclasses.dataclass(frozen=True) + class Subcommand3: + z: int = 2 + + @dataclasses.dataclass(frozen=True) + class Subcommand2: + y: Union[Subcommand1, Subcommand3] + + @dataclasses.dataclass(frozen=True) + class MultipleSubparsers: + a: Union[Subcommand1, Subcommand2] = Subcommand2(Subcommand1(tyro.MISSING)) + + with pytest.raises(SystemExit): + tyro.cli(MultipleSubparsers, args=[]) + with pytest.raises(SystemExit): + tyro.cli(MultipleSubparsers, args=["a:subcommand2"]) + + assert tyro.cli( + MultipleSubparsers, args="a:subcommand1 --a.x 3".split(" ") + ) == MultipleSubparsers(Subcommand1(3)) + assert tyro.cli( + MultipleSubparsers, args="a:subcommand2 a.y:subcommand3 --a.y.z 2".split(" ") + ) == MultipleSubparsers(Subcommand2(Subcommand3())) + assert tyro.cli( + MultipleSubparsers, args="a:subcommand2 a.y:subcommand3 --a.y.z 7".split(" ") + ) == MultipleSubparsers(Subcommand2(Subcommand3(7))) + assert tyro.cli( + MultipleSubparsers, args="a:subcommand2 a.y:subcommand1 --a.y.x 7".split(" ") + ) == MultipleSubparsers(Subcommand2(Subcommand1(7))) + + +def test_nested_subparsers_multiple() -> None: + @dataclasses.dataclass(frozen=True) + class Subcommand1: + x: int = 0 + + @dataclasses.dataclass(frozen=True) + class Subcommand3: + z: int = 2 + + @dataclasses.dataclass(frozen=True) + class Subcommand2: + y: Union[Subcommand1, Subcommand3] + + @dataclasses.dataclass(frozen=True) + class MultipleSubparsers: + a: Union[Subcommand1, Subcommand2] + b: Union[Subcommand1, Subcommand2] + + with pytest.raises(SystemExit): + tyro.cli(MultipleSubparsers, args=[]) + assert tyro.cli( + MultipleSubparsers, args="a:subcommand1 b:subcommand1".split(" ") + ) == MultipleSubparsers(Subcommand1(), Subcommand1()) + assert tyro.cli( + MultipleSubparsers, + args="a:subcommand1 b:subcommand2 b.y:subcommand1".split(" "), + ) == MultipleSubparsers(Subcommand1(), Subcommand2(Subcommand1())) + assert tyro.cli( + MultipleSubparsers, + args="a:subcommand2 a.y:subcommand1 b:subcommand2 b.y:subcommand1".split(" "), + ) == MultipleSubparsers(Subcommand2(Subcommand1()), Subcommand2(Subcommand1())) + assert tyro.cli( + MultipleSubparsers, + args=( + "a:subcommand2 a.y:subcommand1 --a.y.x 3 b:subcommand2 b.y:subcommand1" + " --b.y.x 7".split(" ") + ), + ) == MultipleSubparsers(Subcommand2(Subcommand1(3)), Subcommand2(Subcommand1(7))) + + +def test_nested_subparsers_multiple_in_container() -> None: + @dataclasses.dataclass(frozen=True) + class Subcommand1: + x: int = 0 + + @dataclasses.dataclass(frozen=True) + class Subcommand3: + z: int = 2 + + @dataclasses.dataclass(frozen=True) + class Subcommand2: + y: Union[Subcommand1, Subcommand3] + + @dataclasses.dataclass(frozen=True) + class MultipleSubparsers: + a: Union[Subcommand1, Subcommand2] + b: Union[Subcommand1, Subcommand2] + + @dataclasses.dataclass(frozen=True) + class Root: + inner: MultipleSubparsers + + with pytest.raises(SystemExit): + tyro.cli(Root, args=[]) + assert tyro.cli( + Root, args="inner.a:subcommand1 inner.b:subcommand1".split(" ") + ) == Root(MultipleSubparsers(Subcommand1(), Subcommand1())) + assert tyro.cli( + Root, + args="inner.a:subcommand1 inner.b:subcommand2 inner.b.y:subcommand1".split(" "), + ) == Root(MultipleSubparsers(Subcommand1(), Subcommand2(Subcommand1()))) + assert tyro.cli( + Root, + args=( + "inner.a:subcommand2 inner.a.y:subcommand1 inner.b:subcommand2" + " inner.b.y:subcommand1".split(" ") + ), + ) == Root( + MultipleSubparsers(Subcommand2(Subcommand1()), Subcommand2(Subcommand1())) + ) + assert tyro.cli( + Root, + args=( + "inner.a:subcommand2 inner.a.y:subcommand1 --inner.a.y.x 3" + " inner.b:subcommand2 inner.b.y:subcommand1 --inner.b.y.x 7".split(" ") + ), + ) == Root( + MultipleSubparsers(Subcommand2(Subcommand1(3)), Subcommand2(Subcommand1(7))) + ) + + +def test_tuple_nesting() -> None: + @dataclasses.dataclass(frozen=True) + class Color: + r: int + g: int + b: int + + @dataclasses.dataclass(frozen=True) + class Location: + x: float + y: float + z: float + + def main(x: Tuple[Tuple[Color], Location, float]): + return x + + assert tyro.cli( + main, + args=( + "--x.0.0.r 255 --x.0.0.g 0 --x.0.0.b 0 --x.1.x 5.0 --x.1.y 0.0" + " --x.1.z 2.0 --x.2 4.0".split(" ") + ), + ) == ((Color(255, 0, 0),), Location(5.0, 0.0, 2.0), 4.0) + + +def test_tuple_nesting_union() -> None: + @dataclasses.dataclass(frozen=True) + class Color: + r: int + g: int + b: int + + @dataclasses.dataclass(frozen=True) + class Location: + x: float + y: float + z: float + + def main(x: Union[Tuple[Tuple[Color], Location, float], Tuple[Color, Color]]): + return x + + assert tyro.cli( + main, + args=( + "x:tuple-tuple-color-location-float --x.0.0.r 255 --x.0.0.g 0 --x.0.0.b 0" + " --x.1.x 5.0 --x.1.y 0.0 --x.1.z 2.0 --x.2 4.0".split(" ") + ), + ) == ((Color(255, 0, 0),), Location(5.0, 0.0, 2.0), 4.0) + + +def test_generic_subparsers() -> None: + T = TypeVar("T") + + @dataclasses.dataclass + class A(Generic[T]): + x: T + + def main(x: Union[A[int], A[float]]) -> Any: + return x + + assert tyro.cli(main, args="x:a-float --x.x 3.2".split(" ")) == A(3.2) + assert tyro.cli(main, args="x:a-int --x.x 3".split(" ")) == A(3) + + def main_with_default(x: Union[A[int], A[float]] = A(5)) -> Any: + return x + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main_with_default, args=[]) + + +def test_generic_inherited() -> None: + class UnrelatedParentClass: + pass + + T = TypeVar("T") + + @dataclasses.dataclass + class ActualParentClass(Generic[T]): + x: T # Documentation 1 + + # Documentation 2 + y: T + + z: T = 3 # type: ignore + """Documentation 3""" + + @dataclasses.dataclass + class ChildClass(UnrelatedParentClass, ActualParentClass[int]): + pass + + assert tyro.cli( + ChildClass, args=["--x", "1", "--y", "2", "--z", "3"] + ) == ChildClass(x=1, y=2, z=3) + + +def test_subparser_in_nested() -> None: + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + + @dataclasses.dataclass + class Nested2: + subcommand: Union[A, B] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2 + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert tyro.cli( + Parent, + args="nested1.nested2.subcommand:a --nested1.nested2.subcommand.a 3".split(" "), + ) == Parent(Nested1(Nested2(A(3)))) + assert tyro.cli( + Parent, + args="nested1.nested2.subcommand:b --nested1.nested2.subcommand.b 7".split(" "), + ) == Parent(Nested1(Nested2(B(7)))) + + +def test_frozen_dict() -> None: + def main( + x: Mapping[str, float] = frozendict( # type: ignore + { + "num_epochs": 20, + "batch_size": 64, + } + ) + ): + return x + + assert hash(tyro.cli(main, args="--x.num-epochs 10".split(" "))) == hash( + frozendict({"num_epochs": 10, "batch_size": 64}) # type: ignore + ) + + +def test_nested_in_subparser() -> None: + # https://github.com/brentyi/tyro/issues/9 + @dataclasses.dataclass(frozen=True) + class Subtype: + data: int = 1 + + @dataclasses.dataclass(frozen=True) + class TypeA: + subtype: Subtype = Subtype(1) + + @dataclasses.dataclass(frozen=True) + class TypeB: + subtype: Subtype = Subtype(2) + + @dataclasses.dataclass(frozen=True) + class Wrapper: + supertype: Union[TypeA, TypeB] = TypeA() + + assert tyro.cli(Wrapper, args=[]) == Wrapper() + assert ( + tyro.cli(Wrapper, args="supertype:type-a --supertype.subtype.data 1".split(" ")) + == Wrapper() + ) + + +def test_nested_in_subparser_override_with_default() -> None: + @dataclasses.dataclass(frozen=True) + class Mnist: + binary: bool = False + """Set to load binary version of MNIST dataset.""" + + @dataclasses.dataclass + class ImageNet: + subset: Literal[50, 100, 1000] + """Choose between ImageNet-50, ImageNet-100, ImageNet-1000, etc.""" + + # Possible optimizer configurations. + + Selector = tyro.extras.subcommand_type_from_defaults( + { + "m": Mnist(), + "i": ImageNet(50), + } + ) + + @dataclasses.dataclass(frozen=True) + class DatasetContainer: + dataset: Selector = Mnist() # type: ignore + + @dataclasses.dataclass + class Adam: + learning_rate: float = 1e-3 + betas: Tuple[float, float] = (0.9, 0.999) + container: DatasetContainer = DatasetContainer() + + @dataclasses.dataclass + class Sgd: + learning_rate: float = 3e-4 + container: DatasetContainer = DatasetContainer() + + # Train script. + + Optimizers = tyro.extras.subcommand_type_from_defaults( + {"adam": Adam(container=DatasetContainer(ImageNet(50))), "sgd": Sgd()}, + prefix_names=False, + ) + + @tyro.conf.configure(tyro.conf.OmitSubcommandPrefixes) + def train( + optimizer: Optimizers = Adam(container=DatasetContainer(ImageNet(50))), # type: ignore + ) -> Union[Adam, Sgd]: + return optimizer + + assert tyro.cli(train, args=[]) == Adam(container=DatasetContainer(ImageNet(50))) + assert tyro.cli(train, args=["adam"]) == Adam( + container=DatasetContainer(ImageNet(50)) + ) + assert tyro.cli(train, args=["sgd"]) == Sgd(container=DatasetContainer(Mnist())) + + +def test_underscore_prefix() -> None: + """https://github.com/brentyi/tyro/issues/77""" + + @dataclasses.dataclass + class PrivateConfig: + pass + + @dataclasses.dataclass + class BaseConfig: + _private: PrivateConfig = dataclasses.field( + default_factory=lambda: PrivateConfig() + ) + + @dataclasses.dataclass + class Level1(BaseConfig): + pass + + @dataclasses.dataclass + class Level2(BaseConfig): + child: Level1 = dataclasses.field(default_factory=lambda: Level1()) + + @dataclasses.dataclass + class Level3(BaseConfig): + child: Level2 = dataclasses.field(default_factory=lambda: Level2()) + + tyro.cli(Level3, args=[]) diff --git a/tests/test_py311_generated/test_nested_in_containers_generated.py b/tests/test_py311_generated/test_nested_in_containers_generated.py new file mode 100644 index 000000000..a3eab0e1e --- /dev/null +++ b/tests/test_py311_generated/test_nested_in_containers_generated.py @@ -0,0 +1,385 @@ +import dataclasses +import enum +from typing import Any, Dict, Generic, List, Set, Tuple, TypeVar + +import pytest + +import tyro + + +@dataclasses.dataclass(frozen=True) +class Color: + r: int + g: int + b: int + + +def test_nested_tuple_fixed_single() -> None: + def main(x: Tuple[Color]) -> Any: + return x + + assert tyro.cli(main, args="--x.0.r 255 --x.0.g 127 --x.0.b 5".split(" ")) == ( + Color(255, 127, 5), + ) + + +def test_nested_tuple_fixed_two() -> None: + def main(x: Tuple[Color, Color]) -> Any: + return x + + assert tyro.cli( + main, + args=( + "--x.0.r 255 --x.0.g 127 --x.0.b 5 --x.1.r 255 --x.1.g 127 --x.1.b 0" + ).split(" "), + ) == ( + Color(255, 127, 5), + Color(255, 127, 0), + ) + + +def test_nested_tuple_fixed_three() -> None: + def main(x: Tuple[Color, int, Color]) -> Any: + return x + + assert tyro.cli( + main, + args=( + "--x.0.r 255 --x.0.g 127 --x.0.b 5 --x.1 94709 --x.2.r 255 --x.2.g 127" + " --x.2.b 0" + ).split(" "), + ) == ( + Color(255, 127, 5), + 94709, + Color(255, 127, 0), + ) + + +def test_nested_tuple_recursive() -> None: + def main(x: Tuple[Color, Tuple[Color, Color]]) -> Any: + return x + + assert tyro.cli( + main, + args=( + "--x.0.r 255 --x.0.g 127 --x.0.b 5 --x.1.0.r 255 --x.1.0.g 127 --x.1.0.b 0" + " --x.1.1.r 255 --x.1.1.g 127 --x.1.1.b 0" + ).split(" "), + ) == ( + Color(255, 127, 5), + ( + Color(255, 127, 0), + Color(255, 127, 0), + ), + ) + + +def test_tuple_bad() -> None: + # Unable to infer input length. + def main(x: Tuple[Color, ...]) -> None: + pass + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=[]) + + +def test_set_bad() -> None: + # Unable to infer input length. + def main(x: Set[Color]) -> None: + pass + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=[]) + + +def test_set_ok() -> None: + def main(x: Set[Color] = {Color(255, 0, 0)}) -> Any: + return x + + assert tyro.cli(main, args=[]) == {Color(255, 0, 0)} + assert tyro.cli(main, args="--x.0.r 127".split(" ")) == {Color(127, 0, 0)} + + +def test_list_bad() -> None: + # Unable to infer input length. + def main(x: List[Color]) -> None: + pass + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=[]) + + +def test_list_ok() -> None: + def main(x: List[Color] = [Color(255, 0, 0)]) -> Any: + return x + + assert tyro.cli(main, args=[]) == [Color(255, 0, 0)] + assert tyro.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] + + +def test_list_object() -> None: + def main(x: List[object] = [Color(255, 0, 0)]) -> Any: + return x + + assert tyro.cli(main, args=[]) == [Color(255, 0, 0)] + assert tyro.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] + + +def test_list_any() -> None: + def main(x: List[Any] = [Color(255, 0, 0)]) -> Any: + return x + + assert tyro.cli(main, args=[]) == [Color(255, 0, 0)] + assert tyro.cli(main, args="--x.0.r 127".split(" ")) == [Color(127, 0, 0)] + + +def test_tuple_in_list() -> None: + def main(x: List[Tuple[Color]] = [(Color(255, 0, 0),)]) -> Any: + return x + + assert tyro.cli(main, args=[]) == [(Color(255, 0, 0),)] + assert tyro.cli(main, args="--x.0.0.r 127".split(" ")) == [(Color(127, 0, 0),)] + + +def test_tuple_variable() -> None: + def main(x: Tuple[Color, ...] = (Color(255, 0, 0), Color(255, 0, 127))) -> Any: + return x + + assert tyro.cli(main, args=[]) == (Color(255, 0, 0), Color(255, 0, 127)) + assert tyro.cli(main, args="--x.0.r 127".split(" ")) == ( + Color(127, 0, 0), + Color(255, 0, 127), + ) + + +def test_dict_bad() -> None: + def main(x: Dict[str, Color]) -> Any: + return x + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(main, args=[]) + + +def test_dict_ok() -> None: + def main( + x: Dict[str, Color] = { + "red": Color(255, 0, 0), + "green": Color(0, 255, 0), + "blue": Color(0, 0, 255), + } + ) -> Any: + return x + + assert tyro.cli(main, args=[])["green"] == Color(0, 255, 0) + assert tyro.cli(main, args="--x.green.g 127".split(" "))["green"] == Color( + 0, 127, 0 + ) + + +def test_dict_key_int() -> None: + def main( + x: Dict[int, Color] = { + 0: Color(255, 0, 0), + 1: Color(0, 255, 0), + 2: Color(0, 0, 255), + } + ) -> Any: + return x + + assert tyro.cli(main, args=[])[1] == Color(0, 255, 0) + assert tyro.cli(main, args="--x.1.g 127".split(" "))[1] == Color(0, 127, 0) + + +def test_dict_key_enum() -> None: + class ColorType(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + def main( + x: Dict[ColorType, Color] = { + ColorType.RED: Color(255, 0, 0), + ColorType.GREEN: Color(0, 255, 0), + ColorType.BLUE: Color(0, 0, 255), + } + ) -> Any: + return x + + assert tyro.cli(main, args=[])[ColorType.GREEN] == Color(0, 255, 0) + assert tyro.cli(main, args="--x.GREEN.g 127".split(" "))[ColorType.GREEN] == Color( + 0, 127, 0 + ) + + +def test_dict_nested() -> None: + def main( + x: Dict[str, Tuple[Color, int]] = { + # For each color: RGB and xterm color code. + "red": (Color(255, 0, 0), 9), + "green": (Color(0, 255, 0), 10), + "blue": (Color(0, 0, 255), 12), + } + ) -> Any: + return x + + assert tyro.cli(main, args=[])["green"] == (Color(0, 255, 0), 10) + assert tyro.cli(main, args="--x.green.0.g 127 --x.green.1 2".split(" "))[ + "green" + ] == (Color(0, 127, 0), 2) + + +def test_generic_in_tuple() -> None: + ScalarType = TypeVar("ScalarType", int, float) + + @dataclasses.dataclass + class GenericColor(Generic[ScalarType]): + r: ScalarType + g: ScalarType + b: ScalarType + + def main(x: Tuple[GenericColor[float], GenericColor[int]]) -> Any: + return x + + assert tyro.cli( + main, + args=( + "--x.0.r 0.5 --x.0.g 0.2 --x.0.b 0.3 --x.1.r 25 --x.1.g 2 --x.1.b 3".split( + " " + ) + ), + ) == (GenericColor(0.5, 0.2, 0.3), GenericColor(25, 2, 3)) + + +def test_generic_in_tuple_with_default() -> None: + ScalarType = TypeVar("ScalarType", int, float) + + @dataclasses.dataclass + class GenericColor(Generic[ScalarType]): + r: ScalarType + g: ScalarType + b: ScalarType + + def main( + x: Tuple[GenericColor[float], GenericColor[int]] = ( + GenericColor(0.5, 0.2, 0.3), + GenericColor[int](25, 2, 3), # The subscript should be optional. + ) + ) -> Any: + return x + + assert tyro.cli( + main, + args=( + "--x.0.r 0.5 --x.0.g 0.2 --x.0.b 0.3 --x.1.r 25 --x.1.g 2 --x.1.b 3".split( + " " + ) + ), + ) == (GenericColor(0.5, 0.2, 0.3), GenericColor(25, 2, 3)) + + +def test_generic_in_variable_tuple_with_default() -> None: + ScalarType = TypeVar("ScalarType", int, float) + + @dataclasses.dataclass + class GenericColor(Generic[ScalarType]): + r: ScalarType + g: ScalarType + b: ScalarType + + def main( + x: Tuple[GenericColor, ...] = ( + GenericColor[float](0.5, 0.2, 0.3), + GenericColor[int](25, 2, 3), + ) + ) -> Any: + return x + + assert tyro.cli( + main, + args=( + "--x.0.r 0.5 --x.0.g 0.9 --x.0.b 0.3 --x.1.r 25 --x.1.g 2 --x.1.b 3".split( + " " + ) + ), + ) == (GenericColor(0.5, 0.9, 0.3), GenericColor(25, 2, 3)) + + +def test_generic_in_dict_with_default() -> None: + ScalarType = TypeVar("ScalarType", int, float) + + @dataclasses.dataclass + class GenericColor(Generic[ScalarType]): + r: ScalarType + g: ScalarType + b: ScalarType + + def main( + x: Dict[str, GenericColor] = { + "float": GenericColor(0.5, 0.2, 0.3), + "int": GenericColor[int](25, 2, 3), + } + ) -> Any: + return x + + assert tyro.cli( + main, + args="--x.float.g 0.1".split(" "), + )[ + "float" + ] == GenericColor(0.5, 0.1, 0.3) + assert tyro.cli( + main, + args="--x.int.g 0".split(" "), + ) == {"float": GenericColor(0.5, 0.2, 0.3), "int": GenericColor(25, 0, 3)} + + +def test_generic_in_double_nested_dict_with_default() -> None: + ScalarType = TypeVar("ScalarType", int, float) + + @dataclasses.dataclass + class GenericColor(Generic[ScalarType]): + r: ScalarType + g: ScalarType + b: ScalarType + + def main( + x: Dict[str, Dict[str, GenericColor]] = { + "hello": { + "float": GenericColor(0.5, 0.2, 0.3), + "int": GenericColor[int](25, 2, 3), + } + } + ) -> Any: + return x + + assert tyro.cli( + main, + args="--x.hello.float.g 0.1".split(" "), + )["hello"][ + "float" + ] == GenericColor(0.5, 0.1, 0.3) + assert tyro.cli( + main, + args="--x.hello.int.g 0".split(" "), + ) == { + "hello": {"float": GenericColor(0.5, 0.2, 0.3), "int": GenericColor(25, 0, 3)} + } + + +def test_double_nested_dict_with_inferred_type() -> None: + def main( + x: Dict[str, Any] = { + "hello": { + "a": Color(5, 2, 3), + "b": Color(25, 2, 3), + } + } + ) -> Any: + return x + + assert tyro.cli( + main, + args="--x.hello.a.g 1".split(" "), + )["hello"][ + "a" + ] == Color(5, 1, 3) diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py new file mode 100644 index 000000000..94d0fc6f2 --- /dev/null +++ b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py @@ -0,0 +1,64 @@ +from typing import Any, Literal + +import pytest + +import tyro + + +def test_union_direct(): + assert tyro.cli(int | str, args=["5"]) == 5 + assert tyro.cli(int | str, args=["five"]) == "five" + + +def test_union_basic(): + def main(x: int | str) -> int | str: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == 6 + assert tyro.cli(main, args=["--x", "five"]) == "five" + + +def test_union_with_list(): + def main(x: int | str | list[bool]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == 6 + assert tyro.cli(main, args=["--x", "five"]) == "five" + assert tyro.cli(main, args=["--x", "True"]) == "True" + assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] + + +def test_union_literal(): + def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == "6" + assert tyro.cli(main, args=["--x", "five"]) == "five" + + +def test_super_nested(): + def main( + x: None + | list[ + tuple[ + None | int, + Literal[3, 4], + tuple[int, int] | tuple[str, str], + ] + ] = None + ) -> Any: + return x + + assert tyro.cli(main, args=[]) is None + assert tyro.cli(main, args="--x None".split(" ")) is None + assert tyro.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] + assert tyro.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] + assert tyro.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ + (2, 3, ("x", "2")), + (2, 3, (1, 2)), + ] + with pytest.raises(SystemExit): + tyro.cli(main, args=["--help"]) diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py new file mode 100644 index 000000000..5ef86858c --- /dev/null +++ b/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py @@ -0,0 +1,52 @@ +from typing import Any, Literal, Optional, Union + +import pytest + +import tyro + + +def test_list(): + def main(x: list[bool]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] + + +def test_tuple(): + def main(x: tuple[bool, str]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "True", "False"]) == (True, "False") + + +def test_tuple_variable(): + def main(x: tuple[Union[bool, str], ...]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "True", "Wrong"]) == (True, "Wrong") + + +def test_super_nested(): + def main( + x: Optional[ + list[ + tuple[ + Optional[int], + Literal[3, 4], + Union[tuple[int, int], tuple[str, str]], + ] + ] + ] = None + ) -> Any: + return x + + assert tyro.cli(main, args=[]) is None + assert tyro.cli(main, args="--x None".split(" ")) is None + assert tyro.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] + assert tyro.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] + assert tyro.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ + (2, 3, ("x", "2")), + (2, 3, (1, 2)), + ] + with pytest.raises(SystemExit): + tyro.cli(main, args=["--help"]) diff --git a/tests/test_py311_generated/test_partial_generated.py b/tests/test_py311_generated/test_partial_generated.py new file mode 100644 index 000000000..e1d63aed8 --- /dev/null +++ b/tests/test_py311_generated/test_partial_generated.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import dataclasses +import functools + +from helptext_utils import get_helptext + +import tyro + + +def test_partial_func() -> None: + def main(a: int, b: str) -> str: + return b * a + + assert tyro.cli(functools.partial(main, a=3), args=["--b", "hi"]) == "hihihi" + + +def test_partial_class() -> None: + class Main: + def __init__(self, a: int, b: str) -> None: + self.inner = b * a + + assert tyro.cli(functools.partial(Main, a=3), args=["--b", "hi"]).inner == "hihihi" + + +def test_partial_helptext_func() -> None: + def main(a: int, b: str) -> str: + """Hello!""" + return b * a + + helptext = get_helptext(functools.partial(main, b=3)) + assert "partial" not in helptext + assert "Hello!" in helptext + + +def test_partial_helptext_class() -> None: + class Main: + """Hello!""" + + def __init__(self, a: int, b: str) -> None: + self.inner = b * a + + helptext = get_helptext(functools.partial(Main, b=3)) + assert "partial" not in helptext + assert "Hello!" in helptext + + +def test_partial_helptext_dataclass() -> None: + @dataclasses.dataclass + class Config: + a: int + b: int + """Hello!""" + c: int + + ConfigWithDefaults = functools.partial(functools.partial(Config, a=3), c=5) + assert tyro.cli(ConfigWithDefaults, args=["--b", "4"]) == Config(3, 4, 5) + + helptext = get_helptext(ConfigWithDefaults) + assert "partial" not in helptext + assert "Hello!" in helptext diff --git a/tests/test_py311_generated/test_positional_min_py38_generated.py b/tests/test_py311_generated/test_positional_min_py38_generated.py new file mode 100644 index 000000000..15f8f7911 --- /dev/null +++ b/tests/test_py311_generated/test_positional_min_py38_generated.py @@ -0,0 +1,118 @@ +from typing import List, Optional, Tuple + +import pytest + +import tyro + + +def test_positional(): + def main( + x: int, + y: int, + /, + # Note: it's generally a bad idea to have a mutable object (like a list) as a + # default value. But it should still work. + z: List[int] = [1, 2, 3], + ) -> Tuple[int, int, int]: + """main. + + Args: + x: x + y: y + z: z + + Returns: + Tuple[int, int, int]: Output. + """ + return (x, y, z[0]) + + assert tyro.cli(main, args="1 2 --z 3".split(" ")) == (1, 2, 3) + with pytest.raises(SystemExit): + assert tyro.cli(main, args="--x 1 --y 2 --z 3".split(" ")) == (1, 2, 3) + + +def test_nested_positional(): + class A: + def __init__(self, a: int, hello_world: int, /, c: int): + self.hello_world = hello_world + + def nest1(a: int, b: int, thing: A, /, c: int) -> A: + return thing + + assert isinstance(tyro.cli(nest1, args="0 1 2 3 4 --c 4".split(" ")), A) + assert tyro.cli(nest1, args="0 1 2 3 4 --c 4".split(" ")).hello_world == 3 + with pytest.raises(SystemExit): + tyro.cli(nest1, args="0 1 2 3 4 4 --c 4".split(" ")) + + +def test_nested_positional_alt(): + class B: + def __init__(self, a: int, b: int, /, c: int): + pass + + def nest2(a: int, b: int, /, thing: B, c: int): + return thing + + assert isinstance(tyro.cli(nest2, args="0 1 2 3 --thing.c 4 --c 4".split(" ")), B) + with pytest.raises(SystemExit): + tyro.cli(nest2, args="0 1 2 3 4 --thing.c 4 --c 4".split(" ")) + + +def test_positional_with_underscores(): + """Hyphen replacement works a bit different for positional arguments.""" + + def main(a_multi_word_input: int, /) -> int: + return a_multi_word_input + + assert tyro.cli(main, args=["5"]) == 5 + + +def test_positional_booleans(): + """Make sure that flag behavior is disabled for positional booleans.""" + + def main( + flag1: bool, + flag2: bool = True, + flag3: bool = False, + /, + ) -> Tuple[bool, bool, bool]: + return flag1, flag2, flag3 + + assert tyro.cli(main, args=["True"]) == (True, True, False) + assert tyro.cli(main, args=["True", "False"]) == (True, False, False) + assert tyro.cli(main, args=["False", "False", "True"]) == (False, False, True) + + with pytest.raises(SystemExit): + tyro.cli(main, args=["hmm"]) + with pytest.raises(SystemExit): + tyro.cli(main, args=["true"]) + with pytest.raises(SystemExit): + tyro.cli(main, args=["True", "false"]) + + +def test_optional_list(): + def main(a: Optional[List[int]], /) -> Optional[List[int]]: + return a + + assert tyro.cli(main, args=["None"]) is None + assert tyro.cli(main, args=["1", "2"]) == [1, 2] + assert tyro.cli(main, args=[]) == [] + with pytest.raises(SystemExit): + tyro.cli(main, args=["hm"]) + + +def test_optional_list_with_default(): + def main(a: Optional[List[int]] = None, /) -> Optional[List[int]]: + return a + + assert tyro.cli(main, args=["None"]) is None + assert tyro.cli(main, args=["5", "5"]) == [5, 5] + with pytest.raises(SystemExit): + tyro.cli(main, args=["None", "5"]) + + +def test_positional_tuple(): + def main(x: Tuple[int, int], y: Tuple[str, str], /): + return x, y + + assert tyro.cli(main, args="1 2 3 4".split(" ")) == ((1, 2), ("3", "4")) diff --git a/tests/test_py311_generated/test_pydantic_generated.py b/tests/test_py311_generated/test_pydantic_generated.py new file mode 100644 index 000000000..18e2a51bf --- /dev/null +++ b/tests/test_py311_generated/test_pydantic_generated.py @@ -0,0 +1,108 @@ +import contextlib +import io +import pathlib +from typing import cast + +import pytest +from pydantic import BaseModel, Field + +import tyro + + +def test_pydantic() -> None: + class ManyTypesA(BaseModel): + i: int + s: str = "hello" + f: float = Field(default_factory=lambda: 3.0) + p: pathlib.Path + + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( + ManyTypesA, + args=[ + "--i", + "5", + "--p", + "~", + ], + ) == ManyTypesA(i=5, s="hello", f=3.0, p=pathlib.Path("~")) + + +def test_pydantic_helptext() -> None: + class Helptext(BaseModel): + """This docstring should be printed as a description.""" + + x: int = Field(description="Documentation 1") + + y: int = Field(description="Documentation 2") + + z: int = Field(description="Documentation 3") + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(Helptext, args=["--help"]) + helptext = f.getvalue() + assert tyro._strings.strip_ansi_sequences(cast(str, Helptext.__doc__)) in helptext + + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + assert "Documentation 3" in helptext + + +def test_pydantic_suppress_base_model_helptext() -> None: + class Helptext(BaseModel): + x: int = Field(description="Documentation 1") + + y: int = Field(description="Documentation 2") + + z: int = Field(description="Documentation 3") + + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(Helptext, args=["--help"]) + helptext = f.getvalue() + + assert "Create a new model by parsing and validating" not in helptext + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + assert "Documentation 3" in helptext + + +class HelptextWithFieldDocstring(BaseModel): + """This docstring should be printed as a description.""" + + x: int + """Documentation 1""" + + y: int = Field(description="Documentation 2") + + z: int = Field(description="Documentation 3") + + +def test_pydantic_field_helptext_from_docstring() -> None: + f = io.StringIO() + with pytest.raises(SystemExit): + with contextlib.redirect_stdout(f): + tyro.cli(HelptextWithFieldDocstring, args=["--help"]) + helptext = f.getvalue() + assert ( + tyro._strings.strip_ansi_sequences( + cast(str, HelptextWithFieldDocstring.__doc__) + ) + in helptext + ) + + assert "Documentation 1" in helptext + assert "Documentation 2" in helptext + assert "Documentation 3" in helptext + + +def test_pydantic_positional_annotation() -> None: + class AnnotatedAsPositional(BaseModel): + name: tyro.conf.Positional[str] + """This is annotated as a positional argument.""" + + result = tyro.cli(AnnotatedAsPositional, args=["myname"]) + assert isinstance(result, AnnotatedAsPositional) diff --git a/tests/test_py311_generated/test_strings_generated.py b/tests/test_py311_generated/test_strings_generated.py new file mode 100644 index 000000000..ff7669a75 --- /dev/null +++ b/tests/test_py311_generated/test_strings_generated.py @@ -0,0 +1,70 @@ +from tyro import _strings + + +def test_words_from_name(): + assert _strings.hyphen_separated_from_camel_case("MyHTTPServer") == "my-http-server" + assert ( + _strings.hyphen_separated_from_camel_case("my-http-server") == "my-http-server" + ) + + +def test_make_field_name(): + assert _strings.make_field_name(["hello", "world"]) == "hello.world" + assert _strings.make_field_name(["hello_world", "world"]) == "hello-world.world" + assert ( + _strings.make_field_name(["hello_world", "___hello_world"]) + == "hello-world.___hello-world" + ) + assert ( + _strings.make_field_name(["hello_world", "---hello_world"]) + == "hello-world.---hello-world" + ) + + +def test_postprocess_helptext(): + assert _strings.remove_single_line_breaks("hello world") == "hello world" + assert _strings.remove_single_line_breaks("hello\nworld") == "hello world" + assert _strings.remove_single_line_breaks("hello \nworld") == "hello world" + assert _strings.remove_single_line_breaks("hello\n\nworld") == "hello\n\nworld" + assert ( + _strings.remove_single_line_breaks( + "a paragraph:\nSentence one.\nSentence two.\nSentence three.\n" + ) + == "a paragraph: Sentence one. Sentence two. Sentence three." + ) + assert ( + _strings.remove_single_line_breaks( + "a bulleted list:\n" + "- The first problem.\n" + "- The second problem.\n" + "- The third problem.\n" + ) + == "a bulleted list:\n" + "- The first problem.\n" + "- The second problem.\n" + "- The third problem." + ) + assert ( + _strings.remove_single_line_breaks( + "an indented list:\n" + " The first problem.\n" + " The second problem.\n" + " The third problem.\n" + ) + == "an indented list:\n" + " The first problem.\n" + " The second problem.\n" + " The third problem." + ) + assert ( + _strings.remove_single_line_breaks( + "a numbered list:\n" + "1. The first problem.\n" + "2. The second problem.\n" + "3. The third problem.\n" + ) + == "a numbered list:\n" + "1. The first problem.\n" + "2. The second problem.\n" + "3. The third problem." + ) diff --git a/tests/test_py311_generated/test_union_from_mapping_generated.py b/tests/test_py311_generated/test_union_from_mapping_generated.py new file mode 100644 index 000000000..841710e78 --- /dev/null +++ b/tests/test_py311_generated/test_union_from_mapping_generated.py @@ -0,0 +1,48 @@ +import dataclasses +from typing import Optional + +import tyro + + +@dataclasses.dataclass +class A: + x: int + + +def test_union_from_mapping(): + base_configs = { + "one": A(1), + "two": A(2), + "three": A(3), + } + ConfigUnion = tyro.extras.subcommand_type_from_defaults(base_configs) + + assert tyro.cli(ConfigUnion, args="one".split(" ")) == A(1) + assert tyro.cli(ConfigUnion, args="two".split(" ")) == A(2) + assert tyro.cli(ConfigUnion, args="two --x 4".split(" ")) == A(4) + assert tyro.cli(ConfigUnion, args="three".split(" ")) == A(3) + + +def test_union_from_mapping_in_function(): + base_configs = { + "one": A(1), + "two": A(2), + "three": A(3), + } + + # Hack for mypy. Not needed for pyright. + ConfigUnion = A + ConfigUnion = tyro.extras.subcommand_type_from_defaults(base_configs) # type: ignore + + def main(config: ConfigUnion, flag: bool = False) -> Optional[A]: + if flag: + return config + return None + + assert tyro.cli(main, args="--flag config:one".split(" ")) == A(1) + assert tyro.cli(main, args="--flag config:one --config.x 3".split(" ")) == A(3) + assert tyro.cli(main, args="config:one --config.x 1".split(" ")) is None + + assert tyro.cli(main, args="--flag config:two".split(" ")) == A(2) + assert tyro.cli(main, args="--flag config:two --config.x 3".split(" ")) == A(3) + assert tyro.cli(main, args="config:two --config.x 1".split(" ")) is None diff --git a/tests/test_py311_generated/test_unsafe_cache_generated.py b/tests/test_py311_generated/test_unsafe_cache_generated.py new file mode 100644 index 000000000..35cb41741 --- /dev/null +++ b/tests/test_py311_generated/test_unsafe_cache_generated.py @@ -0,0 +1,31 @@ +from tyro import _unsafe_cache + + +def test_unsafe_cache(): + x = 0 + + @_unsafe_cache.unsafe_cache(maxsize=2) + def f(dummy: int): + nonlocal x + x += 1 + + f(0) + f(0) + f(0) + assert x == 1 + f(1) + f(1) + f(1) + assert x == 2 + f(0) + f(0) + f(0) + assert x == 2 + f(2) + f(2) + f(2) + assert x == 3 + f(0) + f(0) + f(0) + assert x == 4 diff --git a/tests/test_py311_generated/test_unsupported_but_should_work_generated.py b/tests/test_py311_generated/test_unsupported_but_should_work_generated.py new file mode 100644 index 000000000..7341d3251 --- /dev/null +++ b/tests/test_py311_generated/test_unsupported_but_should_work_generated.py @@ -0,0 +1,58 @@ +"""Tests for features that are not officially features, but should work. + +Includes things like omegaconf.MISSING, attrs, etc, which mostly work but either likely +have corner cases or just seem sketchy. +""" +from typing import Tuple + +import omegaconf +import pytest + +import tyro +import tyro._strings + + +def test_omegaconf_missing(): + """Passing in a omegaconf.MISSING default; this will mark an argument as required.""" + + def main( + required_a: int, + optional: int = 3, + required_b: int = None, # type: ignore + ) -> Tuple[int, int, int]: + return (required_a, optional, required_b) # type: ignore + + assert tyro.cli( + main, args="--required-a 3 --optional 4 --required-b 5".split(" ") + ) == (3, 4, 5) + assert tyro.cli(main, args="--required-a 3 --required-b 5".split(" ")) == ( + 3, + 3, + 5, + ) + + with pytest.raises(SystemExit): + tyro.cli(main, args="--required-a 3 --optional 4") + with pytest.raises(SystemExit): + tyro.cli(main, args="--required-a 3") + + def main2( + required_a: int, + optional: int = 3, + required_b: int = omegaconf.MISSING, + ) -> Tuple[int, int, int]: + return (required_a, optional, required_b) + + assert tyro.cli( + main2, args="--required-a 3 --optional 4 --required-b 5".split(" ") + ) == (3, 4, 5) + assert tyro.cli(main2, args="--required-a 3 --required-b 5".split(" ")) == ( + 3, + 3, + 5, + ) + + with pytest.raises(SystemExit): + tyro.cli(main2, args="--required-a 3 --optional 4") + with pytest.raises(SystemExit): + tyro.cli(main2, args="--required-a 3") diff --git a/tyro/_fields.py b/tyro/_fields.py index 3722c15ff..48f6b7718 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -30,7 +30,14 @@ import docstring_parser import typing_extensions -from typing_extensions import get_args, get_type_hints, is_typeddict +from typing_extensions import ( + NotRequired, + Required, + get_args, + get_origin, + get_type_hints, + is_typeddict, +) from . import conf # Avoid circular import. from . import ( @@ -342,19 +349,43 @@ def _field_list_from_typeddict( default_instance not in MISSING_SINGLETONS and default_instance is not EXCLUDE_FROM_CALL ) + total = getattr(cls, "__total__", True) + assert isinstance(total, bool) assert not valid_default_instance or isinstance(default_instance, dict) for name, typ in get_type_hints(cls, include_extras=True).items(): + typ_origin = get_origin(typ) if valid_default_instance: default = default_instance.get(name, MISSING_PROP) # type: ignore - elif getattr(cls, "__total__") is False: + elif typ_origin is Required and total is False: + # Support total=False. + default = MISSING_PROP + elif total is False: + # Support total=False. default = EXCLUDE_FROM_CALL if is_nested_type(typ, MISSING_NONPROP): - raise _instantiators.UnsupportedTypeAnnotationError( - "`total=False` not supported for nested structures." - ) + # total=False behavior is unideal for nested structures. + pass + # raise _instantiators.UnsupportedTypeAnnotationError( + # "`total=False` not supported for nested structures." + # ) + elif typ_origin is NotRequired: + # Support typing.NotRequired[]. + default = EXCLUDE_FROM_CALL else: default = MISSING_PROP + # Nested types need to be populated / can't be excluded from the call. + if default is EXCLUDE_FROM_CALL and is_nested_type(typ, MISSING_NONPROP): + default = MISSING_NONPROP + + if typ_origin in (Required, NotRequired): + args = get_args(typ) + assert ( + len(args) == 1 + ), "typing.Required[] and typing.NotRequired[T] require a concrete type T." + typ = args[0] + del args + field_list.append( FieldDefinition.make( name=name, @@ -561,7 +592,7 @@ def _field_list_from_tuple( # doesn't happen if len(children) == 0: if default_instance in MISSING_SINGLETONS: - raise _instantiators.UnsupportedTypeAnnotationError( + return UnsupportedNestedTypeMessage( "If contained types of a tuple are not specified in the annotation, a" " default instance must be specified." ) @@ -612,7 +643,7 @@ def _field_list_from_sequence_checked( contained_type: Any if len(get_args(f)) == 0: if default_instance in MISSING_SINGLETONS: - raise _instantiators.UnsupportedTypeAnnotationError( + return UnsupportedNestedTypeMessage( f"Sequence type {f} needs either an explicit type or a" " default to infer from." ) @@ -671,9 +702,9 @@ def _field_list_from_dict( f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - if default_instance in MISSING_SINGLETONS: + if default_instance in MISSING_SINGLETONS or len(cast(dict, default_instance)) == 0: return UnsupportedNestedTypeMessage( - "Nested dictionary structures must have a default instance specified." + "Nested dictionary structures must have non-empty default instance specified." ) field_list = [] for k, v in cast(dict, default_instance).items(): diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 1fd5e898c..8ea1afb8b 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -46,6 +46,7 @@ Iterable, List, Optional, + Sequence, Set, Tuple, TypeVar, @@ -315,6 +316,16 @@ def _instantiator_from_container_type( """Attempt to create an instantiator from a container type. Returns `None` is no container type is found.""" + # Default generic types to strings. + if typ in (dict, Dict): + typ = Dict[str, str] + elif typ in (tuple, Tuple): + typ = Tuple[str, ...] # type: ignore + elif typ in (list, List, collections.abc.Sequence, Sequence): + typ = List[str] + elif typ in (set, Set): + typ = Set[str] + type_origin = get_origin(typ) if type_origin is None: return None From 7a50caee8a29cf4b4448677e16a8303143ddc160 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 25 Oct 2023 23:40:46 -0700 Subject: [PATCH 329/491] Implement custom constructor/proxy types (#82) * Add ability to specify custom constructor * Tests, error refinement * Edge cases, more comprehensive tests, examples * Add back missing example (this commit doesn't really belong in this branch) * Add `constructor=` and `constructor_factory=` arguments to `tyro.conf.subcommand()` * Fix helptext edge case * Add mypy-compatible overload for subcommand_cli_from_dict() * Improve help, more tests * Optional subcommands are now handled automatically by `none_proxy` * More tests, improve errors * Fix test for Python 3.7 * Tweak example comments * Add test for subcommands from dict helper --- .../02_nesting/05_subcommands_func.rst | 76 +++++++ .../04_additional/02_dictionaries.rst | 19 +- .../04_additional/10_custom_constructors.rst | 72 +++++++ examples/01_basics/03_collections.py | 34 ++++ examples/02_nesting/05_subcommands_func.py | 33 +++ .../04_additional/10_custom_constructors.py | 43 ++++ tests/test_conf.py | 189 ++++++++++++++++++ tests/test_nested.py | 35 ++++ tyro/_arguments.py | 30 ++- tyro/_calling.py | 111 ++++++---- tyro/_cli.py | 5 +- tyro/_fields.py | 76 ++++--- tyro/_instantiators.py | 8 +- tyro/_parsers.py | 56 ++++-- tyro/_subcommand_matching.py | 2 +- tyro/conf/_confstruct.py | 116 ++++++++++- tyro/conf/_markers.py | 3 + tyro/extras/__init__.py | 3 + tyro/extras/_subcommand_cli_from_dict.py | 66 ++++++ 19 files changed, 869 insertions(+), 108 deletions(-) create mode 100644 docs/source/examples/02_nesting/05_subcommands_func.rst create mode 100644 docs/source/examples/04_additional/10_custom_constructors.rst create mode 100644 examples/01_basics/03_collections.py create mode 100644 examples/02_nesting/05_subcommands_func.py create mode 100644 examples/04_additional/10_custom_constructors.py create mode 100644 tyro/extras/_subcommand_cli_from_dict.py diff --git a/docs/source/examples/02_nesting/05_subcommands_func.rst b/docs/source/examples/02_nesting/05_subcommands_func.rst new file mode 100644 index 000000000..361f6af8c --- /dev/null +++ b/docs/source/examples/02_nesting/05_subcommands_func.rst @@ -0,0 +1,76 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Subcommands from Functions +========================================== + + +:func:`tyro.extras.subcommand_cli_from_dict()` provides a shorthand that generates a +subcommand CLI from a dictionary. + + + +.. code-block:: python + :linenos: + + + import tyro + + + def checkout(branch: str) -> None: + """Check out a branch.""" + print(f"{branch=}") + + + def commit(message: str, all: bool = False) -> None: + """Make a commit.""" + print(f"{message=} {all=}") + + + if __name__ == "__main__": + tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + } + ) + +------------ + +.. raw:: html + + python 02_nesting/05_subcommands_func.py --help + +.. program-output:: python ../../examples/02_nesting/05_subcommands_func.py --help + +------------ + +.. raw:: html + + python 02_nesting/05_subcommands_func.py commit --help + +.. program-output:: python ../../examples/02_nesting/05_subcommands_func.py commit --help + +------------ + +.. raw:: html + + python 02_nesting/05_subcommands_func.py commit --message hello --all + +.. program-output:: python ../../examples/02_nesting/05_subcommands_func.py commit --message hello --all + +------------ + +.. raw:: html + + python 02_nesting/05_subcommands_func.py checkout --help + +.. program-output:: python ../../examples/02_nesting/05_subcommands_func.py checkout --help + +------------ + +.. raw:: html + + python 02_nesting/05_subcommands_func.py checkout --branch main + +.. program-output:: python ../../examples/02_nesting/05_subcommands_func.py checkout --branch main diff --git a/docs/source/examples/04_additional/02_dictionaries.rst b/docs/source/examples/04_additional/02_dictionaries.rst index a9cbee7f2..d8f4dd51c 100644 --- a/docs/source/examples/04_additional/02_dictionaries.rst +++ b/docs/source/examples/04_additional/02_dictionaries.rst @@ -16,10 +16,12 @@ Dictionary inputs can be specified using either a standard ``Dict[K, V]`` annota from typing import Dict, Tuple, TypedDict + from typing_extensions import NotRequired + import tyro - class DictionarySchema( + class DictionarySchemaA( TypedDict, # Setting `total=False` specifies that not all keys need to exist. total=False, @@ -28,17 +30,26 @@ Dictionary inputs can be specified using either a standard ``Dict[K, V]`` annota betas: Tuple[float, float] + class DictionarySchemaB(TypedDict): + learning_rate: NotRequired[float] + """NotRequired[] specifies that a particular key doesn't need to exist.""" + betas: Tuple[float, float] + + def main( - typed_dict: DictionarySchema, + typed_dict_a: DictionarySchemaA, + typed_dict_b: DictionarySchemaB, standard_dict: Dict[str, float] = { "learning_rate": 3e-4, "beta1": 0.9, "beta2": 0.999, }, ) -> None: - assert isinstance(typed_dict, dict) + assert isinstance(typed_dict_a, dict) + assert isinstance(typed_dict_b, dict) assert isinstance(standard_dict, dict) - print("Typed dict:", typed_dict) + print("Typed dict A:", typed_dict_a) + print("Typed dict B:", typed_dict_b) print("Standard dict:", standard_dict) diff --git a/docs/source/examples/04_additional/10_custom_constructors.rst b/docs/source/examples/04_additional/10_custom_constructors.rst new file mode 100644 index 000000000..ed2b08b8a --- /dev/null +++ b/docs/source/examples/04_additional/10_custom_constructors.rst @@ -0,0 +1,72 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Custom Constructors +========================================== + + +For additional flexibility, :func:`tyro.conf.arg()` accepts a ``constructor`` argument, +which makes it easier to load complex objects. + + + +.. code-block:: python + :linenos: + + + import dataclasses + import json as json_ + from typing import Dict + + from typing_extensions import Annotated + + import tyro + + + def dict_json_constructor(json: str) -> dict: + """Construct a dictionary from a JSON string. Raises a ValueError if the result is + not a dictionary.""" + out = json_.loads(json) + if not isinstance(out, dict): + raise ValueError(f"{json} is not a dictionary!") + return out + + + # A dictionary type, but `tyro` will expect a JSON string from the CLI. + JsonDict = Annotated[dict, tyro.conf.arg(constructor=dict_json_constructor)] + + + def main( + dict1: JsonDict, + dict2: JsonDict = {"default": None}, + ) -> None: + print(f"{dict1=}") + print(f"{dict2=}") + + + if __name__ == "__main__": + tyro.cli(main) + +------------ + +.. raw:: html + + python 04_additional/10_custom_constructors.py --help + +.. program-output:: python ../../examples/04_additional/10_custom_constructors.py --help + +------------ + +.. raw:: html + + python 04_additional/10_custom_constructors.py --dict1.json '{"hello": "world"}' + +.. program-output:: python ../../examples/04_additional/10_custom_constructors.py --dict1.json '{"hello": "world"}' + +------------ + +.. raw:: html + + python 04_additional/10_custom_constructors.py --dict1.json '{"hello": "world"}`' --dict2.json '{"hello": "world"}' + +.. program-output:: python ../../examples/04_additional/10_custom_constructors.py --dict1.json '{"hello": "world"}`' --dict2.json '{"hello": "world"}' diff --git a/examples/01_basics/03_collections.py b/examples/01_basics/03_collections.py new file mode 100644 index 000000000..ea4c0d108 --- /dev/null +++ b/examples/01_basics/03_collections.py @@ -0,0 +1,34 @@ +"""Multi-value Arguments + +Arguments of both fixed and variable lengths can be annotated with standard Python +collection types: `typing.List[T]`, `typing.Tuple[T1, T2]`, etc. In Python >=3.9, +`list[T]` and `tuple[T]` are also supported. + +Usage: +`python ./03_collections.py --help` +`python ./03_collections.py --dataset-sources ./data --image-dimensions 16 16` +`python ./03_collections.py --dataset-sources ./data` +""" + +import dataclasses +import pathlib +from typing import Tuple + +import tyro + + +@dataclasses.dataclass(frozen=True) +class TrainConfig: + # Example of a variable-length tuple. `typing.List`, `typing.Sequence`, + # `typing.Set`, `typing.Dict`, etc are all supported as well. + dataset_sources: Tuple[pathlib.Path, ...] + """Paths to load training data from. This can be multiple!""" + + # Fixed-length tuples are also okay. + image_dimensions: Tuple[int, int] = (32, 32) + """Height and width of some image data.""" + + +if __name__ == "__main__": + config = tyro.cli(TrainConfig) + print(config) diff --git a/examples/02_nesting/05_subcommands_func.py b/examples/02_nesting/05_subcommands_func.py new file mode 100644 index 000000000..29e1e14f8 --- /dev/null +++ b/examples/02_nesting/05_subcommands_func.py @@ -0,0 +1,33 @@ +"""Subcommands from Functions + +:func:`tyro.extras.subcommand_cli_from_dict()` provides a shorthand that generates a +subcommand CLI from a dictionary. + +Usage: +`python ./05_subcommands_func.py --help` +`python ./05_subcommands_func.py commit --help` +`python ./05_subcommands_func.py commit --message hello --all` +`python ./05_subcommands_func.py checkout --help` +`python ./05_subcommands_func.py checkout --branch main` +""" + +import tyro + + +def checkout(branch: str) -> None: + """Check out a branch.""" + print(f"{branch=}") + + +def commit(message: str, all: bool = False) -> None: + """Make a commit.""" + print(f"{message=} {all=}") + + +if __name__ == "__main__": + tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + } + ) diff --git a/examples/04_additional/10_custom_constructors.py b/examples/04_additional/10_custom_constructors.py new file mode 100644 index 000000000..2d6b9d154 --- /dev/null +++ b/examples/04_additional/10_custom_constructors.py @@ -0,0 +1,43 @@ +"""Custom Constructors + +For additional flexibility, :func:`tyro.conf.arg()` accepts a `constructor` argument, +which makes it easier to load complex objects. + +Usage: +`python ./10_custom_constructors.py --help` +`python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}"` +`python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}"` --dict2.json "{\"hello\": \"world\"}"` +""" + +import dataclasses +import json as json_ +from typing import Dict + +from typing_extensions import Annotated + +import tyro + + +def dict_json_constructor(json: str) -> dict: + """Construct a dictionary from a JSON string. Raises a ValueError if the result is + not a dictionary.""" + out = json_.loads(json) + if not isinstance(out, dict): + raise ValueError(f"{json} is not a dictionary!") + return out + + +# A dictionary type, but `tyro` will expect a JSON string from the CLI. +JsonDict = Annotated[dict, tyro.conf.arg(constructor=dict_json_constructor)] + + +def main( + dict1: JsonDict, + dict2: JsonDict = {"default": None}, +) -> None: + print(f"{dict1=}") + print(f"{dict2=}") + + +if __name__ == "__main__": + tyro.cli(main) diff --git a/tests/test_conf.py b/tests/test_conf.py index 0bf4c7ad4..96a6f5f84 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1,5 +1,9 @@ import argparse +import contextlib import dataclasses +import io +import json as json_ +import shlex from typing import Any, Dict, Generic, List, Tuple, TypeVar, Union import pytest @@ -901,3 +905,188 @@ class TrainConfig: assert tyro.cli( tyro.conf.OmitArgPrefixes[TrainConfig], args="--num-slots 3".split(" ") ) == TrainConfig(ModelConfig(num_slots=3)) + + +def test_custom_constructor_0() -> None: + def times_two(n: str) -> int: + return int(n) * 2 + + @dataclasses.dataclass + class Config: + x: Annotated[int, tyro.conf.arg(constructor=times_two)] + + assert tyro.cli(Config, args="--x.n 5".split(" ")) == Config(x=10) + + +def test_custom_constructor_1() -> None: + def times_two(n: int) -> int: + return int(n) * 2 + + @dataclasses.dataclass + class Config: + x: Annotated[int, tyro.conf.arg(constructor=times_two)] + + assert tyro.cli(Config, args="--x.n 5".split(" ")) == Config(x=10) + + +def test_custom_constructor_2() -> None: + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=int)] + + assert tyro.cli(Config, args="--x 5".split(" ")) == Config(x=5) + with pytest.raises(SystemExit): + tyro.cli(Config, args="--x 5.23".split(" ")) + + +def test_custom_constructor_3() -> None: + def dict_from_json(json: str) -> dict: + out = json_.loads(json) + if not isinstance(out, dict): + raise ValueError(f"{json} is not a dict!") + return out + + @dataclasses.dataclass + class Config: + x: Annotated[ + dict, + tyro.conf.arg( + metavar="JSON", + constructor=dict_from_json, + ), + ] + + assert tyro.cli( + Config, args=shlex.split('--x.json \'{"hello": "world"}\'') + ) == Config(x={"hello": "world"}) + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Config, args="--x.json 5".split(" ")) + + error = target.getvalue() + assert "Error parsing x: 5 is not a dict!" in error + + +def test_custom_constructor_4() -> None: + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=int)] = 3.23 + + assert tyro.cli(Config, args="--x 5".split(" ")) == Config(x=5) + assert tyro.cli(Config, args=[]) == Config(x=3.23) + + +def test_custom_constructor_5() -> None: + def make_float(a: float, b: float, c: float = 3) -> float: + return a * b * c + + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=make_float)] = 3.23 + + assert tyro.cli(Config, args=[]) == Config(x=3.23) + assert tyro.cli(Config, args="--x.a 5 --x.b 2 --x.c 3".split(" ")) == Config(x=30) + assert tyro.cli(Config, args="--x.a 5 --x.b 2".split(" ")) == Config(x=30) + + # --x.b is required! + with pytest.raises(SystemExit): + tyro.cli(Config, args="--x.a 5".split(" ")) + + # --x.a and --x.b are required! + with pytest.raises(SystemExit): + tyro.cli(Config, args="--x.c 5".split(" ")) + + +def test_custom_constructor_6() -> None: + def make_float(a: tyro.conf.Positional[float], b: float, c: float = 3) -> float: + return a * b * c + + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=make_float)] = 3.23 + + assert tyro.cli(Config, args=[]) == Config(x=3.23) + assert tyro.cli(Config, args="--x.b 2 --x.c 3 5".split(" ")) == Config(x=30) + assert tyro.cli(Config, args="--x.b 2 5".split(" ")) == Config(x=30) + + # --x.b is required! + with pytest.raises(SystemExit): + tyro.cli(Config, args="5".split(" ")) + + # --x.a and --x.b are required! + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Config, args="--x.c 5".split(" ")) + error = target.getvalue() + assert "We're missing" in error + + +def test_custom_constructor_7() -> None: + @dataclasses.dataclass + class Struct: + a: int + b: int + c: int = 3 + + def make_float(struct: Struct) -> float: + return struct.a * struct.b * struct.c + + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=make_float)] = 3.23 + + assert tyro.cli(Config, args=[]) == Config(x=3.23) + assert tyro.cli( + Config, args="--x.struct.a 5 --x.struct.b 2 --x.struct.c 3".split(" ") + ) == Config(x=30) + assert tyro.cli(Config, args="--x.struct.a 5 --x.struct.b 2".split(" ")) == Config( + x=30 + ) + + # --x.struct.b is required! + with pytest.raises(SystemExit): + tyro.cli(Config, args="--x.struct.a 5".split(" ")) + + # --x.struct.a and --x.struct.b are required! + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Config, args="--x.struct.c 5".split(" ")) + error = target.getvalue() + assert "We're missing arguments" in error + assert "'b'" in error + assert "'a'" in error # The 5 is parsed into `a`. + + +def test_custom_constructor_8() -> None: + @dataclasses.dataclass + class Struct: + a: tyro.conf.Positional[int] + b: int + c: int = 3 + + def make_float(struct: Struct) -> float: + return struct.a * struct.b * struct.c + + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=make_float)] = 3.23 + + assert tyro.cli(Config, args=[]) == Config(x=3.23) + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 5".split(" ") + ) == Config(x=30) + assert tyro.cli(Config, args="--x.struct.b 2 5".split(" ")) == Config(x=30) + + # --x.struct.b is required! + with pytest.raises(SystemExit): + tyro.cli(Config, args="5".split(" ")) + + # --x.struct.a and --x.struct.b are required! + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Config, args="--x.struct.b 5".split(" ")) + error = target.getvalue() + assert "We're missing arguments" in error + assert "'a'" in error + assert "'b'" not in error diff --git a/tests/test_nested.py b/tests/test_nested.py index 5739e7e5b..591a1c344 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -1017,3 +1017,38 @@ class Level3(BaseConfig): child: Level2 = dataclasses.field(default_factory=lambda: Level2()) tyro.cli(Level3, args=[]) + + +def test_subcommand_dict_helper() -> None: + def checkout(branch: str) -> str: + """Check out a branch.""" + return branch + + def commit(message: str, all: bool = False) -> Tuple[str, bool]: + """Make a commit.""" + return (message, all) + + assert ( + tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + }, + args="checkout --branch main".split(" "), + ) + == "main" + ) + assert tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + }, + args="commit --message hello".split(" "), + ) == ("hello", False) + assert tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + }, + args="commit --message hello --all".split(" "), + ) == ("hello", True) diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 8854edb54..79924af05 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -164,9 +164,9 @@ def add_argument( ) complete_as_path = ( # Catch types like Path, List[Path], Tuple[Path, ...] etc. - "Path" in str(self.field.typ) + "Path" in str(self.field.type_or_callable) # For string types, we require more evidence. - or ("str" in str(self.field.typ) and name_suggests_path) + or ("str" in str(self.field.type_or_callable) and name_suggests_path) ) if complete_as_path: arg.complete = shtab.DIRECTORY if name_suggests_dir else shtab.FILE # type: ignore @@ -231,7 +231,10 @@ def _rule_handle_defaults( """Set `required=True` if a default value is set.""" # Mark lowered as required if a default is set. - if arg.field.default in _fields.MISSING_SINGLETONS: + if ( + arg.field.default in _fields.MISSING_SINGLETONS + and _markers._OPTIONAL_GROUP not in arg.field.markers + ): return dataclasses.replace(lowered, default=None, required=True) return dataclasses.replace(lowered, default=arg.field.default) @@ -241,7 +244,7 @@ def _rule_handle_boolean_flags( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: - if _resolver.apply_type_from_typevar(arg.field.typ, arg.type_from_typevar) is not bool: # type: ignore + if _resolver.apply_type_from_typevar(arg.field.type_or_callable, arg.type_from_typevar) is not bool: # type: ignore return lowered if ( @@ -290,7 +293,7 @@ def _rule_recursive_instantiator_from_type( return lowered try: instantiator, metadata = _instantiators.instantiator_from_type( - arg.field.typ, + arg.field.type_or_callable, arg.type_from_typevar, arg.field.markers, ) @@ -432,6 +435,23 @@ def _rule_generate_helptext( default_text = f"(repeatable, appends: {' '.join(default_parts)})" elif arg.field.default is _fields.EXCLUDE_FROM_CALL: default_text = "(unset by default)" + elif ( + _markers._OPTIONAL_GROUP in arg.field.markers + and default in _fields.MISSING_SINGLETONS + ): + # Argument in an optional group, but with no default. This is typically used + # when general (non-argument, non-dataclass) object arguments are given a + # default, or when we use `tyro.conf.arg(constructor=...)`. + # + # There are some usage details that aren't communicated right now in the + # helptext. For example: all arguments within an optional group without a + # default should be passed in or none at all. + default_text = "(optional)" + elif _markers._OPTIONAL_GROUP in arg.field.markers: + # Argument in an optional group, but which also have a default. + assert default is not None # Just for type checker. + default_parts = map(shlex.quote, map(str, default)) + default_text = f"(default if used: {' '.join(default_parts)})" elif lowered.nargs is not None and hasattr(default, "__iter__"): # For tuple types, we might have default as (0, 1, 2, 3). # For list types, we might have default as [0, 1, 2, 3]. diff --git a/tyro/_calling.py b/tyro/_calling.py index f8daa752f..5fd3a88f4 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -4,6 +4,7 @@ from __future__ import annotations import dataclasses +import itertools from typing import Any, Callable, Dict, List, Sequence, Set, Tuple, TypeVar, Union from typing_extensions import get_args @@ -18,7 +19,7 @@ class InstantiationError(Exception): the CLI are invalid.""" message: str - arg: _arguments.ArgumentDefinition + arg: Union[_arguments.ArgumentDefinition, str] T = TypeVar("T") @@ -58,12 +59,15 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: _strings.make_field_name([arg.dest_prefix, arg.field.name]) ] = arg + optional_group = any([_markers._OPTIONAL_GROUP in f.markers for f in field_list]) + any_arguments_provided = False + for field in field_list: value: Any prefixed_field_name = _strings.make_field_name([field_name_prefix, field.name]) # Resolve field type. - field_type = field.typ + field_type = field.type_or_callable if prefixed_field_name in arg_from_prefixed_field_name: assert prefixed_field_name not in consumed_keywords @@ -84,10 +88,14 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: # If we run this script with no arguments, we should interpret this # as empty input for x. But the argparse default will be a MISSING # value, and the field default will be inspect.Parameter.empty. - if value in _fields.MISSING_SINGLETONS: - assert field.is_positional() and arg.lowered.nargs in ("?", "*") + if ( + value in _fields.MISSING_SINGLETONS + and field.is_positional_call() + and arg.lowered.nargs in ("?", "*") + ): value = [] else: + any_arguments_provided = True if arg.lowered.nargs == "?": # Special case for optional positional arguments: this is the # only time that arguments don't come back as a list. @@ -150,35 +158,21 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: ) value = subparser_def.default_instance else: - options = map( - lambda x: _resolver.apply_type_from_typevar(x, type_from_typevar), - get_args(_resolver.unwrap_annotated(field_type)[0]), + chosen_f = subparser_def.options[ + list(subparser_def.parser_from_name.keys()).index(subparser_name) + ] + value, consumed_keywords_child = call_from_args( + chosen_f, + subparser_def.parser_from_name[subparser_name], + ( + field.default + if type(field.default) is chosen_f + else _fields.MISSING_NONPROP + ), + value_from_prefixed_field_name, + field_name_prefix=prefixed_field_name, ) - chosen_f = None - for option in options: - if ( - _strings.subparser_name_from_type(prefixed_field_name, option) - == subparser_name - ): - chosen_f = option - break - assert chosen_f is not None - - if chosen_f is type(None): - value = None - else: - value, consumed_keywords_child = call_from_args( - chosen_f, - subparser_def.parser_from_name[subparser_name], - ( - field.default - if type(field.default) is chosen_f - else _fields.MISSING_NONPROP - ), - value_from_prefixed_field_name, - field_name_prefix=prefixed_field_name, - ) - consumed_keywords |= consumed_keywords_child + consumed_keywords |= consumed_keywords_child if value is _fields.EXCLUDE_FROM_CALL: continue @@ -194,6 +188,28 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: else: kwargs[field.call_argname] = value + # Logic for _markers._OPTIONAL_GROUP. + is_missing_list = [ + v in _fields.MISSING_SINGLETONS + for v in itertools.chain(positional_args, kwargs.values()) + ] + if any(is_missing_list): + if not any_arguments_provided: + # No arguments were provided in this group. + return default_instance, consumed_keywords # type: ignore + + message = "either all arguments must be provided or none of them." + if len(kwargs) > 0: + missing_kwargs = [ + k for k, v in kwargs.items() if v in _fields.MISSING_SINGLETONS + ] + if len(missing_kwargs): + message += f" We're missing arguments {missing_kwargs}." + raise InstantiationError( + message, + field_name_prefix, + ) + # Note: we unwrap types both before and after narrowing. This is because narrowing # sometimes produces types like `Tuple[T1, T2, ...]`, where we actually want just # `tuple`. @@ -203,14 +219,23 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: unwrapped_f = _resolver.unwrap_origin_strip_extras(unwrapped_f) unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore - if unwrapped_f in (tuple, list, set): - assert len(positional_args) == 0 - # When tuples are used as nested structures (eg Tuple[SomeDataclass]), we - # use keyword arguments. - assert len(positional_args) == 0 - return unwrapped_f(kwargs.values()), consumed_keywords # type: ignore - elif unwrapped_f is dict: - assert len(positional_args) == 0 - return kwargs, consumed_keywords # type: ignore - else: - return unwrapped_f(*positional_args, **kwargs), consumed_keywords # type: ignore + try: + if unwrapped_f in (tuple, list, set): + assert len(positional_args) == 0 + # When tuples are used as nested structures (eg Tuple[SomeDataclass]), we + # use keyword arguments. + assert len(positional_args) == 0 + return unwrapped_f(kwargs.values()), consumed_keywords # type: ignore + elif unwrapped_f is dict: + assert len(positional_args) == 0 + return kwargs, consumed_keywords # type: ignore + else: + return unwrapped_f(*positional_args, **kwargs), consumed_keywords # type: ignore + + # If unwrapped_f raises a ValueError, wrap the message with a more informative + # InstantiationError if possible. + except ValueError as e: + raise InstantiationError( + e.args[0], + field_name_prefix, + ) diff --git a/tyro/_cli.py b/tyro/_cli.py index 7fe1fd241..14656248c 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -470,12 +470,13 @@ def _cli_impl( Panel( Group( "[bright_red][bold]Error parsing" - f" {e.arg.lowered.name_or_flag}[/bold]:[/bright_red] {e.message}", + f" {e.arg.lowered.name_or_flag if isinstance(e.arg, _arguments.ArgumentDefinition) else e.arg}[/bold]:[/bright_red] {e.message}", *cast( # Cast to appease mypy... List[RenderableType], ( [] - if e.arg.lowered.help is None + if not isinstance(e.arg, _arguments.ArgumentDefinition) + or e.arg.lowered.help is None else [ Rule(style=Style(color="red")), "Argument helptext:", diff --git a/tyro/_fields.py b/tyro/_fields.py index 48f6b7718..295bcb952 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -55,7 +55,7 @@ @dataclasses.dataclass(frozen=True) class FieldDefinition: name: str - typ: TypeForm[Any] + type_or_callable: Union[TypeForm[Any], Callable] default: Any helptext: Optional[str] markers: FrozenSet[_markers._Marker] @@ -77,7 +77,7 @@ def __post_init__(self): @staticmethod def make( name: str, - typ: TypeForm[Any], + type_or_callable: TypeForm[Any], default: Any, helptext: Optional[str], call_argname_override: Optional[Any] = None, @@ -85,18 +85,24 @@ def make( markers: Tuple[_markers._Marker, ...] = (), ): # Try to extract argconf overrides from type. - _, argconfs = _resolver.unwrap_annotated(typ, _confstruct._ArgConfiguration) + _, argconfs = _resolver.unwrap_annotated( + type_or_callable, _confstruct._ArgConfiguration + ) if len(argconfs) == 0: - argconf = _confstruct._ArgConfiguration(None, None, None, True) + argconf = _confstruct._ArgConfiguration(None, None, None, True, None) else: assert len(argconfs) == 1 (argconf,) = argconfs helptext = argconf.help - typ, inferred_markers = _resolver.unwrap_annotated(typ, _markers._Marker) + type_or_callable, inferred_markers = _resolver.unwrap_annotated( + type_or_callable, _markers._Marker + ) return FieldDefinition( name if argconf.name is None else argconf.name, - typ, + type_or_callable + if argconf.constructor_factory is None + else argconf.constructor_factory(), default, helptext, frozenset(inferred_markers).union(markers), @@ -104,7 +110,7 @@ def make( call_argname_override if call_argname_override is not None else name, ) - def add_markers(self, markers: Tuple[_markers._Marker, ...]) -> FieldDefinition: + def add_markers(self, markers: Tuple[Any, ...]) -> FieldDefinition: return dataclasses.replace( self, markers=self.markers.union(markers), @@ -146,6 +152,10 @@ class ExcludeFromCallType(_singleton.Singleton): pass +class NotRequiredButWeDontKnowTheValueType(_singleton.Singleton): + pass + + # We have two types of missing sentinels: a propagating missing value, which when set as # a default will set all child values of nested structures as missing as well, and a # nonpropagating missing sentinel, which does not override child defaults. @@ -153,6 +163,9 @@ class ExcludeFromCallType(_singleton.Singleton): MISSING_NONPROP = NonpropagatingMissingType() # When total=False in a TypedDict, we exclude fields from the constructor by default. +NOT_REQUIRED_BUT_WE_DONT_KNOW_THE_VALUE = NotRequiredButWeDontKnowTheValueType() + + EXCLUDE_FROM_CALL = ExcludeFromCallType() # Note that our "public" missing API will always be the propagating missing sentinel. @@ -184,7 +197,9 @@ class UnsupportedNestedTypeMessage: @_unsafe_cache.unsafe_cache(maxsize=1024) -def is_nested_type(typ: TypeForm[Any], default_instance: DefaultInstance) -> bool: +def is_nested_type( + typ: Union[TypeForm[Any], Callable], default_instance: DefaultInstance +) -> bool: """Determine whether a type should be treated as a 'nested type', where a single type can be broken down into multiple fields (eg for nested dataclasses or classes). @@ -227,12 +242,12 @@ def field_list_from_callable( # Try to resolve types in our list of fields. def resolve(field: FieldDefinition) -> FieldDefinition: - typ = field.typ + typ = field.type_or_callable typ = _resolver.apply_type_from_typevar(typ, type_from_typevar) typ = _resolver.type_from_typevar_constraints(typ) typ = _resolver.narrow_container_types(typ, field.default) typ = _resolver.narrow_union_type(typ, field.default) - field = dataclasses.replace(field, typ=typ) + field = dataclasses.replace(field, type_or_callable=typ) return field field_list = list(map(resolve, field_list)) @@ -389,7 +404,7 @@ def _field_list_from_typeddict( field_list.append( FieldDefinition.make( name=name, - typ=typ, + type_or_callable=typ, default=default, helptext=_docstrings.get_field_docstring(cls, name), ) @@ -420,7 +435,7 @@ def _field_list_from_namedtuple( field_list.append( FieldDefinition.make( name=name, - typ=typ, + type_or_callable=typ, default=default, helptext=_docstrings.get_field_docstring(cls, name), ) @@ -468,7 +483,7 @@ def _field_list_from_dataclass( field_list.append( FieldDefinition.make( name=dc_field.name, - typ=dc_field.type, + type_or_callable=dc_field.type, default=default, helptext=helptext, ) @@ -506,7 +521,7 @@ def _field_list_from_pydantic( field_list.append( FieldDefinition.make( name=pd_field.name, - typ=pd_field.outer_type_, + type_or_callable=pd_field.outer_type_, default=( MISSING_NONPROP if pd_field.required else pd_field.get_default() ), @@ -523,7 +538,7 @@ def _field_list_from_pydantic( field_list.append( FieldDefinition.make( name=name, - typ=pd_field.annotation, + type_or_callable=pd_field.annotation, markers=tuple( meta for meta in pd_field.metadata @@ -569,7 +584,7 @@ def _field_list_from_attrs( field_list.append( FieldDefinition.make( name=attr_field.name, - typ=attr_field.type, + type_or_callable=attr_field.type, default=default, helptext=_docstrings.get_field_docstring(cls, attr_field.name), ) @@ -581,7 +596,7 @@ def _field_list_from_tuple( f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Fixed-length tuples. - field_list = [] + field_list: List[FieldDefinition] = [] children = get_args(f) if Ellipsis in children: return _try_field_list_from_sequence_inner( @@ -615,7 +630,7 @@ def _field_list_from_tuple( # argument, but in practice the brackets are annoying because they # require escaping. name=str(i), - typ=child, + type_or_callable=child, default=default_i, helptext="", # This should really set the positional marker, but the CLI is more @@ -626,7 +641,7 @@ def _field_list_from_tuple( contains_nested = False for field in field_list: - contains_nested |= is_nested_type(field.typ, field.default) + contains_nested |= is_nested_type(field.type_or_callable, field.default) if not contains_nested: # We could also check for variable length children, which can be populated when # the tuple is interpreted as a nested field but not a directly parsed one. @@ -690,7 +705,7 @@ def _try_field_list_from_sequence_inner( field_list.append( FieldDefinition.make( name=str(i), - typ=contained_type, + type_or_callable=contained_type, default=default_i, helptext="", ) @@ -711,7 +726,7 @@ def _field_list_from_dict( field_list.append( FieldDefinition.make( name=str(k) if not isinstance(k, enum.Enum) else k.name, - typ=type(v), + type_or_callable=type(v), default=v, helptext=None, # Dictionary specific key: @@ -726,13 +741,6 @@ def _try_field_list_from_general_callable( cls: Optional[TypeForm[Any]], default_instance: DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - # Handle general callables. - if default_instance not in MISSING_SINGLETONS: - return UnsupportedNestedTypeMessage( - "`default_instance` is supported only for select types:" - " dataclasses, lists, NamedTuple, TypedDict, etc." - ) - # Generate field list from function signature. if not callable(f): return UnsupportedNestedTypeMessage( @@ -744,11 +752,15 @@ def _try_field_list_from_general_callable( params = params[1:] out = _field_list_from_params(f, cls, params) - if not isinstance(out, UnsupportedNestedTypeMessage): + if isinstance(out, UnsupportedNestedTypeMessage): + # Return error message. return out - # Return error message. - assert isinstance(out, UnsupportedNestedTypeMessage) + # If a default is provided: . + if default_instance not in MISSING_SINGLETONS: + for i, field in enumerate(out): + out[i] = field.add_markers((_markers._OPTIONAL_GROUP,)) + return out @@ -836,7 +848,7 @@ def _field_list_from_params( FieldDefinition.make( name=param.name, # Note that param.annotation doesn't resolve forward references. - typ=typ, + type_or_callable=typ, default=default, helptext=helptext, markers=markers, diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 8ea1afb8b..2525b098e 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -149,7 +149,7 @@ def is_type_string_converter(typ: Union[Callable, TypeForm[Any]]) -> bool: def instantiator_from_type( - typ: TypeForm, + typ: Union[TypeForm[Any], Callable], type_from_typevar: Dict[TypeVar, TypeForm[Any]], markers: FrozenSet[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: @@ -191,7 +191,9 @@ def instantiator(strings: List[str]) -> None: # Address container types. If a matching container is found, this will recursively # call instantiator_from_type(). - container_out = _instantiator_from_container_type(typ, type_from_typevar, markers) + container_out = _instantiator_from_container_type( + cast(TypeForm[Any], typ), type_from_typevar, markers + ) if container_out is not None: return container_out @@ -309,7 +311,7 @@ def _instantiator_from_type_inner( def _instantiator_from_container_type( - typ: TypeForm, + typ: TypeForm[Any], type_from_typevar: Dict[TypeVar, TypeForm[Any]], markers: FrozenSet[_markers.Marker], ) -> Optional[Tuple[Instantiator, InstantiatorMetadata]]: diff --git a/tyro/_parsers.py b/tyro/_parsers.py index a8cb131ba..4344338cb 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -97,7 +97,7 @@ def from_callable_or_type( has_required_args = False args = [] - helptext_from_nested_class_field_name = {} + helptext_from_nested_class_field_name: Dict[str, Optional[str]] = {} subparsers = None subparsers_from_prefix = {} @@ -143,15 +143,31 @@ def from_callable_or_type( _strings.make_field_name([field.name, k]) ] = v + class_field_name = _strings.make_field_name([field.name]) if field.helptext is not None: helptext_from_nested_class_field_name[ - _strings.make_field_name([field.name]) + class_field_name ] = field.helptext else: helptext_from_nested_class_field_name[ - _strings.make_field_name([field.name]) + class_field_name ] = _docstrings.get_callable_description(nested_parser.f) + # If arguments are in an optional group, it indicates that the default_instance + # will be used if none of the arguments are passed in. + if ( + len(nested_parser.args) >= 1 + and _markers._OPTIONAL_GROUP in nested_parser.args[0].field.markers + ): + current_helptext = helptext_from_nested_class_field_name[ + class_field_name + ] + helptext_from_nested_class_field_name[class_field_name] = ( + ("" if current_helptext is None else current_helptext + "\n\n") + + "Default: " + + str(field.default) + ) + return ParserSpecification( f=f, description=_strings.remove_single_line_breaks( @@ -269,9 +285,9 @@ def handle_field( ]: """Determine what to do with a single field definition.""" - if isinstance(field.typ, TypeVar): + if isinstance(field.type_or_callable, TypeVar): raise _instantiators.UnsupportedTypeAnnotationError( - f"Field {field.name} has an unbound TypeVar: {field.typ}." + f"Field {field.name} has an unbound TypeVar: {field.type_or_callable}." ) if _markers.Fixed not in field.markers: @@ -288,26 +304,26 @@ def handle_field( and _markers.AvoidSubcommands in field.markers ): # Don't make a subparser. - field = dataclasses.replace(field, typ=type(field.default)) + field = dataclasses.replace(field, type_or_callable=type(field.default)) else: return subparsers_attempt # (2) Handle nested callables. - if _fields.is_nested_type(field.typ, field.default): + if _fields.is_nested_type(field.type_or_callable, field.default): field = dataclasses.replace( field, - typ=_resolver.narrow_type( - field.typ, + type_or_callable=_resolver.narrow_type( + field.type_or_callable, field.default, ), ) return ParserSpecification.from_callable_or_type( ( # Recursively apply marker types. - field.typ + field.type_or_callable if len(field.markers) == 0 else Annotated.__class_getitem__( # type: ignore - (field.typ,) + tuple(field.markers) + (field.type_or_callable,) + tuple(field.markers) ) ), description=None, @@ -337,6 +353,7 @@ class SubparsersSpecification: prefix: str required: bool default_instance: Any + options: Tuple[Union[TypeForm[Any], Callable], ...] @staticmethod def from_field( @@ -346,7 +363,7 @@ def from_field( prefix: str, ) -> Optional[SubparsersSpecification]: # Union of classes should create subparsers. - typ = _resolver.unwrap_annotated(field.typ)[0] + typ = _resolver.unwrap_annotated(field.type_or_callable)[0] if get_origin(typ) is not Union: return None @@ -365,6 +382,19 @@ def from_field( ) for o in options ] + + # If specified, swap types using tyro.conf.subcommand(constructor=...). + for i, option in enumerate(options): + _, found_subcommand_configs = _resolver.unwrap_annotated( + option, _confstruct._SubcommandConfiguration + ) + if ( + len(found_subcommand_configs) > 0 + and found_subcommand_configs[0].constructor_factory is not None + ): + options[i] = found_subcommand_configs[0].constructor_factory() + + # Exit if we don't contain nested types. if not all( [ _fields.is_nested_type(cast(type, o), _fields.MISSING_NONPROP) @@ -433,6 +463,7 @@ def from_field( description=None, default=_fields.MISSING_NONPROP, prefix_name=True, + constructor_factory=None, ) # If names match, borrow subcommand default from field default. @@ -507,6 +538,7 @@ def from_field( prefix=prefix, required=required, default_instance=field.default, + options=tuple(options), ) def apply( diff --git a/tyro/_subcommand_matching.py b/tyro/_subcommand_matching.py index b7e6c6b36..9ccef6b5d 100644 --- a/tyro/_subcommand_matching.py +++ b/tyro/_subcommand_matching.py @@ -80,7 +80,7 @@ def make( return _TypeTree( typ, { - field.name: _TypeTree.make(field.typ, field.default) + field.name: _TypeTree.make(field.type_or_callable, field.default) for field in field_list }, ) diff --git a/tyro/conf/_confstruct.py b/tyro/conf/_confstruct.py index e1ce4d06a..1b5c39fd6 100644 --- a/tyro/conf/_confstruct.py +++ b/tyro/conf/_confstruct.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Any, Optional +from typing import Any, Callable, Optional, Type, Union, overload from .._fields import MISSING_NONPROP @@ -10,18 +10,50 @@ class _SubcommandConfiguration: default: Any description: Optional[str] prefix_name: bool + constructor_factory: Optional[Callable[[], Union[Type, Callable]]] def __hash__(self) -> int: return object.__hash__(self) +@overload def subcommand( name: Optional[str] = None, *, default: Any = MISSING_NONPROP, description: Optional[str] = None, prefix_name: bool = True, + constructor: None = None, + constructor_factory: Optional[Callable[[], Union[Type, Callable]]] = None, ) -> Any: + ... + + +@overload +def subcommand( + name: Optional[str] = None, + *, + default: Any = MISSING_NONPROP, + description: Optional[str] = None, + prefix_name: bool = True, + constructor: Optional[Union[Type, Callable]] = None, + constructor_factory: None = None, +) -> Any: + ... + + +def subcommand( + name: Optional[str] = None, + *, + default: Any = MISSING_NONPROP, + description: Optional[str] = None, + prefix_name: bool = True, + constructor: Optional[Union[Type, Callable]] = None, + constructor_factory: Optional[Callable[[], Union[Type, Callable]]] = None, +) -> Any: + assert not ( + constructor is not None and constructor_factory is not None + ), "`constructor` and `constructor_factory` cannot both be set." """Returns a metadata object for configuring subcommands with `typing.Annotated`. Useful for aesthetics. @@ -35,7 +67,7 @@ def subcommand( This will create two subcommands: `nested-type-a` and `nested-type-b`. - Annotating each type with `tyro.metadata.subcommand()` allows us to override for + Annotating each type with `tyro.conf.subcommand()` allows us to override for each subcommand the (a) name, (b) defaults, (c) helptext, and (d) whether to prefix the name or not. @@ -51,8 +83,31 @@ def subcommand( ] ) ``` + + Arguments: + name: The name of the subcommand in the CLI. + default: A default value for the subcommand, for struct-like types. (eg + dataclasses) + description: Description of this option to use in the helptext. Defaults to + docstring. + prefix_name: Whether to prefix the name of the subcommand based on where it + is in a nested structure. + constructor: A constructor type or function. This should either be (a) a subtype + of an argument's annotated type, or (b) a function with type-annotated + inputs that returns an instance of the annotated type. This will be used in + place of the argument's type for parsing arguments. No validation is done. + constructor_factory: A function that returns a constructor type or function. + Useful when the constructor isn't immediately available. """ - return _SubcommandConfiguration(name, default, description, prefix_name) + return _SubcommandConfiguration( + name, + default, + description, + prefix_name, + constructor_factory=constructor_factory + if constructor is None + else lambda: constructor, + ) @dataclasses.dataclass(frozen=True) @@ -61,26 +116,75 @@ class _ArgConfiguration: metavar: Optional[str] help: Optional[str] prefix_name: bool + constructor_factory: Optional[Callable[[], Union[Type, Callable]]] +@overload def arg( *, name: Optional[str] = None, metavar: Optional[str] = None, help: Optional[str] = None, prefix_name: bool = True, + constructor: None = None, + constructor_factory: Optional[Callable[[], Union[Type, Callable]]] = None, ) -> Any: - """Returns a metadata object for configuring arguments with `typing.Annotated`. - Useful for aesthetics. + ... + + +@overload +def arg( + *, + name: Optional[str] = None, + metavar: Optional[str] = None, + help: Optional[str] = None, + prefix_name: bool = True, + constructor: Optional[Union[Type, Callable]] = None, + constructor_factory: None = None, +) -> Any: + ... - Usage: + +def arg( + *, + name: Optional[str] = None, + metavar: Optional[str] = None, + help: Optional[str] = None, + prefix_name: bool = True, + constructor: Optional[Union[Type, Callable]] = None, + constructor_factory: Optional[Callable[[], Union[Type, Callable]]] = None, +) -> Any: + """Returns a metadata object for fine-grained argument configuration with + `typing.Annotated`. Should typically not be required. ```python x: Annotated[int, tyro.conf.arg(...)] ``` + + Arguments: + name: A new name for the argument. + metavar: Argument name in usage messages. The type is used by default. + help: Helptext for this argument. The docstring is used by default. + prefix_name: Whether or not to prefix the name of the argument based on where + it is in a nested structure. + constructor: A constructor type or function. This should either be (a) a subtype + of an argument's annotated type, or (b) a function with type-annotated + inputs that returns an instance of the annotated type. This will be used in + place of the argument's type for parsing arguments. No validation is done. + constructor_factory: A function that returns a constructor type or function. + Useful when the constructor isn't immediately available. + + Returns: + Object to attach via `typing.Annotated[]`. """ + assert not ( + constructor is not None and constructor_factory is not None + ), "`constructor` and `constructor_factory` cannot both be set." return _ArgConfiguration( name=name, metavar=metavar, help=help, prefix_name=prefix_name, + constructor_factory=constructor_factory + if constructor is None + else lambda: constructor, ) diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index 8ee8bcd9c..d80e30c48 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -29,6 +29,9 @@ _UnpackArgsCall = Annotated[T, None] _UnpackKwargsCall = Annotated[T, None] +# Private marker. +_OPTIONAL_GROUP = Annotated[T, None] + # TODO: the verb tenses here are inconsistent, naming could be revisited. # Perhaps Suppress should be Suppressed? But SuppressedFixed would be weird. diff --git a/tyro/extras/__init__.py b/tyro/extras/__init__.py index d6e6b9819..b4dda741b 100644 --- a/tyro/extras/__init__.py +++ b/tyro/extras/__init__.py @@ -10,3 +10,6 @@ from ._choices_type import literal_type_from_choices as literal_type_from_choices from ._serialization import from_yaml as from_yaml from ._serialization import to_yaml as to_yaml +from ._subcommand_cli_from_dict import ( + subcommand_cli_from_dict as subcommand_cli_from_dict, +) diff --git a/tyro/extras/_subcommand_cli_from_dict.py b/tyro/extras/_subcommand_cli_from_dict.py new file mode 100644 index 000000000..f270dbc83 --- /dev/null +++ b/tyro/extras/_subcommand_cli_from_dict.py @@ -0,0 +1,66 @@ +from typing import Any, Callable, Dict, Optional, Sequence, TypeVar, Union, overload + +from typing_extensions import Annotated + +from .._cli import cli +from ..conf import subcommand + +T = TypeVar("T") + + +@overload +def subcommand_cli_from_dict( + subcommands: Dict[str, Callable[..., T]], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + use_underscores: bool = False, +) -> T: + ... + + +# TODO: hack. We prefer the above signature, which Pyright understands, but as of 1.6.1 +# mypy doesn't reason about the generics properly. +@overload +def subcommand_cli_from_dict( + subcommands: Dict[str, Callable[..., Any]], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + use_underscores: bool = False, +) -> Any: + ... + + +def subcommand_cli_from_dict( + subcommands: Dict[str, Callable[..., Any]], + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + use_underscores: bool = False, +) -> Any: + """Generate a subcommand CLI from a dictionary that maps subcommand name to the + corresponding function to call (or object to instantiate).""" + return cli( + Union.__getitem__( # type: ignore + tuple( + [ + Annotated[ + # The constructor function can return any object. + Any, + # We'll instantiate this object by invoking a subcommand with + # name k, via a constructor. + subcommand(name=k, constructor=v), + ] + for k, v in subcommands.items() + ] + ) + ), + prog=prog, + description=description, + args=args, + use_underscores=use_underscores, + ) From 5907cd37fb732be43881f1ed929d49ff603cb33d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 29 Oct 2023 23:13:06 -0700 Subject: [PATCH 330/491] Bump version, docs + tests updates --- .../02_nesting/05_subcommands_func.rst | 31 +++ .../04_additional/10_custom_constructors.rst | 2 - examples/02_nesting/05_subcommands_func.py | 31 +++ .../04_additional/10_custom_constructors.py | 2 - pyproject.toml | 2 +- .../test_conf_generated.py | 189 ++++++++++++++++++ .../test_nested_generated.py | 35 ++++ tyro/_resolver.py | 18 +- tyro/conf/_markers.py | 9 +- tyro/extras/_subcommand_cli_from_dict.py | 49 ++++- 10 files changed, 353 insertions(+), 15 deletions(-) diff --git a/docs/source/examples/02_nesting/05_subcommands_func.rst b/docs/source/examples/02_nesting/05_subcommands_func.rst index 361f6af8c..3eddf238e 100644 --- a/docs/source/examples/02_nesting/05_subcommands_func.rst +++ b/docs/source/examples/02_nesting/05_subcommands_func.rst @@ -8,6 +8,37 @@ Subcommands from Functions :func:`tyro.extras.subcommand_cli_from_dict()` provides a shorthand that generates a subcommand CLI from a dictionary. +For an input like: + +.. code-block:: python + + tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + } + ) + +This is internally accomplished by generating and calling: + +.. code-block:: python + + from typing import Annotated, Any, Union + import tyro + + tyro.cli( + Union[ + Annotated[ + Any, + tyro.conf.subcommand(name="checkout", constructor=checkout), + ], + Annotated[ + Any, + tyro.conf.subcommand(name="commit", constructor=commit), + ], + ] + ) + .. code-block:: python diff --git a/docs/source/examples/04_additional/10_custom_constructors.rst b/docs/source/examples/04_additional/10_custom_constructors.rst index ed2b08b8a..fb3f8880b 100644 --- a/docs/source/examples/04_additional/10_custom_constructors.rst +++ b/docs/source/examples/04_additional/10_custom_constructors.rst @@ -14,9 +14,7 @@ which makes it easier to load complex objects. :linenos: - import dataclasses import json as json_ - from typing import Dict from typing_extensions import Annotated diff --git a/examples/02_nesting/05_subcommands_func.py b/examples/02_nesting/05_subcommands_func.py index 29e1e14f8..0cf9309b7 100644 --- a/examples/02_nesting/05_subcommands_func.py +++ b/examples/02_nesting/05_subcommands_func.py @@ -3,6 +3,37 @@ :func:`tyro.extras.subcommand_cli_from_dict()` provides a shorthand that generates a subcommand CLI from a dictionary. +For an input like: + +```python +tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + } +) +``` + +This is internally accomplished by generating and calling: + +```python +from typing import Annotated, Any, Union +import tyro + +tyro.cli( + Union[ + Annotated[ + Any, + tyro.conf.subcommand(name="checkout", constructor=checkout), + ], + Annotated[ + Any, + tyro.conf.subcommand(name="commit", constructor=commit), + ], + ] +) +``` + Usage: `python ./05_subcommands_func.py --help` `python ./05_subcommands_func.py commit --help` diff --git a/examples/04_additional/10_custom_constructors.py b/examples/04_additional/10_custom_constructors.py index 2d6b9d154..993133b08 100644 --- a/examples/04_additional/10_custom_constructors.py +++ b/examples/04_additional/10_custom_constructors.py @@ -9,9 +9,7 @@ `python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}"` --dict2.json "{\"hello\": \"world\"}"` """ -import dataclasses import json as json_ -from typing import Dict from typing_extensions import Annotated diff --git a/pyproject.toml b/pyproject.toml index 35fb6ee36..d08495e75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.10" +version = "0.5.11" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index d9370e7d5..15dcdac1e 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -1,5 +1,9 @@ import argparse +import contextlib import dataclasses +import io +import json as json_ +import shlex from typing import Annotated, Any, Dict, Generic, List, Tuple, TypeVar, Union import pytest @@ -900,3 +904,188 @@ class TrainConfig: assert tyro.cli( tyro.conf.OmitArgPrefixes[TrainConfig], args="--num-slots 3".split(" ") ) == TrainConfig(ModelConfig(num_slots=3)) + + +def test_custom_constructor_0() -> None: + def times_two(n: str) -> int: + return int(n) * 2 + + @dataclasses.dataclass + class Config: + x: Annotated[int, tyro.conf.arg(constructor=times_two)] + + assert tyro.cli(Config, args="--x.n 5".split(" ")) == Config(x=10) + + +def test_custom_constructor_1() -> None: + def times_two(n: int) -> int: + return int(n) * 2 + + @dataclasses.dataclass + class Config: + x: Annotated[int, tyro.conf.arg(constructor=times_two)] + + assert tyro.cli(Config, args="--x.n 5".split(" ")) == Config(x=10) + + +def test_custom_constructor_2() -> None: + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=int)] + + assert tyro.cli(Config, args="--x 5".split(" ")) == Config(x=5) + with pytest.raises(SystemExit): + tyro.cli(Config, args="--x 5.23".split(" ")) + + +def test_custom_constructor_3() -> None: + def dict_from_json(json: str) -> dict: + out = json_.loads(json) + if not isinstance(out, dict): + raise ValueError(f"{json} is not a dict!") + return out + + @dataclasses.dataclass + class Config: + x: Annotated[ + dict, + tyro.conf.arg( + metavar="JSON", + constructor=dict_from_json, + ), + ] + + assert tyro.cli( + Config, args=shlex.split('--x.json \'{"hello": "world"}\'') + ) == Config(x={"hello": "world"}) + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Config, args="--x.json 5".split(" ")) + + error = target.getvalue() + assert "Error parsing x: 5 is not a dict!" in error + + +def test_custom_constructor_4() -> None: + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=int)] = 3.23 + + assert tyro.cli(Config, args="--x 5".split(" ")) == Config(x=5) + assert tyro.cli(Config, args=[]) == Config(x=3.23) + + +def test_custom_constructor_5() -> None: + def make_float(a: float, b: float, c: float = 3) -> float: + return a * b * c + + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=make_float)] = 3.23 + + assert tyro.cli(Config, args=[]) == Config(x=3.23) + assert tyro.cli(Config, args="--x.a 5 --x.b 2 --x.c 3".split(" ")) == Config(x=30) + assert tyro.cli(Config, args="--x.a 5 --x.b 2".split(" ")) == Config(x=30) + + # --x.b is required! + with pytest.raises(SystemExit): + tyro.cli(Config, args="--x.a 5".split(" ")) + + # --x.a and --x.b are required! + with pytest.raises(SystemExit): + tyro.cli(Config, args="--x.c 5".split(" ")) + + +def test_custom_constructor_6() -> None: + def make_float(a: tyro.conf.Positional[float], b: float, c: float = 3) -> float: + return a * b * c + + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=make_float)] = 3.23 + + assert tyro.cli(Config, args=[]) == Config(x=3.23) + assert tyro.cli(Config, args="--x.b 2 --x.c 3 5".split(" ")) == Config(x=30) + assert tyro.cli(Config, args="--x.b 2 5".split(" ")) == Config(x=30) + + # --x.b is required! + with pytest.raises(SystemExit): + tyro.cli(Config, args="5".split(" ")) + + # --x.a and --x.b are required! + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Config, args="--x.c 5".split(" ")) + error = target.getvalue() + assert "We're missing" in error + + +def test_custom_constructor_7() -> None: + @dataclasses.dataclass + class Struct: + a: int + b: int + c: int = 3 + + def make_float(struct: Struct) -> float: + return struct.a * struct.b * struct.c + + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=make_float)] = 3.23 + + assert tyro.cli(Config, args=[]) == Config(x=3.23) + assert tyro.cli( + Config, args="--x.struct.a 5 --x.struct.b 2 --x.struct.c 3".split(" ") + ) == Config(x=30) + assert tyro.cli(Config, args="--x.struct.a 5 --x.struct.b 2".split(" ")) == Config( + x=30 + ) + + # --x.struct.b is required! + with pytest.raises(SystemExit): + tyro.cli(Config, args="--x.struct.a 5".split(" ")) + + # --x.struct.a and --x.struct.b are required! + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Config, args="--x.struct.c 5".split(" ")) + error = target.getvalue() + assert "We're missing arguments" in error + assert "'b'" in error + assert "'a'" in error # The 5 is parsed into `a`. + + +def test_custom_constructor_8() -> None: + @dataclasses.dataclass + class Struct: + a: tyro.conf.Positional[int] + b: int + c: int = 3 + + def make_float(struct: Struct) -> float: + return struct.a * struct.b * struct.c + + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=make_float)] = 3.23 + + assert tyro.cli(Config, args=[]) == Config(x=3.23) + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 5".split(" ") + ) == Config(x=30) + assert tyro.cli(Config, args="--x.struct.b 2 5".split(" ")) == Config(x=30) + + # --x.struct.b is required! + with pytest.raises(SystemExit): + tyro.cli(Config, args="5".split(" ")) + + # --x.struct.a and --x.struct.b are required! + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Config, args="--x.struct.b 5".split(" ")) + error = target.getvalue() + assert "We're missing arguments" in error + assert "'a'" in error + assert "'b'" not in error diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index 79e94a6b1..c60ca3e98 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -1026,3 +1026,38 @@ class Level3(BaseConfig): child: Level2 = dataclasses.field(default_factory=lambda: Level2()) tyro.cli(Level3, args=[]) + + +def test_subcommand_dict_helper() -> None: + def checkout(branch: str) -> str: + """Check out a branch.""" + return branch + + def commit(message: str, all: bool = False) -> Tuple[str, bool]: + """Make a commit.""" + return (message, all) + + assert ( + tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + }, + args="checkout --branch main".split(" "), + ) + == "main" + ) + assert tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + }, + args="commit --message hello".split(" "), + ) == ("hello", False) + assert tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + }, + args="commit --message hello --all".split(" "), + ) == ("hello", True) diff --git a/tyro/_resolver.py b/tyro/_resolver.py index cc468898d..3c6852389 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -193,18 +193,24 @@ def unwrap_annotated( - Annotated[int, 1], int => (int, (1,)) - Annotated[int, "1"], int => (int, ()) """ + targets = tuple( + x + for x in getattr(typ, "__tyro_markers__", tuple()) + if search_type is not None and isinstance(x, search_type) + ) + assert isinstance(targets, tuple) if not hasattr(typ, "__metadata__"): - return typ, () + return typ, targets args = get_args(typ) assert len(args) >= 2 - # Don't search for a specific metadata type if `None` is passed in. - if search_type is None: - return args[0], () - # Look through metadata for desired metadata type. - targets = tuple(x for x in args[1:] if isinstance(x, search_type)) + targets = tuple( + x + for x in targets + args[1:] + if search_type is not None and isinstance(x, search_type) + ) return args[0], targets diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index d80e30c48..3fbb717cc 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -136,7 +136,7 @@ def __getitem__(self, key): def configure(*markers: Marker) -> Callable[[CallableType], CallableType]: - """Decorator for configuring functions. + """Decorator for applying configuration options. Configuration markers are implemented via `typing.Annotated` and straightforward to apply to types, for example: @@ -153,10 +153,15 @@ def configure(*markers: Marker) -> Callable[[CallableType], CallableType]: def main(field: bool) -> None: ... ``` + + Args: + markers: Options to apply. """ def _inner(callable: CallableType) -> CallableType: - return Annotated.__class_getitem__((callable,) + tuple(markers)) # type: ignore + # We'll read from __tyro_markers__ in `_resolver.unwrap_annotated()`. + callable.__tyro_markers__ = markers # type: ignore + return callable return _inner diff --git a/tyro/extras/_subcommand_cli_from_dict.py b/tyro/extras/_subcommand_cli_from_dict.py index f270dbc83..0e1cd56cd 100644 --- a/tyro/extras/_subcommand_cli_from_dict.py +++ b/tyro/extras/_subcommand_cli_from_dict.py @@ -42,8 +42,53 @@ def subcommand_cli_from_dict( args: Optional[Sequence[str]] = None, use_underscores: bool = False, ) -> Any: - """Generate a subcommand CLI from a dictionary that maps subcommand name to the - corresponding function to call (or object to instantiate).""" + """Generate a subcommand CLI from a dictionary of functions. + + For an input like: + + ```python + tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + } + ) + ``` + + This is internally accomplished by generating and calling: + + ```python + from typing import Annotated, Any, Union + import tyro + + tyro.cli( + Union[ + Annotated[ + Any, + tyro.conf.subcommand(name="checkout", constructor=checkout), + ], + Annotated[ + Any, + tyro.conf.subcommand(name="commit", constructor=commit), + ], + ] + ) + ``` + + Args: + subcommands: Dictionary that maps the subcommand name to function to call. + prog: The name of the program printed in helptext. Mirrors argument from + `argparse.ArgumentParser()`. + description: Description text for the parser, displayed when the --help flag is + passed in. If not specified, `f`'s docstring is used. Mirrors argument from + `argparse.ArgumentParser()`. + args: If set, parse arguments from a sequence of strings instead of the + commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. + use_underscores: If True, use underscores as a word delimeter instead of hyphens. + This primarily impacts helptext; underscores and hyphens are treated equivalently + when parsing happens. We default helptext to hyphens to follow the GNU style guide. + https://www.gnu.org/software/libc/manual/html_node/Argument-Syntax.html + """ return cli( Union.__getitem__( # type: ignore tuple( From f5486258c368be8cb9480aa59f8b123745805cc9 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 31 Oct 2023 12:40:50 -0700 Subject: [PATCH 331/491] Implement aliases (#83) * Implement aliases * Format --- .../examples/04_additional/11_aliases.rst | 98 +++++++++++++++++++ examples/04_additional/11_aliases.py | 41 ++++++++ pyproject.toml | 2 +- tests/test_conf.py | 79 +++++++++++++++ .../test_conf_generated.py | 79 +++++++++++++++ tyro/_arguments.py | 15 ++- tyro/_fields.py | 28 ++++-- tyro/conf/_confstruct.py | 27 +++-- 8 files changed, 353 insertions(+), 16 deletions(-) create mode 100644 docs/source/examples/04_additional/11_aliases.rst create mode 100644 examples/04_additional/11_aliases.py diff --git a/docs/source/examples/04_additional/11_aliases.rst b/docs/source/examples/04_additional/11_aliases.rst new file mode 100644 index 000000000..e364f40cc --- /dev/null +++ b/docs/source/examples/04_additional/11_aliases.rst @@ -0,0 +1,98 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Argument aliases +========================================== + + +:func:`tyro.conf.arg()` can be used to attach aliases to arguments. + + + +.. code-block:: python + :linenos: + + + from typing_extensions import Annotated + + import tyro + + + def checkout( + branch: Annotated[str, tyro.conf.arg(aliases=["-b"])], + ) -> None: + """Check out a branch.""" + print(f"{branch=}") + + + def commit( + message: Annotated[str, tyro.conf.arg(aliases=["-m"])], + all: Annotated[str, tyro.conf.arg(aliases=["-a"])], + ) -> None: + """Make a commit.""" + print(f"{message=} {all=}") + + + if __name__ == "__main__": + tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + } + ) + +------------ + +.. raw:: html + + python 04_additional/11_aliases.py --help + +.. program-output:: python ../../examples/04_additional/11_aliases.py --help + +------------ + +.. raw:: html + + python 04_additional/11_aliases.py commit --help + +.. program-output:: python ../../examples/04_additional/11_aliases.py commit --help + +------------ + +.. raw:: html + + python 04_additional/11_aliases.py commit --message hello --all + +.. program-output:: python ../../examples/04_additional/11_aliases.py commit --message hello --all + +------------ + +.. raw:: html + + python 04_additional/11_aliases.py commit -m hello -a + +.. program-output:: python ../../examples/04_additional/11_aliases.py commit -m hello -a + +------------ + +.. raw:: html + + python 04_additional/11_aliases.py checkout --help + +.. program-output:: python ../../examples/04_additional/11_aliases.py checkout --help + +------------ + +.. raw:: html + + python 04_additional/11_aliases.py checkout --branch main + +.. program-output:: python ../../examples/04_additional/11_aliases.py checkout --branch main + +------------ + +.. raw:: html + + python 04_additional/11_aliases.py checkout -b main + +.. program-output:: python ../../examples/04_additional/11_aliases.py checkout -b main diff --git a/examples/04_additional/11_aliases.py b/examples/04_additional/11_aliases.py new file mode 100644 index 000000000..2e55c5290 --- /dev/null +++ b/examples/04_additional/11_aliases.py @@ -0,0 +1,41 @@ +"""Argument aliases + +:func:`tyro.conf.arg()` can be used to attach aliases to arguments. + +Usage: +`python ./11_aliases.py --help` +`python ./11_aliases.py commit --help` +`python ./11_aliases.py commit --message hello --all` +`python ./11_aliases.py commit -m hello -a` +`python ./11_aliases.py checkout --help` +`python ./11_aliases.py checkout --branch main` +`python ./11_aliases.py checkout -b main` +""" + +from typing_extensions import Annotated + +import tyro + + +def checkout( + branch: Annotated[str, tyro.conf.arg(aliases=["-b"])], +) -> None: + """Check out a branch.""" + print(f"{branch=}") + + +def commit( + message: Annotated[str, tyro.conf.arg(aliases=["-m"])], + all: Annotated[str, tyro.conf.arg(aliases=["-a"])], +) -> None: + """Make a commit.""" + print(f"{message=} {all=}") + + +if __name__ == "__main__": + tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + } + ) diff --git a/pyproject.toml b/pyproject.toml index d08495e75..7bb0f2d35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.11" +version = "0.5.12" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/tests/test_conf.py b/tests/test_conf.py index 96a6f5f84..5e2a574bb 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1090,3 +1090,82 @@ class Config: assert "We're missing arguments" in error assert "'a'" in error assert "'b'" not in error + + +def test_alias() -> None: + """Arguments with aliases.""" + + @dataclasses.dataclass + class Struct: + a: Annotated[int, tyro.conf.arg(aliases=["--all", "-d"])] + b: int + c: int = 3 + + def make_float(struct: Struct) -> float: + return struct.a * struct.b * struct.c + + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=make_float)] = 3.23 + + assert tyro.cli(Config, args=[]) == Config(x=3.23) + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 --x.struct.a 5".split(" ") + ) == Config(x=30) + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 -d 5".split(" ") + ) == Config(x=30) + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 --all 5".split(" ") + ) == Config(x=30) + assert tyro.cli(Config, args="--x.struct.b 2 --x.struct.a 5".split(" ")) == Config( + x=30 + ) + + # --x.struct.b is required! + with pytest.raises(SystemExit): + tyro.cli(Config, args="--x.struct.a 5".split(" ")) + + # --x.struct.a and --x.struct.b are required! + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Config, args="--x.struct.b 5".split(" ")) + error = target.getvalue() + assert "We're missing arguments" in error + assert "'a'" in error + assert "'b'" not in error + + assert "--x.struct.a INT, --all INT, -d INT" in get_helptext(Config) + + +def test_positional_alias() -> None: + """Positional arguments with aliases (which will be ignored).""" + + @dataclasses.dataclass + class Struct: + a: Annotated[tyro.conf.Positional[int], tyro.conf.arg(aliases=["--all", "-d"])] + b: int + c: int = 3 + + def make_float(struct: Struct) -> float: + return struct.a * struct.b * struct.c + + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=make_float)] = 3.23 + + with pytest.warns(UserWarning): + assert tyro.cli(Config, args=[]) == Config(x=3.23) + with pytest.warns(UserWarning): + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 5".split(" ") + ) == Config(x=30) + + with pytest.raises(SystemExit), pytest.warns(UserWarning): + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 -d 5".split(" ") + ) == Config(x=30) + with pytest.raises(SystemExit), pytest.warns(UserWarning): + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 --all 5".split(" ") + ) == Config(x=30) diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index 15dcdac1e..1e7bc887d 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -1089,3 +1089,82 @@ class Config: assert "We're missing arguments" in error assert "'a'" in error assert "'b'" not in error + + +def test_alias() -> None: + """Arguments with aliases.""" + + @dataclasses.dataclass + class Struct: + a: Annotated[int, tyro.conf.arg(aliases=["--all", "-d"])] + b: int + c: int = 3 + + def make_float(struct: Struct) -> float: + return struct.a * struct.b * struct.c + + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=make_float)] = 3.23 + + assert tyro.cli(Config, args=[]) == Config(x=3.23) + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 --x.struct.a 5".split(" ") + ) == Config(x=30) + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 -d 5".split(" ") + ) == Config(x=30) + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 --all 5".split(" ") + ) == Config(x=30) + assert tyro.cli(Config, args="--x.struct.b 2 --x.struct.a 5".split(" ")) == Config( + x=30 + ) + + # --x.struct.b is required! + with pytest.raises(SystemExit): + tyro.cli(Config, args="--x.struct.a 5".split(" ")) + + # --x.struct.a and --x.struct.b are required! + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(Config, args="--x.struct.b 5".split(" ")) + error = target.getvalue() + assert "We're missing arguments" in error + assert "'a'" in error + assert "'b'" not in error + + assert "--x.struct.a INT, --all INT, -d INT" in get_helptext(Config) + + +def test_positional_alias() -> None: + """Positional arguments with aliases (which will be ignored).""" + + @dataclasses.dataclass + class Struct: + a: Annotated[tyro.conf.Positional[int], tyro.conf.arg(aliases=["--all", "-d"])] + b: int + c: int = 3 + + def make_float(struct: Struct) -> float: + return struct.a * struct.b * struct.c + + @dataclasses.dataclass + class Config: + x: Annotated[float, tyro.conf.arg(constructor=make_float)] = 3.23 + + with pytest.warns(UserWarning): + assert tyro.cli(Config, args=[]) == Config(x=3.23) + with pytest.warns(UserWarning): + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 5".split(" ") + ) == Config(x=30) + + with pytest.raises(SystemExit), pytest.warns(UserWarning): + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 -d 5".split(" ") + ) == Config(x=30) + with pytest.raises(SystemExit), pytest.warns(UserWarning): + assert tyro.cli( + Config, args="--x.struct.b 2 --x.struct.c 3 --all 5".split(" ") + ) == Config(x=30) diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 79924af05..4a6cd8d72 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -141,8 +141,19 @@ def add_argument( if self.field.argconf.metavar is not None: kwargs["metavar"] = self.field.argconf.metavar - # Add argument! - arg = parser.add_argument(name_or_flag, **kwargs) + # Add argument, with aliases if available. + if self.field.argconf.aliases is not None and not self.field.is_positional(): + arg = parser.add_argument( + name_or_flag, *self.field.argconf.aliases, **kwargs + ) + else: + if self.field.argconf.aliases is not None: + import warnings + + warnings.warn( + f"Aliases were specified, but {name_or_flag} is positional. Aliases will be ignored." + ) + arg = parser.add_argument(name_or_flag, **kwargs) # Do our best to tab complete paths. # There will be false positives here, but if choices is unset they should be diff --git a/tyro/_fields.py b/tyro/_fields.py index 295bcb952..95e9ecdb6 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -77,7 +77,7 @@ def __post_init__(self): @staticmethod def make( name: str, - type_or_callable: TypeForm[Any], + type_or_callable: Union[TypeForm[Any], Callable], default: Any, helptext: Optional[str], call_argname_override: Optional[Any] = None, @@ -88,12 +88,26 @@ def make( _, argconfs = _resolver.unwrap_annotated( type_or_callable, _confstruct._ArgConfiguration ) - if len(argconfs) == 0: - argconf = _confstruct._ArgConfiguration(None, None, None, True, None) - else: - assert len(argconfs) == 1 - (argconf,) = argconfs - helptext = argconf.help + argconf = _confstruct._ArgConfiguration( + None, + None, + help=None, + aliases=None, + prefix_name=True, + constructor_factory=None, + ) + for overwrite_argconf in argconfs: + # Apply any annotated argument configuration values. + argconf = dataclasses.replace( + argconf, + **{ + field.name: getattr(overwrite_argconf, field.name) + for field in dataclasses.fields(overwrite_argconf) + if getattr(overwrite_argconf, field.name) is not None + }, + ) + if argconf.help is not None: + helptext = argconf.help type_or_callable, inferred_markers = _resolver.unwrap_annotated( type_or_callable, _markers._Marker diff --git a/tyro/conf/_confstruct.py b/tyro/conf/_confstruct.py index 1b5c39fd6..f225c6b08 100644 --- a/tyro/conf/_confstruct.py +++ b/tyro/conf/_confstruct.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Any, Callable, Optional, Type, Union, overload +from typing import Any, Callable, Optional, Sequence, Tuple, Type, Union, overload from .._fields import MISSING_NONPROP @@ -112,10 +112,13 @@ def subcommand( @dataclasses.dataclass(frozen=True) class _ArgConfiguration: + # These are all optional by default in order to support multiple tyro.conf.arg() + # annotations. A None value means "don't overwrite the current value". name: Optional[str] metavar: Optional[str] help: Optional[str] - prefix_name: bool + aliases: Optional[Tuple[str, ...]] + prefix_name: Optional[bool] constructor_factory: Optional[Callable[[], Union[Type, Callable]]] @@ -125,7 +128,8 @@ def arg( name: Optional[str] = None, metavar: Optional[str] = None, help: Optional[str] = None, - prefix_name: bool = True, + aliases: Optional[Sequence[str]] = None, + prefix_name: Optional[bool] = None, constructor: None = None, constructor_factory: Optional[Callable[[], Union[Type, Callable]]] = None, ) -> Any: @@ -138,7 +142,8 @@ def arg( name: Optional[str] = None, metavar: Optional[str] = None, help: Optional[str] = None, - prefix_name: bool = True, + aliases: Optional[Sequence[str]] = None, + prefix_name: Optional[bool] = None, constructor: Optional[Union[Type, Callable]] = None, constructor_factory: None = None, ) -> Any: @@ -150,7 +155,8 @@ def arg( name: Optional[str] = None, metavar: Optional[str] = None, help: Optional[str] = None, - prefix_name: bool = True, + aliases: Optional[Sequence[str]] = None, + prefix_name: Optional[bool] = None, constructor: Optional[Union[Type, Callable]] = None, constructor_factory: Optional[Callable[[], Union[Type, Callable]]] = None, ) -> Any: @@ -164,8 +170,11 @@ def arg( name: A new name for the argument. metavar: Argument name in usage messages. The type is used by default. help: Helptext for this argument. The docstring is used by default. + aliases: Aliases for this argument. All strings in the sequence should start + with a hyphen (-). Aliases will _not_ currently be prefixed in a nested + structure, and are not supported for positional arguments. prefix_name: Whether or not to prefix the name of the argument based on where - it is in a nested structure. + it is in a nested structure. Arguments are prefixed by default. constructor: A constructor type or function. This should either be (a) a subtype of an argument's annotated type, or (b) a function with type-annotated inputs that returns an instance of the annotated type. This will be used in @@ -179,10 +188,16 @@ def arg( assert not ( constructor is not None and constructor_factory is not None ), "`constructor` and `constructor_factory` cannot both be set." + + if aliases is not None: + for alias in aliases: + assert alias.startswith("-"), "Argument alias needs to start with a hyphen!" + return _ArgConfiguration( name=name, metavar=metavar, help=help, + aliases=tuple(aliases) if aliases is not None else None, prefix_name=prefix_name, constructor_factory=constructor_factory if constructor is None From be0545f9ad9c54502d0f254a5d709985936629f4 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 31 Oct 2023 12:46:47 -0700 Subject: [PATCH 332/491] Fix aliases example, add boolean tests --- .../examples/04_additional/11_aliases.rst | 2 +- examples/04_additional/11_aliases.py | 2 +- tests/test_conf.py | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/source/examples/04_additional/11_aliases.rst b/docs/source/examples/04_additional/11_aliases.rst index e364f40cc..8a13511fc 100644 --- a/docs/source/examples/04_additional/11_aliases.rst +++ b/docs/source/examples/04_additional/11_aliases.rst @@ -27,7 +27,7 @@ Argument aliases def commit( message: Annotated[str, tyro.conf.arg(aliases=["-m"])], - all: Annotated[str, tyro.conf.arg(aliases=["-a"])], + all: Annotated[bool, tyro.conf.arg(aliases=["-a"])] = False, ) -> None: """Make a commit.""" print(f"{message=} {all=}") diff --git a/examples/04_additional/11_aliases.py b/examples/04_additional/11_aliases.py index 2e55c5290..212cffaa2 100644 --- a/examples/04_additional/11_aliases.py +++ b/examples/04_additional/11_aliases.py @@ -26,7 +26,7 @@ def checkout( def commit( message: Annotated[str, tyro.conf.arg(aliases=["-m"])], - all: Annotated[str, tyro.conf.arg(aliases=["-a"])], + all: Annotated[bool, tyro.conf.arg(aliases=["-a"])] = False, ) -> None: """Make a commit.""" print(f"{message=} {all=}") diff --git a/tests/test_conf.py b/tests/test_conf.py index 5e2a574bb..e0d72c706 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1169,3 +1169,20 @@ class Config: assert tyro.cli( Config, args="--x.struct.b 2 --x.struct.c 3 --all 5".split(" ") ) == Config(x=30) + + +def test_flag_alias() -> None: + @dataclasses.dataclass + class Struct: + flag: Annotated[bool, tyro.conf.arg(aliases=["-f", "--flg"])] = False + + assert tyro.cli(Struct, args=[]).flag == False + assert tyro.cli(Struct, args="--flag".split(" ")).flag == True + assert tyro.cli(Struct, args="--no-flag".split(" ")).flag == False + assert tyro.cli(Struct, args="--flg".split(" ")).flag == True + assert tyro.cli(Struct, args="--no-flg".split(" ")).flag == False + assert tyro.cli(Struct, args="-f".split(" ")).flag == True + + # BooleanOptionalAction will ignore arguments that aren't prefixed with --. + with pytest.raises(SystemExit): + tyro.cli(Struct, args="-no-f".split(" ")) From d1a42284e1003aff5f22d6ce3b8f3b4286b944f5 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 13 Nov 2023 12:34:17 -0800 Subject: [PATCH 333/491] Make unpack-style args optional --- tests/test_dcargs.py | 4 +++- tyro/_fields.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 61e5f3890..e82845839 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -669,10 +669,12 @@ def main(x: os.PathLike) -> os.PathLike: assert tyro.cli(main, args=["--x", "/dev/null"]) == pathlib.Path("/dev/null") -def test_variadics() -> None: +def test_unpack() -> None: def main(*args: int, **kwargs: float) -> Tuple[Tuple[int, ...], Dict[str, float]]: return args, kwargs + assert tyro.cli(main, args=[]) == ((), {}) + assert tyro.cli(main, args="--args --kwargs".split(" ")) == ((), {}) assert tyro.cli( main, args="--args 1 2 3 --kwargs learning_rate 1e-4 beta1 0.99".split(" ") ) == ((1, 2, 3), {"learning_rate": 1e-4, "beta1": 0.99}) diff --git a/tyro/_fields.py b/tyro/_fields.py index 95e9ecdb6..9234e68a9 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -846,6 +846,7 @@ def _field_list_from_params( # This will create a `--args T [T ...]` CLI argument. markers = (_markers._UnpackArgsCall,) typ = Tuple.__getitem__((typ, ...)) # type: ignore + default = () elif param.kind is inspect.Parameter.VAR_KEYWORD: # Handle *kwargs signatures. # @@ -857,6 +858,7 @@ def _field_list_from_params( # conjunction. markers = (_markers._UnpackKwargsCall,) typ = Dict.__getitem__((str, typ)) # type: ignore + default = {} field_list.append( FieldDefinition.make( From 5d571acaf94166e2393ad44c079a241e898d7115 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 14 Nov 2023 04:53:04 -0800 Subject: [PATCH 334/491] Fix #86 --- pyproject.toml | 2 +- tests/test_errors.py | 10 ++++++++++ tyro/_calling.py | 43 ++++++++++++++++++++++++------------------- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7bb0f2d35..1d04bb0ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.12" +version = "0.5.13" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/tests/test_errors.py b/tests/test_errors.py index 46c8c88de..3943e6cb5 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -469,3 +469,13 @@ class Args: # Printed in the similar argument list. assert error.count("--flag, --no-flag") == 1 + + +def test_value_error() -> None: + """https://github.com/brentyi/tyro/issues/86""" + + def main() -> None: + raise ValueError("This shouldn't be caught by tyro") + + with pytest.raises(ValueError): + tyro.cli(main, args=[]) diff --git a/tyro/_calling.py b/tyro/_calling.py index 5fd3a88f4..c82b1e2b4 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -219,23 +219,28 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: unwrapped_f = _resolver.unwrap_origin_strip_extras(unwrapped_f) unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore - try: - if unwrapped_f in (tuple, list, set): - assert len(positional_args) == 0 - # When tuples are used as nested structures (eg Tuple[SomeDataclass]), we - # use keyword arguments. - assert len(positional_args) == 0 - return unwrapped_f(kwargs.values()), consumed_keywords # type: ignore - elif unwrapped_f is dict: - assert len(positional_args) == 0 - return kwargs, consumed_keywords # type: ignore - else: + if unwrapped_f in (tuple, list, set): + assert len(positional_args) == 0 + # When tuples are used as nested structures (eg Tuple[SomeDataclass]), we + # use keyword arguments. + assert len(positional_args) == 0 + return unwrapped_f(kwargs.values()), consumed_keywords # type: ignore + elif unwrapped_f is dict: + assert len(positional_args) == 0 + return kwargs, consumed_keywords # type: ignore + else: + if field_name_prefix == "": + # Don't catch any errors for the "root" field. If main() in tyro.cli(main) + # raises a ValueError, this shouldn't be caught. return unwrapped_f(*positional_args, **kwargs), consumed_keywords # type: ignore - - # If unwrapped_f raises a ValueError, wrap the message with a more informative - # InstantiationError if possible. - except ValueError as e: - raise InstantiationError( - e.args[0], - field_name_prefix, - ) + else: + # Try to catch ValueErrors raised by field constructors. + try: + return unwrapped_f(*positional_args, **kwargs), consumed_keywords # type: ignore + # If unwrapped_f raises a ValueError, wrap the message with a more informative + # InstantiationError if possible. + except ValueError as e: + raise InstantiationError( + e.args[0], + field_name_prefix, + ) From a6e9d0531784862c6fe9f2e90085a1bd50d6386b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 14 Nov 2023 04:57:36 -0800 Subject: [PATCH 335/491] Add ValueError test for subcommands --- tests/test_errors.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_errors.py b/tests/test_errors.py index 3943e6cb5..796a6463b 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -479,3 +479,16 @@ def main() -> None: with pytest.raises(ValueError): tyro.cli(main, args=[]) + + +def test_value_error_subcommand() -> None: + """https://github.com/brentyi/tyro/issues/86""" + + def main1() -> None: + raise ValueError("This shouldn't be caught by tyro") + + def main2() -> None: + raise ValueError("This shouldn't be caught by tyro") + + with pytest.raises(ValueError): + tyro.extras.subcommand_cli_from_dict({"main1": main1, "main2": main2}, args=["main1"]) From 5f2553d320abad5c1f759e83e96044b4f65e6b60 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 14 Nov 2023 05:06:37 -0800 Subject: [PATCH 336/491] Fix tuple of structs for Python>=3.9 syntax --- tests/test_new_style_annotations_min_py39.py | 12 ++++++++++++ tyro/_fields.py | 2 +- tyro/_resolver.py | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_new_style_annotations_min_py39.py b/tests/test_new_style_annotations_min_py39.py index 5ef86858c..22cd9160e 100644 --- a/tests/test_new_style_annotations_min_py39.py +++ b/tests/test_new_style_annotations_min_py39.py @@ -1,3 +1,4 @@ +import dataclasses from typing import Any, Literal, Optional, Union import pytest @@ -19,6 +20,17 @@ def main(x: tuple[bool, str]) -> Any: assert tyro.cli(main, args=["--x", "True", "False"]) == (True, "False") +def test_tuple_nested(): + @dataclasses.dataclass + class Args: + a: int + + def main(x: tuple[Args, Args]) -> Any: + return x + + assert tyro.cli(main, args=["--x.0.a", "3", "--x.1.a", "4"]) == (Args(3), Args(4)) + + def test_tuple_variable(): def main(x: tuple[Union[bool, str], ...]) -> Any: return x diff --git a/tyro/_fields.py b/tyro/_fields.py index 9234e68a9..5991ffdb7 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -360,7 +360,7 @@ def _try_field_list_from_callable( return UnsupportedNestedTypeMessage(f"{f} should be parsed directly!") elif ( cls is not None - and issubclass(cls, os.PathLike) + and issubclass(_resolver.unwrap_origin_strip_extras(cls), os.PathLike) and _instantiators.is_type_string_converter(cls) ): return UnsupportedNestedTypeMessage( diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 3c6852389..a702ee8c6 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -26,7 +26,7 @@ from . import _fields, _unsafe_cache from ._typing import TypeForm -TypeOrCallable = TypeVar("TypeOrCallable", TypeForm, Callable) +TypeOrCallable = TypeVar("TypeOrCallable", TypeForm[Any], Callable) def unwrap_origin_strip_extras(typ: TypeOrCallable) -> TypeOrCallable: From fb58768f9c89eb61cd064e95f51b1fb22dc2bbfa Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 14 Nov 2023 05:07:34 -0800 Subject: [PATCH 337/491] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1d04bb0ee..66355e906 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.13" +version = "0.5.14" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } From 4042f0d45098de457f76f074f72226e057a68994 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 14 Nov 2023 05:26:07 -0800 Subject: [PATCH 338/491] Run black --- tests/test_errors.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_errors.py b/tests/test_errors.py index 796a6463b..8a5ace5bb 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -491,4 +491,6 @@ def main2() -> None: raise ValueError("This shouldn't be caught by tyro") with pytest.raises(ValueError): - tyro.extras.subcommand_cli_from_dict({"main1": main1, "main2": main2}, args=["main1"]) + tyro.extras.subcommand_cli_from_dict( + {"main1": main1, "main2": main2}, args=["main1"] + ) From 43061a19870df90f89c322abdd88e040ea5e4c67 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 17 Nov 2023 11:52:35 -0800 Subject: [PATCH 339/491] Add NotRequired test from #87 --- tests/test_dict_namedtuple.py | 52 +++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index a1cedb076..eb54d4a90 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -530,3 +530,55 @@ def test_nested_dict_annotations() -> None: ) assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" del overrided_config + + +def test_functional_typeddict(): + """Source: https://github.com/brentyi/tyro/issues/87""" + NerfMLPHiddenLayers_0 = TypedDict( + "NerfMLPHiddenLayers0", + { + "hidden_layers.0": int, + "hidden_layers.1": int, + "hidden_layers.2": int, + "hidden_layers.3": int, + "hidden_layers.4": int, + "hidden_layers.5": int, + "hidden_layers.6": int, + "hidden_layers.7": int, + }, + ) + NerfMLPHiddenLayers_1 = TypedDict( + "NerfMLPHiddenLayers1", + { + "hidden_layers.0": NotRequired[int], + "hidden_layers.1": NotRequired[int], + "hidden_layers.2": NotRequired[int], + "hidden_layers.3": NotRequired[int], + "hidden_layers.4": NotRequired[int], + "hidden_layers.5": NotRequired[int], + "hidden_layers.6": NotRequired[int], + "hidden_layers.7": NotRequired[int], + }, + ) + NerfMLPHiddenLayers_2 = TypedDict( + "NerfMLPHiddenLayers2", + { + "hidden_layers.0": int, + "hidden_layers.1": int, + "hidden_layers.2": int, + "hidden_layers.3": int, + "hidden_layers.4": int, + "hidden_layers.5": int, + "hidden_layers.6": int, + "hidden_layers.7": int, + }, + total=False, + ) + with pytest.raises(SystemExit): + tyro.cli(NerfMLPHiddenLayers_0, args=["--hidden_layers.0", "3"]) + assert tyro.cli(NerfMLPHiddenLayers_1, args=["--hidden_layers.0", "3"]) == { + "hidden_layers.0": 3 + } + assert tyro.cli(NerfMLPHiddenLayers_2, args=["--hidden_layers.0", "3"]) == { + "hidden_layers.0": 3 + } From 0c4dfa3dfa2b2ea91efc1b58b65ca6c77d39d00f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 17 Nov 2023 13:59:43 -0800 Subject: [PATCH 340/491] Fix TypedDict's total=False, NotRequired[] when default is provided --- pyproject.toml | 2 +- tests/test_conf.py | 12 ++++---- tests/test_dict_namedtuple.py | 52 +++++++++++++++++++++++++++++++++++ tyro/_arguments.py | 2 +- tyro/_calling.py | 1 - tyro/_fields.py | 5 ++-- 6 files changed, 63 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 66355e906..c1ff0f07c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.14" +version = "0.5.15" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/tests/test_conf.py b/tests/test_conf.py index e0d72c706..12a790d62 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1176,12 +1176,12 @@ def test_flag_alias() -> None: class Struct: flag: Annotated[bool, tyro.conf.arg(aliases=["-f", "--flg"])] = False - assert tyro.cli(Struct, args=[]).flag == False - assert tyro.cli(Struct, args="--flag".split(" ")).flag == True - assert tyro.cli(Struct, args="--no-flag".split(" ")).flag == False - assert tyro.cli(Struct, args="--flg".split(" ")).flag == True - assert tyro.cli(Struct, args="--no-flg".split(" ")).flag == False - assert tyro.cli(Struct, args="-f".split(" ")).flag == True + assert tyro.cli(Struct, args=[]).flag is False + assert tyro.cli(Struct, args="--flag".split(" ")).flag is True + assert tyro.cli(Struct, args="--no-flag".split(" ")).flag is False + assert tyro.cli(Struct, args="--flg".split(" ")).flag is True + assert tyro.cli(Struct, args="--no-flg".split(" ")).flag is False + assert tyro.cli(Struct, args="-f".split(" ")).flag is True # BooleanOptionalAction will ignore arguments that aren't prefixed with --. with pytest.raises(SystemExit): diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index eb54d4a90..7c3ab1cc7 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -582,3 +582,55 @@ def test_functional_typeddict(): assert tyro.cli(NerfMLPHiddenLayers_2, args=["--hidden_layers.0", "3"]) == { "hidden_layers.0": 3 } + + +def test_functional_typeddict_with_default(): + """Source: https://github.com/brentyi/tyro/issues/87""" + NerfMLPHiddenLayers_0 = TypedDict( + "NerfMLPHiddenLayers0", + { + "hidden_layers.0": int, + "hidden_layers.1": int, + "hidden_layers.2": int, + "hidden_layers.3": int, + "hidden_layers.4": int, + "hidden_layers.5": int, + "hidden_layers.6": int, + "hidden_layers.7": int, + }, + ) + NerfMLPHiddenLayers_1 = TypedDict( + "NerfMLPHiddenLayers1", + { + "hidden_layers.0": NotRequired[int], + "hidden_layers.1": NotRequired[int], + "hidden_layers.2": NotRequired[int], + "hidden_layers.3": NotRequired[int], + "hidden_layers.4": NotRequired[int], + "hidden_layers.5": NotRequired[int], + "hidden_layers.6": NotRequired[int], + "hidden_layers.7": NotRequired[int], + }, + ) + NerfMLPHiddenLayers_2 = TypedDict( + "NerfMLPHiddenLayers2", + { + "hidden_layers.0": int, + "hidden_layers.1": int, + "hidden_layers.2": int, + "hidden_layers.3": int, + "hidden_layers.4": int, + "hidden_layers.5": int, + "hidden_layers.6": int, + "hidden_layers.7": int, + }, + total=False, + ) + with pytest.raises(SystemExit): + tyro.cli(NerfMLPHiddenLayers_0, args=["--hidden_layers.0", "3"], default={}) + assert tyro.cli( + NerfMLPHiddenLayers_1, args=["--hidden_layers.0", "3"], default={} + ) == {"hidden_layers.0": 3} + assert tyro.cli( + NerfMLPHiddenLayers_2, args=["--hidden_layers.0", "3"], default={} + ) == {"hidden_layers.0": 3} diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 4a6cd8d72..095c34cdd 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -239,7 +239,7 @@ def _rule_handle_defaults( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: - """Set `required=True` if a default value is set.""" + """Set `required=False` if a default value is set.""" # Mark lowered as required if a default is set. if ( diff --git a/tyro/_calling.py b/tyro/_calling.py index c82b1e2b4..40aa67043 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -59,7 +59,6 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: _strings.make_field_name([arg.dest_prefix, arg.field.name]) ] = arg - optional_group = any([_markers._OPTIONAL_GROUP in f.markers for f in field_list]) any_arguments_provided = False for field in field_list: diff --git a/tyro/_fields.py b/tyro/_fields.py index 5991ffdb7..ece54391e 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -378,13 +378,14 @@ def _field_list_from_typeddict( default_instance not in MISSING_SINGLETONS and default_instance is not EXCLUDE_FROM_CALL ) + assert not valid_default_instance or isinstance(default_instance, dict) total = getattr(cls, "__total__", True) assert isinstance(total, bool) assert not valid_default_instance or isinstance(default_instance, dict) for name, typ in get_type_hints(cls, include_extras=True).items(): typ_origin = get_origin(typ) - if valid_default_instance: - default = default_instance.get(name, MISSING_PROP) # type: ignore + if valid_default_instance and name in cast(dict, default_instance): + default = cast(dict, default_instance)[name] elif typ_origin is Required and total is False: # Support total=False. default = MISSING_PROP From 929c32a601d19730bfd862628f4559105cc2fa20 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 18 Nov 2023 03:09:17 -0800 Subject: [PATCH 341/491] Add warning for default values with unexpected types (#88) --- pyproject.toml | 2 +- tests/test_functools.py | 8 ++++---- tests/test_partial.py | 4 ++-- tyro/_fields.py | 43 ++++++++++++++++++++++++++++++++++++++--- 4 files changed, 47 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c1ff0f07c..152f96f52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.15" +version = "0.5.16" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/tests/test_functools.py b/tests/test_functools.py index 684534096..850c73032 100644 --- a/tests/test_functools.py +++ b/tests/test_functools.py @@ -42,7 +42,7 @@ class Main: def __init__(self, a: int, b: str) -> None: self.inner = b * a - helptext = get_helptext(functools.partial(Main, b=3)) + helptext = get_helptext(functools.partial(Main, b="3")) assert "partial" not in helptext assert "Hello!" in helptext @@ -76,7 +76,7 @@ def wrapper(*args, **kwargs) -> int: assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == 3 - helptext = get_helptext(functools.partial(wrapper, b=3)) + helptext = get_helptext(functools.partial(wrapper, b="3")) assert "wraps" not in helptext assert "Hello!" in helptext assert "Argument." in helptext @@ -95,7 +95,7 @@ def wrapper(*args, **kwargs) -> int: assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == 3 - helptext = get_helptext(functools.partial(wrapper, b=3)) + helptext = get_helptext(functools.partial(wrapper, b="3")) assert "wraps" not in helptext assert "Hello!" in helptext @@ -127,7 +127,7 @@ def wrapper(*args, **kwargs) -> str: == "hellohellohello" ) - helptext = get_helptext(functools.partial(wrapper, b=3)) + helptext = get_helptext(functools.partial(wrapper, b="3")) assert "wraps" not in helptext assert "Hello!" in helptext assert "Second field." in helptext diff --git a/tests/test_partial.py b/tests/test_partial.py index e1d63aed8..86a017451 100644 --- a/tests/test_partial.py +++ b/tests/test_partial.py @@ -28,7 +28,7 @@ def main(a: int, b: str) -> str: """Hello!""" return b * a - helptext = get_helptext(functools.partial(main, b=3)) + helptext = get_helptext(functools.partial(main, b="3")) assert "partial" not in helptext assert "Hello!" in helptext @@ -40,7 +40,7 @@ class Main: def __init__(self, a: int, b: str) -> None: self.inner = b * a - helptext = get_helptext(functools.partial(Main, b=3)) + helptext = get_helptext(functools.partial(Main, b="3")) assert "partial" not in helptext assert "Hello!" in helptext diff --git a/tyro/_fields.py b/tyro/_fields.py index ece54391e..1b520b40a 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -9,6 +9,7 @@ import functools import inspect import itertools +import numbers import os import sys import typing @@ -56,9 +57,12 @@ class FieldDefinition: name: str type_or_callable: Union[TypeForm[Any], Callable] + """Type or callable for this field. This should have all Annotated[] annotations + stripped.""" default: Any helptext: Optional[str] markers: FrozenSet[_markers._Marker] + custom_constructor: bool argconf: _confstruct._ArgConfiguration @@ -119,9 +123,12 @@ def make( else argconf.constructor_factory(), default, helptext, - frozenset(inferred_markers).union(markers), - argconf, - call_argname_override if call_argname_override is not None else name, + markers=frozenset(inferred_markers).union(markers), + custom_constructor=argconf.constructor_factory is not None, + argconf=argconf, + call_argname=call_argname_override + if call_argname_override is not None + else name, ) def add_markers(self, markers: Tuple[Any, ...]) -> FieldDefinition: @@ -202,6 +209,11 @@ class NotRequiredButWeDontKnowTheValueType(_singleton.Singleton): except ImportError: pass +DEFAULT_SENTINEL_SINGLETONS = MISSING_SINGLETONS + [ + NOT_REQUIRED_BUT_WE_DONT_KNOW_THE_VALUE, + EXCLUDE_FROM_CALL, +] + @dataclasses.dataclass(frozen=True) class UnsupportedNestedTypeMessage: @@ -261,6 +273,31 @@ def resolve(field: FieldDefinition) -> FieldDefinition: typ = _resolver.type_from_typevar_constraints(typ) typ = _resolver.narrow_container_types(typ, field.default) typ = _resolver.narrow_union_type(typ, field.default) + + # Check that the default value matches the final resolved type. + try: + if ( + not isinstance(field.default, typ) # type: ignore + # If a custom constructor is set, field.type_or_callable may not be + # matched to the annotated type. + and not field.custom_constructor + and field.default not in DEFAULT_SENTINEL_SINGLETONS + # The numeric tower in Python is wacky. This logic is non-critical, so + # we'll just skip it (+the complexity) for numbers. + and not isinstance(field.default, numbers.Number) + ): + # If the default value doesn't match the resolved type, we expand the + # type. This is inspired by https://github.com/brentyi/tyro/issues/88. + warnings.warn( + f"The field {field.name} is annotated with type {field.type_or_callable}, " + f"but the default value {field.default} has type {type(field.default)}. " + f"We'll try to handle this gracefully, but it may cause unexpected behavior." + ) + typ = Union[typ, type(field.default)] # type: ignore + except TypeError: + # An isinstance() check wasn't possible. + pass + field = dataclasses.replace(field, type_or_callable=typ) return field From afe73d6fdffbd5fcc3c4799f219db283216c8259 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 18 Nov 2023 03:18:11 -0800 Subject: [PATCH 342/491] Fix *args corner case, closes #84 --- tests/test_dcargs.py | 21 +++++++++++++++++++++ tyro/_calling.py | 4 ++++ 2 files changed, 25 insertions(+) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index e82845839..bb200374a 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -680,6 +680,27 @@ def main(*args: int, **kwargs: float) -> Tuple[Tuple[int, ...], Dict[str, float] ) == ((1, 2, 3), {"learning_rate": 1e-4, "beta1": 0.99}) +def test_unpack_with_other_args() -> None: + def main( + other1: str, other2: str = "3", *args: int, **kwargs: float + ) -> Tuple[str, str, Tuple[int, ...], Dict[str, float]]: + return other1, other2, args, kwargs + + assert tyro.cli(main, args=["--other1", "hi"]) == ("hi", "3", (), {}) + assert tyro.cli(main, args="--other1 ok --args --kwargs".split(" ")) == ( + "ok", + "3", + (), + {}, + ) + assert tyro.cli( + main, + args="--args 1 2 3 --kwargs learning_rate 1e-4 beta1 0.99 --other1 hello".split( + " " + ), + ) == ("hello", "3", (1, 2, 3), {"learning_rate": 1e-4, "beta1": 0.99}) + + def test_empty_container() -> None: @dataclasses.dataclass class A: diff --git a/tyro/_calling.py b/tyro/_calling.py index 40aa67043..c9fd178e9 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -177,12 +177,16 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: continue if _markers._UnpackArgsCall in field.markers: + if len(positional_args) == 0 and len(kwargs) > 0: + positional_args.extend(kwargs.values()) + kwargs.clear() assert isinstance(value, tuple) positional_args.extend(value) elif _markers._UnpackKwargsCall in field.markers: assert isinstance(value, dict) kwargs.update(value) elif field.is_positional_call(): + assert len(kwargs) == 0 positional_args.append(value) else: kwargs[field.call_argname] = value From a58227079d2c8f76f1b64460f29f7bd23180a811 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 18 Nov 2023 03:22:39 -0800 Subject: [PATCH 343/491] Add test for #88 --- tests/test_errors.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_errors.py b/tests/test_errors.py index 8a5ace5bb..496ff1029 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -494,3 +494,17 @@ def main2() -> None: tyro.extras.subcommand_cli_from_dict( {"main1": main1, "main2": main2}, args=["main1"] ) + + +def test_wrong_annotation() -> None: + @dataclasses.dataclass + class Args: + x: dict = None # type: ignore + + with pytest.warns(UserWarning): + assert tyro.cli(Args, args=[]).x is None + with pytest.warns(UserWarning): + assert tyro.cli(Args, args="--x key value k v".split(" ")).x == { + "key": "value", + "k": "v", + } From 01e9a3ce9b60f1ce1e0ee2231eaac23bd89e21eb Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 18 Nov 2023 03:47:51 -0800 Subject: [PATCH 344/491] Eliminate unnecessary warnings --- pyproject.toml | 2 +- tyro/_fields.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 152f96f52..883ae11e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.16" +version = "0.5.17" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/tyro/_fields.py b/tyro/_fields.py index 1b520b40a..92eb1b4b7 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -277,7 +277,8 @@ def resolve(field: FieldDefinition) -> FieldDefinition: # Check that the default value matches the final resolved type. try: if ( - not isinstance(field.default, typ) # type: ignore + type(typ) is type + and not isinstance(field.default, typ) # type: ignore # If a custom constructor is set, field.type_or_callable may not be # matched to the annotated type. and not field.custom_constructor From 2e79cb9bca7762aa73a3540f8133a50268b74a1e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 19 Nov 2023 03:19:06 -0800 Subject: [PATCH 345/491] Housekeeping: coverage, remove dead branches, fix minor edge cases --- .github/workflows/mypy.yml | 4 +- pyproject.toml | 2 +- tests/test_collections.py | 22 ++++ tests/test_conf.py | 2 + tests/test_errors.py | 45 +++++++- tests/test_nested.py | 24 +++- tests/test_new_style_annotations_min_py39.py | 10 +- .../test_collections_generated.py | 22 ++++ .../test_conf_generated.py | 19 ++++ .../test_dcargs_generated.py | 25 ++++- .../test_dict_namedtuple_generated.py | 104 ++++++++++++++++++ .../test_errors_generated.py | 83 +++++++++++++- .../test_functools_generated.py | 8 +- .../test_nested_generated.py | 24 +++- ...ew_style_annotations_min_py39_generated.py | 20 +++- .../test_partial_generated.py | 4 +- tyro/_calling.py | 2 +- tyro/_fields.py | 73 ++++++------ tyro/_instantiators.py | 9 +- tyro/_parsers.py | 23 ++-- tyro/_resolver.py | 10 +- tyro/_subcommand_matching.py | 5 +- 22 files changed, 449 insertions(+), 91 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index b8efb59d5..125eb9529 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: "Set up Python 3.8" + - name: "Set up Python 3.10" uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.10" - name: Install dependencies run: | pip install --upgrade pip diff --git a/pyproject.toml b/pyproject.toml index 883ae11e3..f43387e1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ tyro = ["py.typed"] profile = "black" [tool.mypy] -python_version = "3.8" +python_version = "3.10" ignore_missing_imports = true warn_unused_configs = true exclude = "^tests/test_py311_generated/.*" diff --git a/tests/test_collections.py b/tests/test_collections.py index ca689b4ee..4f2d245a9 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -460,8 +460,30 @@ def main(x: tuple = (0, 1, 2, "hello")) -> Any: assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == (0, 1, 2, "3") +def test_tuple_narrowing_empty_default() -> None: + def main(x: tuple = ()) -> Any: + return x + + assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == ("0", "1", "2", "3") + + def test_no_type_collections(): assert tyro.cli(dict, args="a b c d".split(" ")) == {"a": "b", "c": "d"} assert tyro.cli(list, args="a b c d".split(" ")) == ["a", "b", "c", "d"] assert tyro.cli(tuple, args="a b c d".split(" ")) == ("a", "b", "c", "d") assert tyro.cli(set, args="a b c d".split(" ")) == {"a", "b", "c", "d"} + + +def test_list_narrowing_alt() -> None: + def main(x: list = [1, "1"]) -> list: + return x + + assert tyro.cli(main, args="--x 3 four 5".split(" ")) == [3, "four", 5] + + +def test_list_narrowing_direct() -> None: + assert tyro.cli(list, default=[1, "2"], args="3 four 5".split(" ")) == [ + 3, + "four", + 5, + ] diff --git a/tests/test_conf.py b/tests/test_conf.py index 12a790d62..54d3f4f80 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -723,6 +723,7 @@ def test_append_lists() -> None: class A: x: tyro.conf.UseAppendAction[List[int]] + assert tyro.cli(A, args=[]) == A(x=[]) assert tyro.cli(A, args="--x 1 --x 2 --x 3".split(" ")) == A(x=[1, 2, 3]) assert tyro.cli(A, args=[]) == A(x=[]) with pytest.raises(SystemExit): @@ -736,6 +737,7 @@ def test_append_tuple() -> None: class A: x: tyro.conf.UseAppendAction[Tuple[int, ...]] + assert tyro.cli(A, args=[]) == A(x=()) assert tyro.cli(A, args="--x 1 --x 2 --x 3".split(" ")) == A(x=(1, 2, 3)) assert tyro.cli(A, args=[]) == A(x=()) with pytest.raises(SystemExit): diff --git a/tests/test_errors.py b/tests/test_errors.py index 496ff1029..e9cb504be 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -1,9 +1,10 @@ import contextlib import dataclasses import io -from typing import List, Tuple, TypeVar, Union +from typing import Dict, List, Tuple, TypeVar, Union import pytest +from typing_extensions import Literal import tyro @@ -508,3 +509,45 @@ class Args: "key": "value", "k": "v", } + + +def test_bad_dict_control() -> None: + assert tyro.cli( + Dict[Literal["a", "b"], Literal["c", "d"]], args="a c b d".split(" ") + ) == {"a": "c", "b": "d"} + + +def test_bad_dict_key() -> None: + with pytest.raises(SystemExit): + tyro.cli(Dict[Literal["a", "b"], Literal["c", "d"]], args="c c d d".split(" ")) + + +def test_bad_dict_val() -> None: + assert tyro.cli( + Dict[Literal["a", "b"], Literal["c", "d"]], args="a c b d".split(" ") + ) == {"a": "c", "b": "d"} + with pytest.raises(SystemExit): + tyro.cli(Dict[Literal["a", "b"], Literal["c", "d"]], args="a a b b".split(" ")) + + +def test_invalid_subcommand_default() -> None: + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + + @dataclasses.dataclass + class C: + c: int + + assert tyro.cli(Union[A, B], args="a --a 5".split(" ")) == A(5) # type: ignore + assert tyro.cli(Union[A, B], args="b --b 5".split(" ")) == B(5) # type: ignore + + with pytest.warns(UserWarning): + assert tyro.cli(Union[A, B], default=C(3), args=[]) == C(3) # type: ignore + + with pytest.warns(UserWarning): + assert tyro.cli(Union[A, B], default=C(3), args="b --b 5".split(" ")) == B(5) # type: ignore diff --git a/tests/test_nested.py b/tests/test_nested.py index 591a1c344..e05a59b12 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -3,6 +3,7 @@ import pytest from frozendict import frozendict # type: ignore +from helptext_utils import get_helptext from typing_extensions import Annotated, Literal import tyro @@ -406,10 +407,8 @@ class DefaultSubparser: # Tolerate bad static types: https://github.com/brentyi/tyro/issues/20 # Should give us a bunch of warnings! with pytest.warns(UserWarning): - assert tyro.cli( - DefaultSubparser, args=["--x", "1", "--bc", "3"] - ) == DefaultSubparser( - 1, 3 # type: ignore + assert tyro.cli(DefaultSubparser, args=["--x", "1"]) == DefaultSubparser( + 1, 5 # type: ignore ) @@ -428,7 +427,7 @@ class C: c: int with pytest.warns(UserWarning): - assert tyro.cli(Union[A, B], default=C(3), args=["--c", "2"]) == C(2) # type: ignore + assert tyro.cli(Union[A, B], default=C(3), args=["c", "--c", "2"]) == C(2) # type: ignore def test_optional_subparser() -> None: @@ -1052,3 +1051,18 @@ def commit(message: str, all: bool = False) -> Tuple[str, bool]: }, args="commit --message hello --all".split(" "), ) == ("hello", True) + + +def test_subcommand_by_type_tree() -> None: + @dataclasses.dataclass(frozen=True) + class A: + a: Union[int, str] + + @dataclasses.dataclass + class Args: + inner: Union[ + Annotated[A, tyro.conf.subcommand(name="alt", default=A(5))], A + ] = A("hello") + + assert tyro.cli(Args, args=[]) == Args(A("hello")) + assert "default: inner:alt" in get_helptext(Args) diff --git a/tests/test_new_style_annotations_min_py39.py b/tests/test_new_style_annotations_min_py39.py index 22cd9160e..9d04bb029 100644 --- a/tests/test_new_style_annotations_min_py39.py +++ b/tests/test_new_style_annotations_min_py39.py @@ -6,21 +6,21 @@ import tyro -def test_list(): +def test_list() -> None: def main(x: list[bool]) -> Any: return x assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] -def test_tuple(): +def test_tuple() -> None: def main(x: tuple[bool, str]) -> Any: return x assert tyro.cli(main, args=["--x", "True", "False"]) == (True, "False") -def test_tuple_nested(): +def test_tuple_nested() -> None: @dataclasses.dataclass class Args: a: int @@ -31,14 +31,14 @@ def main(x: tuple[Args, Args]) -> Any: assert tyro.cli(main, args=["--x.0.a", "3", "--x.1.a", "4"]) == (Args(3), Args(4)) -def test_tuple_variable(): +def test_tuple_variable() -> None: def main(x: tuple[Union[bool, str], ...]) -> Any: return x assert tyro.cli(main, args=["--x", "True", "Wrong"]) == (True, "Wrong") -def test_super_nested(): +def test_super_nested() -> None: def main( x: Optional[ list[ diff --git a/tests/test_py311_generated/test_collections_generated.py b/tests/test_py311_generated/test_collections_generated.py index 4b8dd3d61..47b3286b3 100644 --- a/tests/test_py311_generated/test_collections_generated.py +++ b/tests/test_py311_generated/test_collections_generated.py @@ -460,8 +460,30 @@ def main(x: tuple = (0, 1, 2, "hello")) -> Any: assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == (0, 1, 2, "3") +def test_tuple_narrowing_empty_default() -> None: + def main(x: tuple = ()) -> Any: + return x + + assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == ("0", "1", "2", "3") + + def test_no_type_collections(): assert tyro.cli(dict, args="a b c d".split(" ")) == {"a": "b", "c": "d"} assert tyro.cli(list, args="a b c d".split(" ")) == ["a", "b", "c", "d"] assert tyro.cli(tuple, args="a b c d".split(" ")) == ("a", "b", "c", "d") assert tyro.cli(set, args="a b c d".split(" ")) == {"a", "b", "c", "d"} + + +def test_list_narrowing_alt() -> None: + def main(x: list = [1, "1"]) -> list: + return x + + assert tyro.cli(main, args="--x 3 four 5".split(" ")) == [3, "four", 5] + + +def test_list_narrowing_direct() -> None: + assert tyro.cli(list, default=[1, "2"], args="3 four 5".split(" ")) == [ + 3, + "four", + 5, + ] diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index 1e7bc887d..cdfb903f4 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -722,6 +722,7 @@ def test_append_lists() -> None: class A: x: tyro.conf.UseAppendAction[List[int]] + assert tyro.cli(A, args=[]) == A(x=[]) assert tyro.cli(A, args="--x 1 --x 2 --x 3".split(" ")) == A(x=[1, 2, 3]) assert tyro.cli(A, args=[]) == A(x=[]) with pytest.raises(SystemExit): @@ -735,6 +736,7 @@ def test_append_tuple() -> None: class A: x: tyro.conf.UseAppendAction[Tuple[int, ...]] + assert tyro.cli(A, args=[]) == A(x=()) assert tyro.cli(A, args="--x 1 --x 2 --x 3".split(" ")) == A(x=(1, 2, 3)) assert tyro.cli(A, args=[]) == A(x=()) with pytest.raises(SystemExit): @@ -1168,3 +1170,20 @@ class Config: assert tyro.cli( Config, args="--x.struct.b 2 --x.struct.c 3 --all 5".split(" ") ) == Config(x=30) + + +def test_flag_alias() -> None: + @dataclasses.dataclass + class Struct: + flag: Annotated[bool, tyro.conf.arg(aliases=["-f", "--flg"])] = False + + assert tyro.cli(Struct, args=[]).flag is False + assert tyro.cli(Struct, args="--flag".split(" ")).flag is True + assert tyro.cli(Struct, args="--no-flag".split(" ")).flag is False + assert tyro.cli(Struct, args="--flg".split(" ")).flag is True + assert tyro.cli(Struct, args="--no-flg".split(" ")).flag is False + assert tyro.cli(Struct, args="-f".split(" ")).flag is True + + # BooleanOptionalAction will ignore arguments that aren't prefixed with --. + with pytest.raises(SystemExit): + tyro.cli(Struct, args="-no-f".split(" ")) diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py index b03e7e43a..145d5eae9 100644 --- a/tests/test_py311_generated/test_dcargs_generated.py +++ b/tests/test_py311_generated/test_dcargs_generated.py @@ -672,15 +672,38 @@ def main(x: os.PathLike) -> os.PathLike: assert tyro.cli(main, args=["--x", "/dev/null"]) == pathlib.Path("/dev/null") -def test_variadics() -> None: +def test_unpack() -> None: def main(*args: int, **kwargs: float) -> Tuple[Tuple[int, ...], Dict[str, float]]: return args, kwargs + assert tyro.cli(main, args=[]) == ((), {}) + assert tyro.cli(main, args="--args --kwargs".split(" ")) == ((), {}) assert tyro.cli( main, args="--args 1 2 3 --kwargs learning_rate 1e-4 beta1 0.99".split(" ") ) == ((1, 2, 3), {"learning_rate": 1e-4, "beta1": 0.99}) +def test_unpack_with_other_args() -> None: + def main( + other1: str, other2: str = "3", *args: int, **kwargs: float + ) -> Tuple[str, str, Tuple[int, ...], Dict[str, float]]: + return other1, other2, args, kwargs + + assert tyro.cli(main, args=["--other1", "hi"]) == ("hi", "3", (), {}) + assert tyro.cli(main, args="--other1 ok --args --kwargs".split(" ")) == ( + "ok", + "3", + (), + {}, + ) + assert tyro.cli( + main, + args="--args 1 2 3 --kwargs learning_rate 1e-4 beta1 0.99 --other1 hello".split( + " " + ), + ) == ("hello", "3", (1, 2, 3), {"learning_rate": 1e-4, "beta1": 0.99}) + + def test_empty_container() -> None: @dataclasses.dataclass class A: diff --git a/tests/test_py311_generated/test_dict_namedtuple_generated.py b/tests/test_py311_generated/test_dict_namedtuple_generated.py index 5fcce0e06..e3b2e6326 100644 --- a/tests/test_py311_generated/test_dict_namedtuple_generated.py +++ b/tests/test_py311_generated/test_dict_namedtuple_generated.py @@ -541,3 +541,107 @@ def test_nested_dict_annotations() -> None: ) assert overrided_config["optimizer"]["scheduler"]["schedule-type"] == "exponential" del overrided_config + + +def test_functional_typeddict(): + """Source: https://github.com/brentyi/tyro/issues/87""" + NerfMLPHiddenLayers_0 = TypedDict( + "NerfMLPHiddenLayers0", + { + "hidden_layers.0": int, + "hidden_layers.1": int, + "hidden_layers.2": int, + "hidden_layers.3": int, + "hidden_layers.4": int, + "hidden_layers.5": int, + "hidden_layers.6": int, + "hidden_layers.7": int, + }, + ) + NerfMLPHiddenLayers_1 = TypedDict( + "NerfMLPHiddenLayers1", + { + "hidden_layers.0": NotRequired[int], + "hidden_layers.1": NotRequired[int], + "hidden_layers.2": NotRequired[int], + "hidden_layers.3": NotRequired[int], + "hidden_layers.4": NotRequired[int], + "hidden_layers.5": NotRequired[int], + "hidden_layers.6": NotRequired[int], + "hidden_layers.7": NotRequired[int], + }, + ) + NerfMLPHiddenLayers_2 = TypedDict( + "NerfMLPHiddenLayers2", + { + "hidden_layers.0": int, + "hidden_layers.1": int, + "hidden_layers.2": int, + "hidden_layers.3": int, + "hidden_layers.4": int, + "hidden_layers.5": int, + "hidden_layers.6": int, + "hidden_layers.7": int, + }, + total=False, + ) + with pytest.raises(SystemExit): + tyro.cli(NerfMLPHiddenLayers_0, args=["--hidden_layers.0", "3"]) + assert tyro.cli(NerfMLPHiddenLayers_1, args=["--hidden_layers.0", "3"]) == { + "hidden_layers.0": 3 + } + assert tyro.cli(NerfMLPHiddenLayers_2, args=["--hidden_layers.0", "3"]) == { + "hidden_layers.0": 3 + } + + +def test_functional_typeddict_with_default(): + """Source: https://github.com/brentyi/tyro/issues/87""" + NerfMLPHiddenLayers_0 = TypedDict( + "NerfMLPHiddenLayers0", + { + "hidden_layers.0": int, + "hidden_layers.1": int, + "hidden_layers.2": int, + "hidden_layers.3": int, + "hidden_layers.4": int, + "hidden_layers.5": int, + "hidden_layers.6": int, + "hidden_layers.7": int, + }, + ) + NerfMLPHiddenLayers_1 = TypedDict( + "NerfMLPHiddenLayers1", + { + "hidden_layers.0": NotRequired[int], + "hidden_layers.1": NotRequired[int], + "hidden_layers.2": NotRequired[int], + "hidden_layers.3": NotRequired[int], + "hidden_layers.4": NotRequired[int], + "hidden_layers.5": NotRequired[int], + "hidden_layers.6": NotRequired[int], + "hidden_layers.7": NotRequired[int], + }, + ) + NerfMLPHiddenLayers_2 = TypedDict( + "NerfMLPHiddenLayers2", + { + "hidden_layers.0": int, + "hidden_layers.1": int, + "hidden_layers.2": int, + "hidden_layers.3": int, + "hidden_layers.4": int, + "hidden_layers.5": int, + "hidden_layers.6": int, + "hidden_layers.7": int, + }, + total=False, + ) + with pytest.raises(SystemExit): + tyro.cli(NerfMLPHiddenLayers_0, args=["--hidden_layers.0", "3"], default={}) + assert tyro.cli( + NerfMLPHiddenLayers_1, args=["--hidden_layers.0", "3"], default={} + ) == {"hidden_layers.0": 3} + assert tyro.cli( + NerfMLPHiddenLayers_2, args=["--hidden_layers.0", "3"], default={} + ) == {"hidden_layers.0": 3} diff --git a/tests/test_py311_generated/test_errors_generated.py b/tests/test_py311_generated/test_errors_generated.py index 46c8c88de..e41a8cf7a 100644 --- a/tests/test_py311_generated/test_errors_generated.py +++ b/tests/test_py311_generated/test_errors_generated.py @@ -1,7 +1,7 @@ import contextlib import dataclasses import io -from typing import List, Tuple, TypeVar, Union +from typing import Dict, List, Literal, Tuple, TypeVar, Union import pytest @@ -469,3 +469,84 @@ class Args: # Printed in the similar argument list. assert error.count("--flag, --no-flag") == 1 + + +def test_value_error() -> None: + """https://github.com/brentyi/tyro/issues/86""" + + def main() -> None: + raise ValueError("This shouldn't be caught by tyro") + + with pytest.raises(ValueError): + tyro.cli(main, args=[]) + + +def test_value_error_subcommand() -> None: + """https://github.com/brentyi/tyro/issues/86""" + + def main1() -> None: + raise ValueError("This shouldn't be caught by tyro") + + def main2() -> None: + raise ValueError("This shouldn't be caught by tyro") + + with pytest.raises(ValueError): + tyro.extras.subcommand_cli_from_dict( + {"main1": main1, "main2": main2}, args=["main1"] + ) + + +def test_wrong_annotation() -> None: + @dataclasses.dataclass + class Args: + x: dict = None # type: ignore + + with pytest.warns(UserWarning): + assert tyro.cli(Args, args=[]).x is None + with pytest.warns(UserWarning): + assert tyro.cli(Args, args="--x key value k v".split(" ")).x == { + "key": "value", + "k": "v", + } + + +def test_bad_dict_control() -> None: + assert tyro.cli( + Dict[Literal["a", "b"], Literal["c", "d"]], args="a c b d".split(" ") + ) == {"a": "c", "b": "d"} + + +def test_bad_dict_key() -> None: + with pytest.raises(SystemExit): + tyro.cli(Dict[Literal["a", "b"], Literal["c", "d"]], args="c c d d".split(" ")) + + +def test_bad_dict_val() -> None: + assert tyro.cli( + Dict[Literal["a", "b"], Literal["c", "d"]], args="a c b d".split(" ") + ) == {"a": "c", "b": "d"} + with pytest.raises(SystemExit): + tyro.cli(Dict[Literal["a", "b"], Literal["c", "d"]], args="a a b b".split(" ")) + + +def test_invalid_subcommand_default() -> None: + @dataclasses.dataclass + class A: + a: int + + @dataclasses.dataclass + class B: + b: int + + @dataclasses.dataclass + class C: + c: int + + assert tyro.cli(Union[A, B], args="a --a 5".split(" ")) == A(5) # type: ignore + assert tyro.cli(Union[A, B], args="b --b 5".split(" ")) == B(5) # type: ignore + + with pytest.warns(UserWarning): + assert tyro.cli(Union[A, B], default=C(3), args=[]) == C(3) # type: ignore + + with pytest.warns(UserWarning): + assert tyro.cli(Union[A, B], default=C(3), args="b --b 5".split(" ")) == B(5) # type: ignore diff --git a/tests/test_py311_generated/test_functools_generated.py b/tests/test_py311_generated/test_functools_generated.py index 684534096..850c73032 100644 --- a/tests/test_py311_generated/test_functools_generated.py +++ b/tests/test_py311_generated/test_functools_generated.py @@ -42,7 +42,7 @@ class Main: def __init__(self, a: int, b: str) -> None: self.inner = b * a - helptext = get_helptext(functools.partial(Main, b=3)) + helptext = get_helptext(functools.partial(Main, b="3")) assert "partial" not in helptext assert "Hello!" in helptext @@ -76,7 +76,7 @@ def wrapper(*args, **kwargs) -> int: assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == 3 - helptext = get_helptext(functools.partial(wrapper, b=3)) + helptext = get_helptext(functools.partial(wrapper, b="3")) assert "wraps" not in helptext assert "Hello!" in helptext assert "Argument." in helptext @@ -95,7 +95,7 @@ def wrapper(*args, **kwargs) -> int: assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == 3 - helptext = get_helptext(functools.partial(wrapper, b=3)) + helptext = get_helptext(functools.partial(wrapper, b="3")) assert "wraps" not in helptext assert "Hello!" in helptext @@ -127,7 +127,7 @@ def wrapper(*args, **kwargs) -> str: == "hellohellohello" ) - helptext = get_helptext(functools.partial(wrapper, b=3)) + helptext = get_helptext(functools.partial(wrapper, b="3")) assert "wraps" not in helptext assert "Hello!" in helptext assert "Second field." in helptext diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index c60ca3e98..44bd668f6 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -13,6 +13,7 @@ import pytest from frozendict import frozendict # type: ignore +from helptext_utils import get_helptext import tyro @@ -415,10 +416,8 @@ class DefaultSubparser: # Tolerate bad static types: https://github.com/brentyi/tyro/issues/20 # Should give us a bunch of warnings! with pytest.warns(UserWarning): - assert tyro.cli( - DefaultSubparser, args=["--x", "1", "--bc", "3"] - ) == DefaultSubparser( - 1, 3 # type: ignore + assert tyro.cli(DefaultSubparser, args=["--x", "1"]) == DefaultSubparser( + 1, 5 # type: ignore ) @@ -437,7 +436,7 @@ class C: c: int with pytest.warns(UserWarning): - assert tyro.cli(Union[A, B], default=C(3), args=["--c", "2"]) == C(2) # type: ignore + assert tyro.cli(Union[A, B], default=C(3), args=["c", "--c", "2"]) == C(2) # type: ignore def test_optional_subparser() -> None: @@ -1061,3 +1060,18 @@ def commit(message: str, all: bool = False) -> Tuple[str, bool]: }, args="commit --message hello --all".split(" "), ) == ("hello", True) + + +def test_subcommand_by_type_tree() -> None: + @dataclasses.dataclass(frozen=True) + class A: + a: Union[int, str] + + @dataclasses.dataclass + class Args: + inner: Union[ + Annotated[A, tyro.conf.subcommand(name="alt", default=A(5))], A + ] = A("hello") + + assert tyro.cli(Args, args=[]) == Args(A("hello")) + assert "default: inner:alt" in get_helptext(Args) diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py index 5ef86858c..9d04bb029 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py @@ -1,3 +1,4 @@ +import dataclasses from typing import Any, Literal, Optional, Union import pytest @@ -5,28 +6,39 @@ import tyro -def test_list(): +def test_list() -> None: def main(x: list[bool]) -> Any: return x assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] -def test_tuple(): +def test_tuple() -> None: def main(x: tuple[bool, str]) -> Any: return x assert tyro.cli(main, args=["--x", "True", "False"]) == (True, "False") -def test_tuple_variable(): +def test_tuple_nested() -> None: + @dataclasses.dataclass + class Args: + a: int + + def main(x: tuple[Args, Args]) -> Any: + return x + + assert tyro.cli(main, args=["--x.0.a", "3", "--x.1.a", "4"]) == (Args(3), Args(4)) + + +def test_tuple_variable() -> None: def main(x: tuple[Union[bool, str], ...]) -> Any: return x assert tyro.cli(main, args=["--x", "True", "Wrong"]) == (True, "Wrong") -def test_super_nested(): +def test_super_nested() -> None: def main( x: Optional[ list[ diff --git a/tests/test_py311_generated/test_partial_generated.py b/tests/test_py311_generated/test_partial_generated.py index e1d63aed8..86a017451 100644 --- a/tests/test_py311_generated/test_partial_generated.py +++ b/tests/test_py311_generated/test_partial_generated.py @@ -28,7 +28,7 @@ def main(a: int, b: str) -> str: """Hello!""" return b * a - helptext = get_helptext(functools.partial(main, b=3)) + helptext = get_helptext(functools.partial(main, b="3")) assert "partial" not in helptext assert "Hello!" in helptext @@ -40,7 +40,7 @@ class Main: def __init__(self, a: int, b: str) -> None: self.inner = b * a - helptext = get_helptext(functools.partial(Main, b=3)) + helptext = get_helptext(functools.partial(Main, b="3")) assert "partial" not in helptext assert "Hello!" in helptext diff --git a/tyro/_calling.py b/tyro/_calling.py index c9fd178e9..ec4a18986 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -218,7 +218,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: # `tuple`. unwrapped_f = f unwrapped_f = _resolver.unwrap_origin_strip_extras(unwrapped_f) - unwrapped_f = _resolver.narrow_type(unwrapped_f, default_instance) + unwrapped_f = _resolver.narrow_subtypes(unwrapped_f, default_instance) unwrapped_f = _resolver.unwrap_origin_strip_extras(unwrapped_f) unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore diff --git a/tyro/_fields.py b/tyro/_fields.py index 92eb1b4b7..e4f14ccbc 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -254,7 +254,7 @@ def field_list_from_callable( """ # Resolve generic types. f, type_from_typevar = _resolver.resolve_generic_types(f) - f = _resolver.narrow_type(f, default_instance) + f = _resolver.narrow_subtypes(f, default_instance) # Try to generate field list. field_list = _try_field_list_from_callable(f, default_instance) @@ -271,33 +271,34 @@ def resolve(field: FieldDefinition) -> FieldDefinition: typ = field.type_or_callable typ = _resolver.apply_type_from_typevar(typ, type_from_typevar) typ = _resolver.type_from_typevar_constraints(typ) - typ = _resolver.narrow_container_types(typ, field.default) + typ = _resolver.narrow_collection_types(typ, field.default) typ = _resolver.narrow_union_type(typ, field.default) # Check that the default value matches the final resolved type. - try: - if ( - type(typ) is type - and not isinstance(field.default, typ) # type: ignore - # If a custom constructor is set, field.type_or_callable may not be - # matched to the annotated type. - and not field.custom_constructor - and field.default not in DEFAULT_SENTINEL_SINGLETONS - # The numeric tower in Python is wacky. This logic is non-critical, so - # we'll just skip it (+the complexity) for numbers. - and not isinstance(field.default, numbers.Number) - ): - # If the default value doesn't match the resolved type, we expand the - # type. This is inspired by https://github.com/brentyi/tyro/issues/88. - warnings.warn( - f"The field {field.name} is annotated with type {field.type_or_callable}, " - f"but the default value {field.default} has type {type(field.default)}. " - f"We'll try to handle this gracefully, but it may cause unexpected behavior." - ) - typ = Union[typ, type(field.default)] # type: ignore - except TypeError: - # An isinstance() check wasn't possible. - pass + # There's some similar Union-specific logic for this in narrow_union_type(). We + # may be able to consolidate this. + if ( + # Be relatively conservative: isinstance() can be checked on non-type + # types (like unions in Python >=3.10), but we'll only consider single types + # for now. + type(typ) is type + and not isinstance(field.default, typ) # type: ignore + # If a custom constructor is set, field.type_or_callable may not be + # matched to the annotated type. + and not field.custom_constructor + and field.default not in DEFAULT_SENTINEL_SINGLETONS + # The numeric tower in Python is wacky. This logic is non-critical, so + # we'll just skip it (+the complexity) for numbers. + and not isinstance(field.default, numbers.Number) + ): + # If the default value doesn't match the resolved type, we expand the + # type. This is inspired by https://github.com/brentyi/tyro/issues/88. + warnings.warn( + f"The field {field.name} is annotated with type {field.type_or_callable}, " + f"but the default value {field.default} has type {type(field.default)}. " + f"We'll try to handle this gracefully, but it may cause unexpected behavior." + ) + typ = Union[typ, type(field.default)] # type: ignore field = dataclasses.replace(field, type_or_callable=typ) return field @@ -339,7 +340,8 @@ def _try_field_list_from_callable( # Unwrap generics. f, _ = _resolver.resolve_generic_types(f) - f = _resolver.narrow_type(f, default_instance) + f = _resolver.narrow_subtypes(f, default_instance) + f = _resolver.narrow_collection_types(f, default_instance) f_origin = _resolver.unwrap_origin_strip_extras(cast(TypeForm, f)) # If `f` is a type: @@ -389,7 +391,7 @@ def _try_field_list_from_callable( set, typing.Sequence, ): - return _field_list_from_sequence_checked(f, default_instance) + return _field_list_from_nontuple_sequence_checked(f, default_instance) # General cases. if ( @@ -705,18 +707,19 @@ def _field_list_from_tuple( return field_list -def _field_list_from_sequence_checked( +def _field_list_from_nontuple_sequence_checked( f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: + """Tuples are handled differently due to variadics (tuple[T1, T2, T3, ..., Tn]), + while list[], sequence[], set[], etc only have one typevar.""" contained_type: Any if len(get_args(f)) == 0: - if default_instance in MISSING_SINGLETONS: - return UnsupportedNestedTypeMessage( - f"Sequence type {f} needs either an explicit type or a" - " default to infer from." - ) - assert isinstance(default_instance, Iterable) - contained_type = next(iter(default_instance)) + # A raw collection type (list, tuple, etc) was passed in, but narrowing also failed. + assert default_instance in MISSING_SINGLETONS, f"{default_instance} {f}" + return UnsupportedNestedTypeMessage( + f"Sequence type {f} needs either an explicit type or a" + " default to infer from." + ) else: (contained_type,) = get_args(f) return _try_field_list_from_sequence_inner(contained_type, default_instance) diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 2525b098e..d1525fd93 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -609,6 +609,7 @@ def _instantiator_from_sequence( ) -> Tuple[Instantiator, InstantiatorMetadata]: """Instantiator for variable-length sequences: list, sets, Tuple[T, ...], etc.""" container_type = get_origin(typ) + assert container_type is not None if container_type is collections.abc.Sequence: container_type = list @@ -626,11 +627,9 @@ def _instantiator_from_sequence( markers=markers - {_markers.UseAppendAction}, ) - def append_sequence_instantiator(strings: Optional[List[List[str]]]) -> Any: - if strings is None: - assert container_type is not None - return container_type() - return container_type(make(s) for s in strings) # type: ignore + def append_sequence_instantiator(strings: List[List[str]]) -> Any: + assert strings is not None + return container_type(cast(_StandardInstantiator, make)(s) for s in strings) return append_sequence_instantiator, InstantiatorMetadata( nargs=inner_meta.nargs, diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 4344338cb..f7edb6f86 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -4,7 +4,6 @@ import argparse import dataclasses -import warnings from typing import ( Any, Callable, @@ -312,7 +311,7 @@ def handle_field( if _fields.is_nested_type(field.type_or_callable, field.default): field = dataclasses.replace( field, - type_or_callable=_resolver.narrow_type( + type_or_callable=_resolver.narrow_subtypes( field.type_or_callable, field.default, ), @@ -433,17 +432,15 @@ def from_field( default_name = _subcommand_matching.match_subcommand( field.default, subcommand_config_from_name, subcommand_type_from_name ) - if default_name is None: - # This should really be an error, but we can raise a warning to make - # hacking at subcommands easier: - # https://github.com/brentyi/tyro/issues/20 - warnings.warn( - f"`{prefix}` was provided a default value of type" - f" {type(field.default)} but no matching subcommand was found. A" - " type may be missing in the Union type declaration for" - f" `{prefix}`, which currently expects {options}." - ) - return None + + # This should never be triggered because union types are expanded when + # a bad default value is provided. + assert default_name is not None, ( + f"`{prefix}` was provided a default value of type" + f" {type(field.default)} but no matching subcommand was found. A" + " type may be missing in the Union type declaration for" + f" `{prefix}`, which currently expects {options}." + ) # Add subcommands for each option. parser_from_name: Dict[str, ParserSpecification] = {} diff --git a/tyro/_resolver.py b/tyro/_resolver.py index a702ee8c6..d0d070749 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -131,7 +131,7 @@ def type_from_typevar_constraints(typ: TypeOrCallable) -> TypeOrCallable: @_unsafe_cache.unsafe_cache(maxsize=1024) -def narrow_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCallable: +def narrow_subtypes(typ: TypeOrCallable, default_instance: Any) -> TypeOrCallable: """Type narrowing: if we annotate as Animal but specify a default instance of Cat, we should parse as Cat. @@ -167,10 +167,12 @@ def narrow_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCallable: return typ -def narrow_container_types( +def narrow_collection_types( typ: TypeOrCallable, default_instance: Any ) -> TypeOrCallable: """TypeForm narrowing for containers. Infers types of container contents.""" + if hasattr(default_instance, "__len__") and len(default_instance) == 0: + return typ if typ is list and isinstance(default_instance, list): typ = List.__getitem__(Union.__getitem__(tuple(map(type, default_instance)))) # type: ignore elif typ is set and isinstance(default_instance, set): @@ -264,7 +266,7 @@ def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCalla -- For (A): - We raise a warning, then take the type of the default value. + We raise a warning, then add the type of the default value to the union. Loosely motivated by: https://github.com/brentyi/tyro/issues/20 -- @@ -312,7 +314,7 @@ def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCalla f"{type(default_instance)} does not match any type in Union:" f" {options_unwrapped}" ) - return type(default_instance) + return Union.__getitem__(options + (type(default_instance),)) # type: ignore except TypeError: pass diff --git a/tyro/_subcommand_matching.py b/tyro/_subcommand_matching.py index 9ccef6b5d..951524cd1 100644 --- a/tyro/_subcommand_matching.py +++ b/tyro/_subcommand_matching.py @@ -55,8 +55,9 @@ def match_subcommand( ): return subcommand_name - # Failed! - return None + # Failed. This should never happen, we'll raise an error outside of this function if + # this is the case. + return None # pragma: no cover @dataclasses.dataclass(frozen=True) From ef9297329433595c276f7fd9ce741c21405bfe7e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 23 Nov 2023 14:22:29 -0800 Subject: [PATCH 346/491] Fix type narrowing edge case for unions (#90) * Fix type narrowing edge case for unions * Link issue from test * Run test gen * Fix Python 3.7 + mypy * Tweak --- pyproject.toml | 2 +- tests/test_conf.py | 32 +++++++++++++++++++ .../test_conf_generated.py | 32 +++++++++++++++++++ tyro/_fields.py | 11 +++++++ 4 files changed, 76 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f43387e1e..b4c46cb2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.17" +version = "0.5.18" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/tests/test_conf.py b/tests/test_conf.py index 54d3f4f80..a820737a1 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1188,3 +1188,35 @@ class Struct: # BooleanOptionalAction will ignore arguments that aren't prefixed with --. with pytest.raises(SystemExit): tyro.cli(Struct, args="-no-f".split(" ")) + + +def test_subcommand_constructor_mix() -> None: + """https://github.com/brentyi/tyro/issues/89""" + + def checkout(branch: str) -> str: + """Check out a branch.""" + return branch + + def commit(message: str, all: bool = False) -> str: + """Make a commit.""" + return f"{message} {all}" + + @dataclasses.dataclass + class Arg: + foo: int = 1 + + t: Any = Union[ + Annotated[ + Any, + tyro.conf.subcommand(name="checkout", constructor=checkout), + ], + Annotated[ + Any, + tyro.conf.subcommand(name="commit", constructor=commit), + ], + Arg, + ] + + assert tyro.cli(t, args=["arg"]) == Arg() + assert tyro.cli(t, args=["checkout", "--branch", "main"]) == "main" + assert tyro.cli(t, args=["commit", "--message", "hi", "--all"]) == "hi True" diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index cdfb903f4..482f67ec1 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -1187,3 +1187,35 @@ class Struct: # BooleanOptionalAction will ignore arguments that aren't prefixed with --. with pytest.raises(SystemExit): tyro.cli(Struct, args="-no-f".split(" ")) + + +def test_subcommand_constructor_mix() -> None: + """https://github.com/brentyi/tyro/issues/89""" + + def checkout(branch: str) -> str: + """Check out a branch.""" + return branch + + def commit(message: str, all: bool = False) -> str: + """Make a commit.""" + return f"{message} {all}" + + @dataclasses.dataclass + class Arg: + foo: int = 1 + + t: Any = Union[ + Annotated[ + Any, + tyro.conf.subcommand(name="checkout", constructor=checkout), + ], + Annotated[ + Any, + tyro.conf.subcommand(name="commit", constructor=commit), + ], + Arg, + ] + + assert tyro.cli(t, args=["arg"]) == Arg() + assert tyro.cli(t, args=["checkout", "--branch", "main"]) == "main" + assert tyro.cli(t, args=["commit", "--message", "hi", "--all"]) == "hi True" diff --git a/tyro/_fields.py b/tyro/_fields.py index e4f14ccbc..d464fcbf6 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -232,6 +232,17 @@ def is_nested_type( TODO: we should come up with a better name than 'nested type', which is a little bit misleading.""" + + # Need to swap types. + _, argconfs = _resolver.unwrap_annotated( + typ, search_type=conf._confstruct._ArgConfiguration + ) + _, subcommand_confs = _resolver.unwrap_annotated( + typ, search_type=conf._confstruct._SubcommandConfiguration + ) + for union_conf in argconfs + subcommand_confs: + if union_conf.constructor_factory is not None: + typ = union_conf.constructor_factory() return not isinstance( _try_field_list_from_callable(typ, default_instance), UnsupportedNestedTypeMessage, From bf65c5684a343853dfda515363b975a28717e04a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 24 Nov 2023 16:14:42 -0800 Subject: [PATCH 347/491] Retain subcommand annotations (#92) * Retain subcommand annotations * Fix mypy * Tweaks * Fix OmitArgPrefixes edge case, mypy * Fix mypy --- pyproject.toml | 2 +- tests/test_conf.py | 30 +++++++++++-------- .../test_conf_generated.py | 30 +++++++++++-------- tyro/_arguments.py | 8 ++++- tyro/_parsers.py | 10 +++++-- tyro/_resolver.py | 11 ++++--- tyro/conf/_confstruct.py | 8 ++--- 7 files changed, 58 insertions(+), 41 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b4c46cb2b..bd32d4e21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.18" +version = "0.5.19" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/tests/test_conf.py b/tests/test_conf.py index a820737a1..16bb78999 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -915,9 +915,9 @@ def times_two(n: str) -> int: @dataclasses.dataclass class Config: - x: Annotated[int, tyro.conf.arg(constructor=times_two)] + x: Annotated[int, tyro.conf.arg(name="x-renamed", constructor=times_two)] - assert tyro.cli(Config, args="--x.n 5".split(" ")) == Config(x=10) + assert tyro.cli(Config, args="--x-renamed.n 5".split(" ")) == Config(x=10) def test_custom_constructor_1() -> None: @@ -1205,18 +1205,22 @@ def commit(message: str, all: bool = False) -> str: class Arg: foo: int = 1 - t: Any = Union[ - Annotated[ - Any, - tyro.conf.subcommand(name="checkout", constructor=checkout), - ], - Annotated[ - Any, - tyro.conf.subcommand(name="commit", constructor=commit), + t: Any = Annotated[ + Union[ + Annotated[ + Any, + tyro.conf.subcommand(name="checkout-renamed", constructor=checkout), + ], + Annotated[ + Any, + tyro.conf.subcommand(name="commit", constructor=commit), + tyro.conf.FlagConversionOff, + ], + Arg, ], - Arg, + tyro.conf.OmitArgPrefixes, # Should do nothing. ] assert tyro.cli(t, args=["arg"]) == Arg() - assert tyro.cli(t, args=["checkout", "--branch", "main"]) == "main" - assert tyro.cli(t, args=["commit", "--message", "hi", "--all"]) == "hi True" + assert tyro.cli(t, args=["checkout-renamed", "--branch", "main"]) == "main" + assert tyro.cli(t, args=["commit", "--message", "hi", "--all", "True"]) == "hi True" diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index 482f67ec1..b90207c51 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -914,9 +914,9 @@ def times_two(n: str) -> int: @dataclasses.dataclass class Config: - x: Annotated[int, tyro.conf.arg(constructor=times_two)] + x: Annotated[int, tyro.conf.arg(name="x-renamed", constructor=times_two)] - assert tyro.cli(Config, args="--x.n 5".split(" ")) == Config(x=10) + assert tyro.cli(Config, args="--x-renamed.n 5".split(" ")) == Config(x=10) def test_custom_constructor_1() -> None: @@ -1204,18 +1204,22 @@ def commit(message: str, all: bool = False) -> str: class Arg: foo: int = 1 - t: Any = Union[ - Annotated[ - Any, - tyro.conf.subcommand(name="checkout", constructor=checkout), - ], - Annotated[ - Any, - tyro.conf.subcommand(name="commit", constructor=commit), + t: Any = Annotated[ + Union[ + Annotated[ + Any, + tyro.conf.subcommand(name="checkout-renamed", constructor=checkout), + ], + Annotated[ + Any, + tyro.conf.subcommand(name="commit", constructor=commit), + tyro.conf.FlagConversionOff, + ], + Arg, ], - Arg, + tyro.conf.OmitArgPrefixes, # Should do nothing. ] assert tyro.cli(t, args=["arg"]) == Arg() - assert tyro.cli(t, args=["checkout", "--branch", "main"]) == "main" - assert tyro.cli(t, args=["commit", "--message", "hi", "--all"]) == "hi True" + assert tyro.cli(t, args=["checkout-renamed", "--branch", "main"]) == "main" + assert tyro.cli(t, args=["commit", "--message", "hi", "--all", "True"]) == "hi True" diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 095c34cdd..0ef5f9901 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -500,7 +500,13 @@ def _rule_set_name_or_flag_and_dest( name_or_flag = "--" + name_or_flag # Strip. - if name_or_flag.startswith("--") and arg.subcommand_prefix != "": + if ( + # If OmitArgPrefixes was applied, then the subcommand prefix was already + # stripped. :) + _markers.OmitArgPrefixes not in arg.field.markers + and name_or_flag.startswith("--") + and arg.subcommand_prefix != "" + ): # This will run even when unused because we want the assert. strip_prefix = "--" + arg.subcommand_prefix + "." assert name_or_flag.startswith(strip_prefix) diff --git a/tyro/_parsers.py b/tyro/_parsers.py index f7edb6f86..1ef36e222 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -391,7 +391,12 @@ def from_field( len(found_subcommand_configs) > 0 and found_subcommand_configs[0].constructor_factory is not None ): - options[i] = found_subcommand_configs[0].constructor_factory() + options[i] = Annotated.__class_getitem__( # type: ignore + ( + found_subcommand_configs[0].constructor_factory(), + *_resolver.unwrap_annotated(option, Any)[1], # type: ignore + ) + ) # Exit if we don't contain nested types. if not all( @@ -411,7 +416,7 @@ def from_field( subcommand_name = _strings.subparser_name_from_type( prefix, type(None) if option is none_proxy else cast(type, option) ) - option, found_subcommand_configs = _resolver.unwrap_annotated( + option_unwrapped, found_subcommand_configs = _resolver.unwrap_annotated( option, _confstruct._SubcommandConfiguration ) if len(found_subcommand_configs) != 0: @@ -448,7 +453,6 @@ def from_field( subcommand_name = _strings.subparser_name_from_type( prefix, type(None) if option is none_proxy else cast(type, option) ) - option, _ = _resolver.unwrap_annotated(option) # Get a subcommand config: either pulled from the type annotations or the # field default. diff --git a/tyro/_resolver.py b/tyro/_resolver.py index d0d070749..931025837 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -13,7 +13,6 @@ Dict, FrozenSet, List, - Optional, Set, Tuple, TypeVar, @@ -186,7 +185,7 @@ def narrow_collection_types( def unwrap_annotated( - typ: TypeOrCallable, search_type: Optional[TypeForm[MetadataType]] = None + typ: TypeOrCallable, search_type: TypeForm[MetadataType] = Any # type: ignore ) -> Tuple[TypeOrCallable, Tuple[MetadataType, ...]]: """Helper for parsing typing.Annotated types. @@ -198,11 +197,11 @@ def unwrap_annotated( targets = tuple( x for x in getattr(typ, "__tyro_markers__", tuple()) - if search_type is not None and isinstance(x, search_type) + if search_type is Any or isinstance(x, search_type) ) assert isinstance(targets, tuple) if not hasattr(typ, "__metadata__"): - return typ, targets + return typ, targets # type: ignore args = get_args(typ) assert len(args) >= 2 @@ -211,9 +210,9 @@ def unwrap_annotated( targets = tuple( x for x in targets + args[1:] - if search_type is not None and isinstance(x, search_type) + if search_type is Any or isinstance(x, search_type) ) - return args[0], targets + return args[0], targets # type: ignore def apply_type_from_typevar( diff --git a/tyro/conf/_confstruct.py b/tyro/conf/_confstruct.py index f225c6b08..9e5d85b67 100644 --- a/tyro/conf/_confstruct.py +++ b/tyro/conf/_confstruct.py @@ -51,9 +51,6 @@ def subcommand( constructor: Optional[Union[Type, Callable]] = None, constructor_factory: Optional[Callable[[], Union[Type, Callable]]] = None, ) -> Any: - assert not ( - constructor is not None and constructor_factory is not None - ), "`constructor` and `constructor_factory` cannot both be set." """Returns a metadata object for configuring subcommands with `typing.Annotated`. Useful for aesthetics. @@ -99,6 +96,9 @@ def subcommand( constructor_factory: A function that returns a constructor type or function. Useful when the constructor isn't immediately available. """ + assert not ( + constructor is not None and constructor_factory is not None + ), "`constructor` and `constructor_factory` cannot both be set." return _SubcommandConfiguration( name, default, @@ -167,7 +167,7 @@ def arg( ``` Arguments: - name: A new name for the argument. + name: A new name for the argument in the CLI. metavar: Argument name in usage messages. The type is used by default. help: Helptext for this argument. The docstring is used by default. aliases: Aliases for this argument. All strings in the sequence should start From 25ef3017e0d810946e5df7e39202318a11a086b5 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 25 Nov 2023 02:23:50 -0800 Subject: [PATCH 348/491] Fix tuple + subcommand edge cases (#93) * Fix more tuple + subcommand edge cases * Add one more test * Add one more test, auto-camelcase for subcommand name generation with generics * Update generated tests * Fix subcommand tests, improve Python 3.11 test generation * Fix mypy errors, run ruff on generated tests * Bump version * Fix generics corner case, remove unused line * Drop unnecessary parentheses around unions --- pyproject.toml | 2 +- tests/test_conf.py | 20 +-- tests/test_nested.py | 11 +- tests/test_py311_generated/README.md | 3 +- tests/test_py311_generated/_generate.py | 43 +++++- .../test_collections_generated.py | 9 +- .../test_completion_generated.py | 3 +- .../test_conf_generated.py | 88 ++++++------- .../test_dcargs_generated.py | 11 +- .../test_dict_namedtuple_generated.py | 3 +- .../test_errors_generated.py | 46 ++++--- .../test_forward_ref_generated.py | 2 +- ...st_generics_and_serialization_generated.py | 9 +- .../test_helptext_generated.py | 48 +++---- .../test_mixed_unions_generated.py | 5 +- .../test_nested_generated.py | 76 +++++------ ...ew_style_annotations_min_py39_generated.py | 6 +- .../test_tuple_with_subcommands_generated.py | 109 ++++++++++++++++ tests/test_tuple_with_subcommands.py | 123 ++++++++++++++++++ tyro/_arguments.py | 36 ++--- tyro/_calling.py | 8 +- tyro/_cli.py | 3 +- tyro/_fields.py | 29 +++-- tyro/_instantiators.py | 6 + tyro/_parsers.py | 103 +++++++++------ tyro/_resolver.py | 10 +- tyro/_strings.py | 1 + tyro/_subcommand_matching.py | 2 +- tyro/conf/_markers.py | 8 +- 29 files changed, 562 insertions(+), 261 deletions(-) create mode 100644 tests/test_py311_generated/test_tuple_with_subcommands_generated.py create mode 100644 tests/test_tuple_with_subcommands.py diff --git a/pyproject.toml b/pyproject.toml index bd32d4e21..c426672f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.5.19" +version = "0.6.0" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/tests/test_conf.py b/tests/test_conf.py index 16bb78999..492f0501e 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -37,7 +37,7 @@ class DefaultInstanceSubparser: args=[ "--x", "1", - "bc:default-instance-http-server", + "default-instance-http-server", "--y", "5", "--no-flag", @@ -45,7 +45,7 @@ class DefaultInstanceSubparser: ) == tyro.cli( DefaultInstanceSubparser, - args=["--x", "1", "bc:default-instance-http-server", "--y", "5"], + args=["--x", "1", "default-instance-http-server", "--y", "5"], default=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=3, flag=False) ), @@ -55,11 +55,11 @@ class DefaultInstanceSubparser: assert ( tyro.cli( DefaultInstanceSubparser, - args=["--x", "1", "bc:default-instance-http-server", "--y", "8"], + args=["--x", "1", "default-instance-http-server", "--y", "8"], ) == tyro.cli( DefaultInstanceSubparser, - args=["--x", "1", "bc:default-instance-http-server", "--y", "8"], + args=["--x", "1", "default-instance-http-server", "--y", "8"], default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) @@ -616,7 +616,7 @@ class DefaultInstanceSubparser: tyro.cli( tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[ - "bc:default-instance-http-server", + "default-instance-http-server", "--x", "1", "--y", @@ -627,7 +627,7 @@ class DefaultInstanceSubparser: == tyro.cli( tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[ - "bc:default-instance-http-server", + "default-instance-http-server", "--x", "1", "--y", @@ -643,7 +643,7 @@ class DefaultInstanceSubparser: tyro.cli( tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[ - "bc:default-instance-http-server", + "default-instance-http-server", "--x", "1", "--y", @@ -653,7 +653,7 @@ class DefaultInstanceSubparser: == tyro.cli( tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[ - "bc:default-instance-http-server", + "default-instance-http-server", "--x", "1", "--y", @@ -696,7 +696,7 @@ def func(parent: DefaultInstanceSubparser) -> DefaultInstanceSubparser: assert tyro.cli( func, args=[ - "parent.bc:http-server", + "http-server", "--parent.x", "1", # --y and --no-flag are in a subcommand with prefix omission. @@ -708,7 +708,7 @@ def func(parent: DefaultInstanceSubparser) -> DefaultInstanceSubparser: assert tyro.cli( func, args=[ - "parent.bc:http-server", + "http-server", "--parent.x", "1", # --y is in a subcommand with prefix omission. diff --git a/tests/test_nested.py b/tests/test_nested.py index e05a59b12..2db8df02b 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -219,10 +219,7 @@ class ModelSettings: tyro.conf.OmitSubcommandPrefixes[ tyro.conf.ConsolidateSubcommandArgs[ModelSettings] ], - args=( - "output-head-settings:output-head-settings optimizer-settings:None" - " --number-of-outputs 5".split(" ") - ), + args=("output-head-settings None" " --number-of-outputs 5".split(" ")), ) == ModelSettings(OutputHeadSettings(5), None) assert tyro.cli( @@ -230,10 +227,8 @@ class ModelSettings: tyro.conf.ConsolidateSubcommandArgs[ModelSettings] ], args=( - "output-head-settings:output-head-settings" - " optimizer-settings:optimizer-settings --name sgd --number-of-outputs 5".split( - " " - ) + "output-head-settings" + " optimizer-settings --name sgd --number-of-outputs 5".split(" ") ), ) == ModelSettings(OutputHeadSettings(5), OptimizerSettings("sgd")) diff --git a/tests/test_py311_generated/README.md b/tests/test_py311_generated/README.md index 534da8d6f..50b518af5 100644 --- a/tests/test_py311_generated/README.md +++ b/tests/test_py311_generated/README.md @@ -1,2 +1,3 @@ This folder contains autogenerated tests, which replaces `typing_extension` -imports with `typing` imports. These can sometimes break `A is B`-style checks. +imports with `typing` imports and old-style union syntax (`Union[A, B]`) with +new ones (`A | B`). diff --git a/tests/test_py311_generated/_generate.py b/tests/test_py311_generated/_generate.py index e07e8c252..6b80bb616 100644 --- a/tests/test_py311_generated/_generate.py +++ b/tests/test_py311_generated/_generate.py @@ -1,9 +1,44 @@ """Generate a Python 3.11 version of tests. This will use imports from `typing` instead -of `typing_extensions`.""" +of `typing_extensions`, and replace Union[A, B] types with A | B.""" import pathlib +import subprocess for test_path in pathlib.Path(__file__).absolute().parent.parent.glob("test_*.py"): - ( - pathlib.Path(__file__).absolute().parent / (test_path.stem + "_generated.py") - ).write_text(test_path.read_text().replace("typing_extensions", "typing")) + content = test_path.read_text().replace("typing_extensions", "typing") + + while "Union[" in content: + new_content, _, b = content.partition("Union[") + + if b.strip()[0] == '"': + break # Don't bother with forward references! + + new_content_parts = [new_content] + + bracket_count = 0 + for i, char in enumerate(b): + if char == "[": + bracket_count += 1 + elif char == "]": + bracket_count -= 1 + + if char == "," and bracket_count == 0: + new_content_parts.append("|") + elif bracket_count == -1: + while new_content_parts[-1] in (" ", "|"): + new_content_parts.pop(-1) + new_content_parts.append(b[i + 1 :]) + break + elif char != "\n": + new_content_parts.append(char) + + content = "".join(new_content_parts) + + out_path = pathlib.Path(__file__).absolute().parent / ( + test_path.stem + "_generated.py" + ) + out_path.write_text(content) + + subprocess.run(["isort", str(out_path)]) + subprocess.run(["black", str(out_path)]) + subprocess.run(["ruff", "--fix", str(out_path)]) diff --git a/tests/test_py311_generated/test_collections_generated.py b/tests/test_py311_generated/test_collections_generated.py index 47b3286b3..2df1b6517 100644 --- a/tests/test_py311_generated/test_collections_generated.py +++ b/tests/test_py311_generated/test_collections_generated.py @@ -14,7 +14,6 @@ Sequence, Set, Tuple, - Union, ) import pytest @@ -310,7 +309,7 @@ class A: def test_union_over_collections() -> None: - def main(a: Union[Tuple[float, ...], Tuple[int, ...]]) -> Any: + def main(a: Tuple[float, ...] | Tuple[int, ...]) -> Any: return a assert tyro.cli(main, args="--a 3.3 3.3 7.0".split(" ")) == (3.3, 3.3, 7.0) @@ -318,7 +317,7 @@ def main(a: Union[Tuple[float, ...], Tuple[int, ...]]) -> Any: def test_union_over_collections_2() -> None: - def main(a: Union[Tuple[str, float, str], Tuple[str, str, float]]) -> Any: + def main(a: Tuple[str, float, str] | Tuple[str, str, float]) -> Any: return a assert tyro.cli(main, args="--a 3.3 hey 7.0".split(" ")) == ("3.3", "hey", 7.0) @@ -326,7 +325,7 @@ def main(a: Union[Tuple[str, float, str], Tuple[str, str, float]]) -> Any: def test_union_over_collections_3() -> None: - def main(a: Union[Tuple[int, int], Tuple[int, int, int]]) -> Tuple[int, ...]: + def main(a: Tuple[int, int] | Tuple[int, int, int]) -> Tuple[int, ...]: return a assert tyro.cli(main, args=["--a", "5", "5"]) == (5, 5) @@ -392,7 +391,7 @@ def main( Tuple[ Optional[int], Literal[3, 4], - Union[Tuple[int, int], Tuple[str, str]], + Tuple[int, int] | Tuple[str, str], ] ] ] = None diff --git a/tests/test_py311_generated/test_completion_generated.py b/tests/test_py311_generated/test_completion_generated.py index e1d6a9419..41527d9cd 100644 --- a/tests/test_py311_generated/test_completion_generated.py +++ b/tests/test_py311_generated/test_completion_generated.py @@ -1,7 +1,6 @@ import contextlib import dataclasses import io -from typing import Union import pytest @@ -26,7 +25,7 @@ class TypeB: @dataclasses.dataclass(frozen=True) class Wrapper: - supertype: Union[TypeA, TypeB] = TypeA() + supertype: TypeA | TypeB = TypeA() def test_bash(): diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index b90207c51..6d0025441 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -4,7 +4,7 @@ import io import json as json_ import shlex -from typing import Annotated, Any, Dict, Generic, List, Tuple, TypeVar, Union +from typing import Annotated, Any, Dict, Generic, List, Tuple, TypeVar import pytest from helptext_utils import get_helptext @@ -25,9 +25,9 @@ class DefaultInstanceSMTPServer: @dataclasses.dataclass class DefaultInstanceSubparser: x: int - # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + # bc: DefaultInstanceHTTPServer| DefaultInstanceSMTPServer bc: tyro.conf.OmitSubcommandPrefixes[ - Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + DefaultInstanceHTTPServer | DefaultInstanceSMTPServer ] assert ( @@ -36,7 +36,7 @@ class DefaultInstanceSubparser: args=[ "--x", "1", - "bc:default-instance-http-server", + "default-instance-http-server", "--y", "5", "--no-flag", @@ -44,7 +44,7 @@ class DefaultInstanceSubparser: ) == tyro.cli( DefaultInstanceSubparser, - args=["--x", "1", "bc:default-instance-http-server", "--y", "5"], + args=["--x", "1", "default-instance-http-server", "--y", "5"], default=DefaultInstanceSubparser( x=1, bc=DefaultInstanceHTTPServer(y=3, flag=False) ), @@ -54,11 +54,11 @@ class DefaultInstanceSubparser: assert ( tyro.cli( DefaultInstanceSubparser, - args=["--x", "1", "bc:default-instance-http-server", "--y", "8"], + args=["--x", "1", "default-instance-http-server", "--y", "8"], ) == tyro.cli( DefaultInstanceSubparser, - args=["--x", "1", "bc:default-instance-http-server", "--y", "8"], + args=["--x", "1", "default-instance-http-server", "--y", "8"], default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) @@ -78,7 +78,7 @@ class DefaultInstanceSMTPServer: class DefaultInstanceSubparser: x: int bc: tyro.conf.AvoidSubcommands[ - Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + DefaultInstanceHTTPServer | DefaultInstanceSMTPServer ] assert ( @@ -119,7 +119,7 @@ class DefaultInstanceSMTPServer: @dataclasses.dataclass class DefaultInstanceSubparser: x: int - bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + bc: DefaultInstanceHTTPServer | DefaultInstanceSMTPServer assert ( tyro.cli( @@ -164,10 +164,9 @@ class B: @dataclasses.dataclass class Nested2: - subcommand: Union[ - Annotated[A, tyro.conf.subcommand("command-a", default=A(7))], - Annotated[B, tyro.conf.subcommand("command-b", default=B(9))], - ] + subcommand: Annotated[ + A, tyro.conf.subcommand("command-a", default=A(7)) + ] | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] @dataclasses.dataclass class Nested1: @@ -223,10 +222,8 @@ class Nested2(Generic[T]): @dataclasses.dataclass class Nested1: nested2: Nested2[ - Union[ - Annotated[A, tyro.conf.subcommand("command-a", default=A(7))], - Annotated[B, tyro.conf.subcommand("command-b", default=B(9))], - ] + Annotated[A, tyro.conf.subcommand("command-a", default=A(7))] + | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] ] @dataclasses.dataclass @@ -274,10 +271,9 @@ class B: @dataclasses.dataclass class Nested2(Generic[T]): - subcommand: Union[ - Annotated[T, tyro.conf.subcommand("command-a", default=A(7))], - Annotated[B, tyro.conf.subcommand("command-b", default=B(9))], - ] + subcommand: Annotated[ + T, tyro.conf.subcommand("command-a", default=A(7)) + ] | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] @dataclasses.dataclass class Nested1: @@ -329,10 +325,10 @@ class B: @dataclasses.dataclass class Nested: - subcommand: Union[ - Annotated[B, tyro.conf.subcommand("one", default=default_one)], - Annotated[B, tyro.conf.subcommand("two")], - Annotated[B, tyro.conf.subcommand("three", default=default_three)], + subcommand: Annotated[ + B, tyro.conf.subcommand("one", default=default_one) + ] | Annotated[B, tyro.conf.subcommand("two")] | Annotated[ + B, tyro.conf.subcommand("three", default=default_three) ] # Match by hash. @@ -606,16 +602,16 @@ class DefaultInstanceSMTPServer: @dataclasses.dataclass class DefaultInstanceSubparser: x: int - # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + # bc: DefaultInstanceHTTPServer| DefaultInstanceSMTPServer bc: tyro.conf.OmitSubcommandPrefixes[ - Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + DefaultInstanceHTTPServer | DefaultInstanceSMTPServer ] = dataclasses.field(default_factory=DefaultInstanceHTTPServer) assert ( tyro.cli( tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[ - "bc:default-instance-http-server", + "default-instance-http-server", "--x", "1", "--y", @@ -626,7 +622,7 @@ class DefaultInstanceSubparser: == tyro.cli( tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[ - "bc:default-instance-http-server", + "default-instance-http-server", "--x", "1", "--y", @@ -642,7 +638,7 @@ class DefaultInstanceSubparser: tyro.cli( tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[ - "bc:default-instance-http-server", + "default-instance-http-server", "--x", "1", "--y", @@ -652,7 +648,7 @@ class DefaultInstanceSubparser: == tyro.cli( tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[ - "bc:default-instance-http-server", + "default-instance-http-server", "--x", "1", "--y", @@ -682,8 +678,8 @@ class DefaultInstanceSMTPServer: @dataclasses.dataclass class DefaultInstanceSubparser: x: int - # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] - bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + # bc: DefaultInstanceHTTPServer| DefaultInstanceSMTPServer + bc: DefaultInstanceHTTPServer | DefaultInstanceSMTPServer @tyro.conf.configure( tyro.conf.OmitSubcommandPrefixes, @@ -695,7 +691,7 @@ def func(parent: DefaultInstanceSubparser) -> DefaultInstanceSubparser: assert tyro.cli( func, args=[ - "parent.bc:http-server", + "http-server", "--parent.x", "1", # --y and --no-flag are in a subcommand with prefix omission. @@ -707,7 +703,7 @@ def func(parent: DefaultInstanceSubparser) -> DefaultInstanceSubparser: assert tyro.cli( func, args=[ - "parent.bc:http-server", + "http-server", "--parent.x", "1", # --y is in a subcommand with prefix omission. @@ -1205,18 +1201,16 @@ class Arg: foo: int = 1 t: Any = Annotated[ - Union[ - Annotated[ - Any, - tyro.conf.subcommand(name="checkout-renamed", constructor=checkout), - ], - Annotated[ - Any, - tyro.conf.subcommand(name="commit", constructor=commit), - tyro.conf.FlagConversionOff, - ], - Arg, - ], + Annotated[ + Any, + tyro.conf.subcommand(name="checkout-renamed", constructor=checkout), + ] + | Annotated[ + Any, + tyro.conf.subcommand(name="commit", constructor=commit), + tyro.conf.FlagConversionOff, + ] + | Arg, tyro.conf.OmitArgPrefixes, # Should do nothing. ] diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py index 145d5eae9..ee6f8f311 100644 --- a/tests/test_py311_generated/test_dcargs_generated.py +++ b/tests/test_py311_generated/test_dcargs_generated.py @@ -18,7 +18,6 @@ Tuple, TypeAlias, TypeVar, - Union, ) import pytest @@ -232,7 +231,7 @@ class A: def test_union_basic() -> None: - def main(x: Union[int, str]) -> Union[int, str]: + def main(x: int | str) -> int | str: return x assert tyro.cli(main, args=["--x", "5"]) == 5 @@ -241,7 +240,7 @@ def main(x: Union[int, str]) -> Union[int, str]: def test_union_with_list() -> None: - def main(x: Union[int, str, List[bool]]) -> Any: + def main(x: int | str | List[bool]) -> Any: return x assert tyro.cli(main, args=["--x", "5"]) == 5 @@ -252,7 +251,7 @@ def main(x: Union[int, str, List[bool]]) -> Any: def test_union_literal() -> None: - def main(x: Union[Literal[1, 2], Literal[3, 4, 5], str]) -> Union[int, str]: + def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: return x assert tyro.cli(main, args=["--x", "5"]) == 5 @@ -555,7 +554,7 @@ def main() -> argparse.ArgumentParser: def test_pathlike_custom_class() -> None: class CustomPath(pathlib.PurePosixPath): - def __new__(cls, *args: Union[str, os.PathLike]): + def __new__(cls, *args: str | os.PathLike): return super().__new__(cls, *args) def main(a: CustomPath) -> CustomPath: @@ -708,7 +707,7 @@ def test_empty_container() -> None: @dataclasses.dataclass class A: x: Tuple[int, ...] = (1, 2, 3) - y: Union[int, str, List[bool]] = dataclasses.field( + y: int | str | List[bool] = dataclasses.field( default_factory=lambda: [False, False, True] ) diff --git a/tests/test_py311_generated/test_dict_namedtuple_generated.py b/tests/test_py311_generated/test_dict_namedtuple_generated.py index e3b2e6326..44e560990 100644 --- a/tests/test_py311_generated/test_dict_namedtuple_generated.py +++ b/tests/test_py311_generated/test_dict_namedtuple_generated.py @@ -13,7 +13,6 @@ Required, Tuple, TypedDict, - Union, cast, ) @@ -56,7 +55,7 @@ def main(params: Mapping[Literal[1, 3, 5, 7], bool] = {5: False, 1: True}) -> An def test_tuple_in_dict() -> None: - def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> dict: + def main(x: Dict[Tuple[int, int] | Tuple[str, str], Tuple[int, int]]) -> dict: return x assert tyro.cli(main, args="--x 1 1 2 2 3 3 4 4".split(" ")) == { diff --git a/tests/test_py311_generated/test_errors_generated.py b/tests/test_py311_generated/test_errors_generated.py index e41a8cf7a..dbf343697 100644 --- a/tests/test_py311_generated/test_errors_generated.py +++ b/tests/test_py311_generated/test_errors_generated.py @@ -1,7 +1,7 @@ import contextlib import dataclasses import io -from typing import Dict, List, Literal, Tuple, TypeVar, Union +from typing import Dict, List, Literal, Tuple, TypeVar import pytest @@ -35,7 +35,7 @@ def main(x: Tuple[List[str], List[str]]) -> None: def test_ambiguous_collection_3() -> None: - def main(x: List[Union[Tuple[int, int], Tuple[int, int, int]]]) -> None: + def main(x: List[Tuple[int, int] | Tuple[int, int, int]]) -> None: pass with pytest.raises(tyro.UnsupportedTypeAnnotationError): @@ -193,7 +193,7 @@ class ClassB: target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): - tyro.cli(Union[ClassA, ClassB], args="--reward.trac".split(" ")) # type: ignore + tyro.cli(ClassA | ClassB, args="--reward.trac".split(" ")) # type: ignore error = target.getvalue() assert "Unrecognized argument" in error @@ -218,7 +218,7 @@ class ClassB: target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): - tyro.cli(Union[ClassA, ClassB], args="--fjdkslaj --reward.trac".split(" ")) # type: ignore + tyro.cli(ClassA | ClassB, args="--fjdkslaj --reward.trac".split(" ")) # type: ignore error = target.getvalue() assert "Unrecognized argument" in error @@ -244,7 +244,7 @@ class ClassB: target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): - tyro.cli(Union[ClassA, ClassB], args="--rd.trac".split(" ")) # type: ignore + tyro.cli(ClassA | ClassB, args="--rd.trac".split(" ")) # type: ignore error = target.getvalue() assert "Unrecognized argument" in error @@ -270,7 +270,7 @@ class ClassB: target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): - tyro.cli(Union[ClassA, ClassB], args="--track".split(" ")) # type: ignore + tyro.cli(ClassA | ClassB, args="--track".split(" ")) # type: ignore error = target.getvalue() assert "Unrecognized argument" in error @@ -312,7 +312,7 @@ class ClassB: target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): - tyro.cli(Union[ClassA, ClassB], args="--track".split(" ")) # type: ignore + tyro.cli(ClassA | ClassB, args="--track".split(" ")) # type: ignore error = target.getvalue() assert "Unrecognized argument" in error @@ -376,9 +376,15 @@ class ClassI: target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): tyro.cli( # type: ignore - Union[ - ClassA, ClassB, ClassC, ClassD, ClassE, ClassF, ClassG, ClassH, ClassI - ], + ClassA + | ClassB + | ClassC + | ClassD + | ClassE + | ClassF + | ClassG + | ClassH + | ClassI, args="--track".split(" "), ) @@ -435,9 +441,15 @@ class ClassI: target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): tyro.cli( # type: ignore - Union[ - ClassA, ClassB, ClassC, ClassD, ClassE, ClassF, ClassG, ClassH, ClassI - ], + ClassA + | ClassB + | ClassC + | ClassD + | ClassE + | ClassF + | ClassG + | ClassH + | ClassI, args="--track --ffff".split(" "), ) @@ -542,11 +554,11 @@ class B: class C: c: int - assert tyro.cli(Union[A, B], args="a --a 5".split(" ")) == A(5) # type: ignore - assert tyro.cli(Union[A, B], args="b --b 5".split(" ")) == B(5) # type: ignore + assert tyro.cli(A | B, args="a --a 5".split(" ")) == A(5) # type: ignore + assert tyro.cli(A | B, args="b --b 5".split(" ")) == B(5) # type: ignore with pytest.warns(UserWarning): - assert tyro.cli(Union[A, B], default=C(3), args=[]) == C(3) # type: ignore + assert tyro.cli(A | B, default=C(3), args=[]) == C(3) # type: ignore with pytest.warns(UserWarning): - assert tyro.cli(Union[A, B], default=C(3), args="b --b 5".split(" ")) == B(5) # type: ignore + assert tyro.cli(A | B, default=C(3), args="b --b 5".split(" ")) == B(5) # type: ignore diff --git a/tests/test_py311_generated/test_forward_ref_generated.py b/tests/test_py311_generated/test_forward_ref_generated.py index ccd29863e..e54438bd0 100644 --- a/tests/test_py311_generated/test_forward_ref_generated.py +++ b/tests/test_py311_generated/test_forward_ref_generated.py @@ -9,7 +9,7 @@ @dataclasses.dataclass class A1: x: int - bc: "Union[B, C]" + bc: "B| C" @dataclasses.dataclass diff --git a/tests/test_py311_generated/test_generics_and_serialization_generated.py b/tests/test_py311_generated/test_generics_and_serialization_generated.py index 54c4633e5..223601cd8 100644 --- a/tests/test_py311_generated/test_generics_and_serialization_generated.py +++ b/tests/test_py311_generated/test_generics_and_serialization_generated.py @@ -2,7 +2,7 @@ import dataclasses import enum import io -from typing import Annotated, Generic, List, Tuple, Type, TypeVar, Union +from typing import Annotated, Generic, List, Tuple, Type, TypeVar import pytest import yaml @@ -259,7 +259,7 @@ class CommandTwo: @dataclasses.dataclass class Subparser(Generic[T1, T2]): - command: Union[T1, T2] + command: T1 | T2 parsed_instance = tyro.cli( Subparser[CommandOne, CommandTwo], @@ -298,7 +298,7 @@ class Command(Generic[T]): @dataclasses.dataclass class Subparser(Generic[T1, T2]): - command: Union[T1, T2] + command: T1 | T2 parsed_instance = tyro.cli( Subparser[Command[int], Command[float]], @@ -370,7 +370,6 @@ def main(x: ActualParentClass[int] = ChildClass(5, 5)) -> ActualParentClass: def test_pculbertson() -> None: # https://github.com/brentyi/tyro/issues/7 - from typing import Union @dataclasses.dataclass(frozen=True) class TypeA: @@ -382,7 +381,7 @@ class TypeB: @dataclasses.dataclass class Wrapper: - subclass: Union[TypeA, TypeB] = TypeA(1) + subclass: TypeA | TypeB = TypeA(1) wrapper1 = Wrapper() # Create Wrapper object. assert wrapper1 == tyro.extras.from_yaml(Wrapper, tyro.extras.to_yaml(wrapper1)) diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py index 2591f18f9..7defaf199 100644 --- a/tests/test_py311_generated/test_helptext_generated.py +++ b/tests/test_py311_generated/test_helptext_generated.py @@ -13,7 +13,6 @@ Optional, Tuple, TypeVar, - Union, cast, ) @@ -413,11 +412,11 @@ class Subcommand3: @dataclasses.dataclass class MultipleSubparsers: # Field a description. - a: Union[Subcommand1, Subcommand2, Subcommand3] + a: Subcommand1 | Subcommand2 | Subcommand3 # Field b description. - b: Union[Subcommand1, Subcommand2, Subcommand3] + b: Subcommand1 | Subcommand2 | Subcommand3 # Field c description. - c: Union[Subcommand1, Subcommand2, Subcommand3] = dataclasses.field( + c: Subcommand1 | Subcommand2 | Subcommand3 = dataclasses.field( default_factory=Subcommand3 ) @@ -461,7 +460,7 @@ class OptionalHelptext: def test_metavar_0() -> None: - def main(x: Union[Literal[0, 1, 2, 3], Tuple[int, int]]) -> None: + def main(x: Literal[0, 1, 2, 3] | Tuple[int, int]) -> None: pass helptext = get_helptext(main) @@ -470,11 +469,7 @@ def main(x: Union[Literal[0, 1, 2, 3], Tuple[int, int]]) -> None: def test_metavar_1() -> None: def main( - x: Union[ - Literal[0, 1, 2, 3], - Literal["hey,there", "hello"], - List[int], - ] + x: Literal[0, 1, 2, 3] | Literal["hey,there", "hello"] | List[int] ) -> None: pass @@ -487,7 +482,7 @@ def test_metavar_2() -> None: def main( x: Tuple[ Literal[0, 1, 2, 3], - Union[int, str], + int | str, ] ) -> None: pass @@ -497,12 +492,7 @@ def main( def test_metavar_3() -> None: - def main( - x: Union[ - Literal[0, 1, 2, 3], - Union[Tuple[int, int], Tuple[str]], - ] - ) -> None: + def main(x: Literal[0, 1, 2, 3] | Tuple[int, int] | Tuple[str]) -> None: pass helptext = get_helptext(main) @@ -511,11 +501,7 @@ def main( def test_metavar_4() -> None: def main( - x: Union[ - Literal[0, 1, 2, 3], - Union[Tuple[int, int], Tuple[str, str, str]], - Literal[True], - ] + x: Literal[0, 1, 2, 3] | Tuple[int, int] | Tuple[str, str, str] | Literal[True] ) -> None: pass @@ -524,9 +510,7 @@ def main( def test_metavar_5() -> None: - def main( - x: List[Union[Tuple[int, int], Tuple[str, str]]] = [(1, 1), (2, 2)] - ) -> None: + def main(x: List[Tuple[int, int] | Tuple[str, str]] = [(1, 1), (2, 2)]) -> None: pass helptext = get_helptext(main) @@ -534,7 +518,7 @@ def main( def test_metavar_6() -> None: - def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> dict: + def main(x: Dict[Tuple[int, int] | Tuple[str, str], Tuple[int, int]]) -> dict: return x helptext = get_helptext(main) @@ -619,11 +603,11 @@ class SubcommandThree: @dataclasses.dataclass class MultipleSubparsers: # Field a description. - a: Union[SubcommandOne, SubcommandTwo, SubcommandThree] + a: SubcommandOne | SubcommandTwo | SubcommandThree # Field b description. - b: Union[SubcommandOne, SubcommandTwo, SubcommandThree] + b: SubcommandOne | SubcommandTwo | SubcommandThree # Field c description. - c: Union[SubcommandOne, SubcommandTwo, SubcommandThree] = dataclasses.field( + c: SubcommandOne | SubcommandTwo | SubcommandThree = dataclasses.field( default_factory=SubcommandThree ) @@ -667,11 +651,11 @@ class SubcommandThree: @dataclasses.dataclass class MultipleSubparsers: # Field a description. - a: Union[SubcommandOne, SubcommandTwo, SubcommandThree] + a: SubcommandOne | SubcommandTwo | SubcommandThree # Field b description. - b: Union[SubcommandOne, SubcommandTwo, SubcommandThree] + b: SubcommandOne | SubcommandTwo | SubcommandThree # Field c description. - c: Union[SubcommandOne, SubcommandTwo, SubcommandThree] = dataclasses.field( + c: SubcommandOne | SubcommandTwo | SubcommandThree = dataclasses.field( default_factory=SubcommandThree ) diff --git a/tests/test_py311_generated/test_mixed_unions_generated.py b/tests/test_py311_generated/test_mixed_unions_generated.py index 47fb35833..15272ec95 100644 --- a/tests/test_py311_generated/test_mixed_unions_generated.py +++ b/tests/test_py311_generated/test_mixed_unions_generated.py @@ -10,7 +10,6 @@ import dataclasses -from typing import Union import pytest @@ -30,7 +29,7 @@ class DefaultSMTPServer: class DefaultSubparser: x: int # Note that we add [int, str] to the annotation here... this should be ignored. - bc: Union[int, str, DefaultHTTPServer, DefaultSMTPServer] = dataclasses.field( + bc: int | str | DefaultHTTPServer | DefaultSMTPServer = dataclasses.field( default_factory=lambda: DefaultHTTPServer(5) ) @@ -75,7 +74,7 @@ class DefaultSMTPServer: class DefaultSubparser: x: int # Note that we add [int, str] to the annotation here... this should be ignored. - bc: Union[int, str, DefaultHTTPServer, DefaultSMTPServer] = 5 + bc: int | str | DefaultHTTPServer | DefaultSMTPServer = 5 assert ( tyro.cli(DefaultSubparser, args=["--x", "1", "--bc", "5"]) diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index 44bd668f6..fed6d61dc 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -8,7 +8,6 @@ Optional, Tuple, TypeVar, - Union, ) import pytest @@ -228,10 +227,7 @@ class ModelSettings: tyro.conf.OmitSubcommandPrefixes[ tyro.conf.ConsolidateSubcommandArgs[ModelSettings] ], - args=( - "output-head-settings:output-head-settings optimizer-settings:None" - " --number-of-outputs 5".split(" ") - ), + args=("output-head-settings None" " --number-of-outputs 5".split(" ")), ) == ModelSettings(OutputHeadSettings(5), None) assert tyro.cli( @@ -239,10 +235,8 @@ class ModelSettings: tyro.conf.ConsolidateSubcommandArgs[ModelSettings] ], args=( - "output-head-settings:output-head-settings" - " optimizer-settings:optimizer-settings --name sgd --number-of-outputs 5".split( - " " - ) + "output-head-settings" + " optimizer-settings --name sgd --number-of-outputs 5".split(" ") ), ) == ModelSettings(OutputHeadSettings(5), OptimizerSettings("sgd")) @@ -259,7 +253,7 @@ class SMTPServer: @dataclasses.dataclass class Subparser: x: int - bc: Union[HTTPServer, SMTPServer] + bc: HTTPServer | SMTPServer assert tyro.cli( Subparser, args=["--x", "1", "bc:http-server", "--bc.y", "3"] @@ -291,10 +285,10 @@ class SMTPServer: @dataclasses.dataclass class Subparser: x: int - bc: Union[HTTPServer, SMTPServer] + bc: HTTPServer | SMTPServer assert tyro.cli( - Union[HTTPServer, SMTPServer], args=["http-server", "--y", "3"] # type: ignore + HTTPServer | SMTPServer, args=["http-server", "--y", "3"] # type: ignore ) == HTTPServer(y=3) @@ -310,7 +304,7 @@ class DefaultSMTPServer: @dataclasses.dataclass class DefaultSubparser: x: int - bc: Union[DefaultHTTPServer, DefaultSMTPServer] = dataclasses.field( + bc: DefaultHTTPServer | DefaultSMTPServer = dataclasses.field( default_factory=lambda: DefaultHTTPServer(5) ) @@ -354,7 +348,7 @@ class DefaultInstanceSMTPServer: @dataclasses.dataclass class DefaultInstanceSubparser: x: int - bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + bc: DefaultInstanceHTTPServer | DefaultInstanceSMTPServer assert ( tyro.cli( @@ -409,7 +403,7 @@ class DefaultSMTPServer: @dataclasses.dataclass class DefaultSubparser: x: int - bc: Union[DefaultHTTPServer, DefaultSMTPServer] = dataclasses.field( + bc: DefaultHTTPServer | DefaultSMTPServer = dataclasses.field( default_factory=lambda: 5 # type: ignore ) @@ -436,7 +430,7 @@ class C: c: int with pytest.warns(UserWarning): - assert tyro.cli(Union[A, B], default=C(3), args=["c", "--c", "2"]) == C(2) # type: ignore + assert tyro.cli(A | B, default=C(3), args=["c", "--c", "2"]) == C(2) # type: ignore def test_optional_subparser() -> None: @@ -451,7 +445,7 @@ class OptionalSMTPServer: @dataclasses.dataclass class OptionalSubparser: x: int - bc: Optional[Union[OptionalHTTPServer, OptionalSMTPServer]] + bc: Optional[OptionalHTTPServer | OptionalSMTPServer] assert tyro.cli( OptionalSubparser, args=["--x", "1", "bc:optional-http-server", "--bc.y", "3"] @@ -520,9 +514,9 @@ class Subcommand3: @dataclasses.dataclass class MultipleSubparsers: - a: Union[Subcommand1, Subcommand2, Subcommand3] - b: Union[Subcommand1, Subcommand2, Subcommand3] - c: Union[Subcommand1, Subcommand2, Subcommand3] + a: Subcommand1 | Subcommand2 | Subcommand3 + b: Subcommand1 | Subcommand2 | Subcommand3 + c: Subcommand1 | Subcommand2 | Subcommand3 with pytest.raises(SystemExit): tyro.cli(MultipleSubparsers, args=[]) @@ -568,9 +562,9 @@ class Subcommand3: @dataclasses.dataclass class MultipleSubparsers: - a: Union[Subcommand1, Subcommand2, Subcommand3] = Subcommand1(tyro.MISSING) - b: Union[Subcommand1, Subcommand2, Subcommand3] = Subcommand2(7) - c: Union[Subcommand1, Subcommand2, Subcommand3] = Subcommand3(3) + a: Subcommand1 | Subcommand2 | Subcommand3 = Subcommand1(tyro.MISSING) + b: Subcommand1 | Subcommand2 | Subcommand3 = Subcommand2(7) + c: Subcommand1 | Subcommand2 | Subcommand3 = Subcommand3(3) with pytest.raises(SystemExit): tyro.cli( @@ -661,11 +655,11 @@ class Subcommand3: @dataclasses.dataclass(frozen=True) class Subcommand2: - y: Union[Subcommand1, Subcommand3] + y: Subcommand1 | Subcommand3 @dataclasses.dataclass(frozen=True) class MultipleSubparsers: - a: Union[Subcommand1, Subcommand2] = Subcommand2(Subcommand1(tyro.MISSING)) + a: Subcommand1 | Subcommand2 = Subcommand2(Subcommand1(tyro.MISSING)) with pytest.raises(SystemExit): tyro.cli(MultipleSubparsers, args=[]) @@ -697,12 +691,12 @@ class Subcommand3: @dataclasses.dataclass(frozen=True) class Subcommand2: - y: Union[Subcommand1, Subcommand3] + y: Subcommand1 | Subcommand3 @dataclasses.dataclass(frozen=True) class MultipleSubparsers: - a: Union[Subcommand1, Subcommand2] - b: Union[Subcommand1, Subcommand2] + a: Subcommand1 | Subcommand2 + b: Subcommand1 | Subcommand2 with pytest.raises(SystemExit): tyro.cli(MultipleSubparsers, args=[]) @@ -737,12 +731,12 @@ class Subcommand3: @dataclasses.dataclass(frozen=True) class Subcommand2: - y: Union[Subcommand1, Subcommand3] + y: Subcommand1 | Subcommand3 @dataclasses.dataclass(frozen=True) class MultipleSubparsers: - a: Union[Subcommand1, Subcommand2] - b: Union[Subcommand1, Subcommand2] + a: Subcommand1 | Subcommand2 + b: Subcommand1 | Subcommand2 @dataclasses.dataclass(frozen=True) class Root: @@ -815,7 +809,7 @@ class Location: y: float z: float - def main(x: Union[Tuple[Tuple[Color], Location, float], Tuple[Color, Color]]): + def main(x: Tuple[Tuple[Color], Location, float] | Tuple[Color, Color]): return x assert tyro.cli( @@ -834,13 +828,13 @@ def test_generic_subparsers() -> None: class A(Generic[T]): x: T - def main(x: Union[A[int], A[float]]) -> Any: + def main(x: A[int] | A[float]) -> Any: return x assert tyro.cli(main, args="x:a-float --x.x 3.2".split(" ")) == A(3.2) assert tyro.cli(main, args="x:a-int --x.x 3".split(" ")) == A(3) - def main_with_default(x: Union[A[int], A[float]] = A(5)) -> Any: + def main_with_default(x: A[int] | A[float] = A(5)) -> Any: return x with pytest.raises(tyro.UnsupportedTypeAnnotationError): @@ -883,7 +877,7 @@ class B: @dataclasses.dataclass class Nested2: - subcommand: Union[A, B] + subcommand: A | B @dataclasses.dataclass class Nested1: @@ -935,7 +929,7 @@ class TypeB: @dataclasses.dataclass(frozen=True) class Wrapper: - supertype: Union[TypeA, TypeB] = TypeA() + supertype: TypeA | TypeB = TypeA() assert tyro.cli(Wrapper, args=[]) == Wrapper() assert ( @@ -989,7 +983,7 @@ class Sgd: @tyro.conf.configure(tyro.conf.OmitSubcommandPrefixes) def train( optimizer: Optimizers = Adam(container=DatasetContainer(ImageNet(50))), # type: ignore - ) -> Union[Adam, Sgd]: + ) -> Adam | Sgd: return optimizer assert tyro.cli(train, args=[]) == Adam(container=DatasetContainer(ImageNet(50))) @@ -1065,13 +1059,13 @@ def commit(message: str, all: bool = False) -> Tuple[str, bool]: def test_subcommand_by_type_tree() -> None: @dataclasses.dataclass(frozen=True) class A: - a: Union[int, str] + a: int | str @dataclasses.dataclass class Args: - inner: Union[ - Annotated[A, tyro.conf.subcommand(name="alt", default=A(5))], A - ] = A("hello") + inner: Annotated[A, tyro.conf.subcommand(name="alt", default=A(5))] | A = A( + "hello" + ) assert tyro.cli(Args, args=[]) == Args(A("hello")) assert "default: inner:alt" in get_helptext(Args) diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py index 9d04bb029..d1e01c056 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Any, Literal, Optional, Union +from typing import Any, Literal, Optional import pytest @@ -32,7 +32,7 @@ def main(x: tuple[Args, Args]) -> Any: def test_tuple_variable() -> None: - def main(x: tuple[Union[bool, str], ...]) -> Any: + def main(x: tuple[bool | str, ...]) -> Any: return x assert tyro.cli(main, args=["--x", "True", "Wrong"]) == (True, "Wrong") @@ -45,7 +45,7 @@ def main( tuple[ Optional[int], Literal[3, 4], - Union[tuple[int, int], tuple[str, str]], + tuple[int, int] | tuple[str, str], ] ] ] = None diff --git a/tests/test_py311_generated/test_tuple_with_subcommands_generated.py b/tests/test_py311_generated/test_tuple_with_subcommands_generated.py new file mode 100644 index 000000000..9419bce56 --- /dev/null +++ b/tests/test_py311_generated/test_tuple_with_subcommands_generated.py @@ -0,0 +1,109 @@ +# mypy: disable-error-code="call-overload,misc" +# +# Mypy errors from passing union types directly into tyro.cli() as Type[T]. We would +# benefit from TypeForm[T]: https://github.com/python/mypy/issues/9773 +"""Tests adapted from https://github.com/brentyi/tyro/issues/89, which catches edge +cases when combining nested tuple types, renamed arguments, and subcommands. + +Largely written by @wookayin. +""" + +import dataclasses +from pathlib import Path +from typing import Annotated, Generic, Tuple, TypeVar + +import tyro.conf + +T = TypeVar("T") + + +@dataclasses.dataclass +class Checkout(Generic[T]): + """Check out a branch.""" + + branch: T + + +@dataclasses.dataclass +class Commit: + """Commit something.""" + + input: tyro.conf.Positional[Path] + + +@dataclasses.dataclass +class Arg: + verbose: bool = True + + +def test_case1() -> None: + o = tyro.cli( + Checkout[str] | Commit, + args=["commit", "./path.txt"], + ) + assert o == Commit(input=Path("path.txt")) + + +def test_case2() -> None: + arg, action = tyro.cli( + Tuple[ + Arg, + Annotated[ + Checkout[str] | Commit, + tyro.conf.arg(name=""), + ], + ], + args=["commit", "./path.txt"], + ) + + assert isinstance(arg, Arg) + assert isinstance(action, Commit) + assert action.input == Path("./path.txt") + + +def test_case3() -> None: + o = tyro.cli( + Tuple[ + Annotated[ + Arg, + tyro.conf.arg(name=""), + ], + Annotated[ + Annotated[Checkout[str], tyro.conf.subcommand(name="checkout")] + | Annotated[Commit, tyro.conf.subcommand(name="commit")], + tyro.conf.arg(name=""), + ], + ], + args=["commit", "./path.txt"], + ) + assert o == (Arg(), Commit(Path("./path.txt"))) + + +def test_case4() -> None: + o = tyro.cli( + Tuple[ + Annotated[ + Annotated[Checkout[str], tyro.conf.subcommand(name="checkout")] + | Annotated[Commit, tyro.conf.subcommand(name="commit")], + tyro.conf.arg(name=""), + ] + ], + args=["commit", "./path.txt"], + ) + assert o == (Commit(Path("./path.txt")),) + + +def test_case5() -> None: + assert tyro.cli( + tyro.conf.OmitArgPrefixes[ + Tuple[ + Annotated[ + Checkout[str], + tyro.conf.subcommand(prefix_name=False), + ] + | Annotated[Commit, tyro.conf.subcommand(prefix_name=False)], + Arg, + ] + ], + args=["--no-verbose", "checkout-str", "--branch", "branch"], + ) == (Checkout("branch"), Arg(False)) diff --git a/tests/test_tuple_with_subcommands.py b/tests/test_tuple_with_subcommands.py new file mode 100644 index 000000000..88f1eba5d --- /dev/null +++ b/tests/test_tuple_with_subcommands.py @@ -0,0 +1,123 @@ +# mypy: disable-error-code="call-overload,misc" +# +# Mypy errors from passing union types directly into tyro.cli() as Type[T]. We would +# benefit from TypeForm[T]: https://github.com/python/mypy/issues/9773 +"""Tests adapted from https://github.com/brentyi/tyro/issues/89, which catches edge +cases when combining nested tuple types, renamed arguments, and subcommands. + +Largely written by @wookayin. +""" + +import dataclasses +from pathlib import Path +from typing import Generic, Tuple, TypeVar, Union + +from typing_extensions import Annotated + +import tyro.conf + +T = TypeVar("T") + + +@dataclasses.dataclass +class Checkout(Generic[T]): + """Check out a branch.""" + + branch: T + + +@dataclasses.dataclass +class Commit: + """Commit something.""" + + input: tyro.conf.Positional[Path] + + +@dataclasses.dataclass +class Arg: + verbose: bool = True + + +def test_case1() -> None: + o = tyro.cli( + Union[ + Checkout[str], + Commit, + ], + args=["commit", "./path.txt"], + ) + assert o == Commit(input=Path("path.txt")) + + +def test_case2() -> None: + arg, action = tyro.cli( + Tuple[ + Arg, + Annotated[ + Union[ + Checkout[str], + Commit, + ], + tyro.conf.arg(name=""), + ], + ], + args=["commit", "./path.txt"], + ) + + assert isinstance(arg, Arg) + assert isinstance(action, Commit) + assert action.input == Path("./path.txt") + + +def test_case3() -> None: + o = tyro.cli( + Tuple[ + Annotated[ + Arg, + tyro.conf.arg(name=""), + ], + Annotated[ + Union[ + Annotated[Checkout[str], tyro.conf.subcommand(name="checkout")], + Annotated[Commit, tyro.conf.subcommand(name="commit")], + ], + tyro.conf.arg(name=""), + ], + ], + args=["commit", "./path.txt"], + ) + assert o == (Arg(), Commit(Path("./path.txt"))) + + +def test_case4() -> None: + o = tyro.cli( + Tuple[ + Annotated[ + Union[ + Annotated[Checkout[str], tyro.conf.subcommand(name="checkout")], + Annotated[Commit, tyro.conf.subcommand(name="commit")], + ], + tyro.conf.arg(name=""), + ] + ], + args=["commit", "./path.txt"], + ) + assert o == (Commit(Path("./path.txt")),) + + +def test_case5() -> None: + assert tyro.cli( + tyro.conf.OmitArgPrefixes[ + Tuple[ + Union[ + Annotated[ + Checkout[str], + tyro.conf.subcommand(prefix_name=False), + ], + Annotated[Commit, tyro.conf.subcommand(prefix_name=False)], + ], + Arg, + ] + ], + args=["--no-verbose", "checkout-str", "--branch", "branch"], + ) == (Checkout("branch"), Arg(False)) diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 0ef5f9901..1c14b9a35 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -105,8 +105,8 @@ def format_usage(self): class ArgumentDefinition: """Structure containing everything needed to define an argument.""" - dest_prefix: str # True prefix. (eg for the argument's dest field) - name_prefix: str # User-facing prefix. + intern_prefix: str # True prefix. (eg for the argument's dest field) + extern_prefix: str # User-facing prefix. subcommand_prefix: str # Prefix for nesting. field: _fields.FieldDefinition type_from_typevar: Dict[TypeVar, TypeForm[Any]] @@ -163,14 +163,14 @@ def add_argument( # The conditions are intended to be conservative; if a directory path is # registered as a normal file one that's OK, the reverse on the other # hand will be overly restrictive. - self.field.name.endswith("_dir") - or self.field.name.endswith("_directory") - or self.field.name.endswith("_folder") + self.field.intern_name.endswith("_dir") + or self.field.intern_name.endswith("_directory") + or self.field.intern_name.endswith("_folder") ) name_suggests_path = ( - self.field.name.endswith("_file") - or self.field.name.endswith("_path") - or self.field.name.endswith("_filename") + self.field.intern_name.endswith("_file") + or self.field.intern_name.endswith("_path") + or self.field.intern_name.endswith("_filename") or name_suggests_dir ) complete_as_path = ( @@ -275,7 +275,7 @@ def _rule_handle_boolean_flags( ) assert False, ( - f"Expected a boolean as a default for {arg.field.name}, but got" + f"Expected a boolean as a default for {arg.field.intern_name}, but got" f" {lowered.default}." ) @@ -312,7 +312,7 @@ def _rule_recursive_instantiator_from_type( if arg.field.default in _fields.MISSING_SINGLETONS: raise _instantiators.UnsupportedTypeAnnotationError( "Unsupported type annotation for the field" - f" {_strings.make_field_name([arg.name_prefix, arg.field.name])}. To" + f" {_strings.make_field_name([arg.extern_prefix, arg.field.intern_name])}. To" " suppress this error, assign the field a default value." ) from e else: @@ -416,7 +416,9 @@ def _rule_generate_helptext( primary_help = arg.field.helptext if primary_help is None and _markers.Positional in arg.field.markers: - primary_help = _strings.make_field_name([arg.name_prefix, arg.field.name]) + primary_help = _strings.make_field_name( + [arg.extern_prefix, arg.field.intern_name] + ) if primary_help is not None and primary_help != "": help_parts.append(_rich_tag_if_enabled(primary_help, "helptext")) @@ -489,10 +491,10 @@ def _rule_set_name_or_flag_and_dest( lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: name_or_flag = _strings.make_field_name( - [arg.name_prefix, arg.field.name] + [arg.extern_prefix, arg.field.extern_name] if arg.field.argconf.prefix_name and _markers.OmitArgPrefixes not in arg.field.markers - else [arg.field.name] + else [arg.field.extern_name] ) # Prefix keyword arguments with --. @@ -504,19 +506,20 @@ def _rule_set_name_or_flag_and_dest( # If OmitArgPrefixes was applied, then the subcommand prefix was already # stripped. :) _markers.OmitArgPrefixes not in arg.field.markers + and _markers.Positional not in arg.field.markers and name_or_flag.startswith("--") and arg.subcommand_prefix != "" ): # This will run even when unused because we want the assert. strip_prefix = "--" + arg.subcommand_prefix + "." - assert name_or_flag.startswith(strip_prefix) if _markers.OmitSubcommandPrefixes in arg.field.markers: + assert name_or_flag.startswith(strip_prefix), name_or_flag name_or_flag = "--" + name_or_flag[len(strip_prefix) :] return dataclasses.replace( lowered, name_or_flag=name_or_flag, - dest=_strings.make_field_name([arg.dest_prefix, arg.field.name]), + dest=_strings.make_field_name([arg.intern_prefix, arg.field.intern_name]), ) @@ -543,6 +546,9 @@ def _rule_positional_special_handling( return dataclasses.replace( lowered, + name_or_flag=_strings.make_field_name( + [arg.intern_prefix, arg.field.intern_name] + ), dest=None, required=None, # Can't be passed in for positionals. metavar=metavar, diff --git a/tyro/_calling.py b/tyro/_calling.py index ec4a18986..f9bbfbb30 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -56,14 +56,16 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: arg_from_prefixed_field_name: Dict[str, _arguments.ArgumentDefinition] = {} for arg in parser_definition.args: arg_from_prefixed_field_name[ - _strings.make_field_name([arg.dest_prefix, arg.field.name]) + _strings.make_field_name([arg.intern_prefix, arg.field.intern_name]) ] = arg any_arguments_provided = False for field in field_list: value: Any - prefixed_field_name = _strings.make_field_name([field_name_prefix, field.name]) + prefixed_field_name = _strings.make_field_name( + [field_name_prefix, field.intern_name] + ) # Resolve field type. field_type = field.type_or_callable @@ -135,7 +137,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: consumed_keywords |= consumed_keywords_child else: # Unions over dataclasses (subparsers). This is the only other option. - subparser_def = parser_definition.subparsers_from_prefix[ + subparser_def = parser_definition.subparsers_from_intern_prefix[ prefixed_field_name ] subparser_dest = _strings.make_subparser_dest(name=prefixed_field_name) diff --git a/tyro/_cli.py b/tyro/_cli.py index 14656248c..beb1bef0b 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -376,7 +376,8 @@ def _cli_impl( description=description, parent_classes=set(), # Used for recursive calls. default_instance=default_instance_internal, # Overrides for default values. - prefix="", # Used for recursive calls. + intern_prefix="", # Used for recursive calls. + extern_prefix="", # Used for recursive calls. subcommand_prefix="", # Used for recursive calls. ) diff --git a/tyro/_fields.py b/tyro/_fields.py index d464fcbf6..d313c1980 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -55,7 +55,8 @@ @dataclasses.dataclass(frozen=True) class FieldDefinition: - name: str + intern_name: str + extern_name: str type_or_callable: Union[TypeForm[Any], Callable] """Type or callable for this field. This should have all Annotated[] annotations stripped.""" @@ -75,7 +76,7 @@ def __post_init__(self): _markers.Fixed in self.markers or _markers.Suppress in self.markers ) and self.default in MISSING_SINGLETONS: raise _instantiators.UnsupportedTypeAnnotationError( - f"Field {self.name} is missing a default value!" + f"Field {self.intern_name} is missing a default value!" ) @staticmethod @@ -117,12 +118,13 @@ def make( type_or_callable, _markers._Marker ) return FieldDefinition( - name if argconf.name is None else argconf.name, - type_or_callable + intern_name=name, + extern_name=name if argconf.name is None else argconf.name, + type_or_callable=type_or_callable if argconf.constructor_factory is None else argconf.constructor_factory(), - default, - helptext, + default=default, + helptext=helptext, markers=frozenset(inferred_markers).union(markers), custom_constructor=argconf.constructor_factory is not None, argconf=argconf, @@ -143,7 +145,7 @@ def is_positional(self) -> bool: # Explicit positionals. _markers.Positional in self.markers # Dummy dataclasses should have a single positional field. - or self.name == _strings.dummy_field_name + or self.intern_name == _strings.dummy_field_name or ( # Make required arguments positional. _markers.PositionalRequiredArgs in self.markers @@ -157,7 +159,7 @@ def is_positional_call(self) -> bool: # Explicit positionals. _markers._PositionalCall in self.markers # Dummy dataclasses should have a single positional field. - or self.name == _strings.dummy_field_name + or self.intern_name == _strings.dummy_field_name ) @@ -305,7 +307,7 @@ def resolve(field: FieldDefinition) -> FieldDefinition: # If the default value doesn't match the resolved type, we expand the # type. This is inspired by https://github.com/brentyi/tyro/issues/88. warnings.warn( - f"The field {field.name} is annotated with type {field.type_or_callable}, " + f"The field {field.intern_name} is annotated with type {field.type_or_callable}, " f"but the default value {field.default} has type {type(field.default)}. " f"We'll try to handle this gracefully, but it may cause unexpected behavior." ) @@ -707,7 +709,16 @@ def _field_list_from_tuple( contains_nested = False for field in field_list: + if get_origin(field.type_or_callable) is Union: + for option in get_args(field.type_or_callable): + # The second argument here is the default value, which can help with + # narrowing but is generall not necessary. + contains_nested |= is_nested_type(option, MISSING_NONPROP) + contains_nested |= is_nested_type(field.type_or_callable, field.default) + + if contains_nested: + break if not contains_nested: # We could also check for variable length children, which can be populated when # the tuple is interpreted as a nested field but not a directly parsed one. diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index d1525fd93..55ec07183 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -64,6 +64,8 @@ get_type_hints, ) +from . import _resolver + # There are cases where typing.Literal doesn't match typing_extensions.Literal: # https://github.com/python/typing_extensions/pull/148 try: @@ -130,6 +132,10 @@ def is_type_string_converter(typ: Union[Callable, TypeForm[Any]]) -> bool: # Some checks we can do if the signature is available! for i, param in enumerate(signature.parameters.values()): annotation = type_annotations.get(param.name, param.annotation) + + # Hack: apply_type_from_typevar applies shims, like UnionType => Union + # conversion. + annotation = _resolver.apply_type_from_typevar(annotation, {}) if i == 0 and not ( (get_origin(annotation) is Union and str in get_args(annotation)) or annotation in (str, inspect.Parameter.empty) diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 1ef36e222..ec311c18e 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -51,8 +51,9 @@ class ParserSpecification: subparsers: Optional[SubparsersSpecification] # - A set of subparser groups, which reflect the tree structure built by the # hierarchy of a nested config structure. - subparsers_from_prefix: Dict[str, SubparsersSpecification] - prefix: str + subparsers_from_intern_prefix: Dict[str, SubparsersSpecification] + intern_prefix: str + extern_prefix: str has_required_args: bool consolidate_subcommand_args: bool @@ -64,7 +65,8 @@ def from_callable_or_type( default_instance: Union[ T, _fields.PropagatingMissingType, _fields.NonpropagatingMissingType ], - prefix: str, + intern_prefix: str, + extern_prefix: str, subcommand_prefix: str = "", ) -> ParserSpecification: """Create a parser definition from a callable or type.""" @@ -96,7 +98,7 @@ def from_callable_or_type( has_required_args = False args = [] - helptext_from_nested_class_field_name: Dict[str, Optional[str]] = {} + helptext_from_intern_prefixed_field_name: Dict[str, Optional[str]] = {} subparsers = None subparsers_from_prefix = {} @@ -106,7 +108,8 @@ def from_callable_or_type( field, type_from_typevar=type_from_typevar, parent_classes=parent_classes, - prefix=prefix, + intern_prefix=intern_prefix, + extern_prefix=extern_prefix, subcommand_prefix=subcommand_prefix, ) if isinstance(field_out, _arguments.ArgumentDefinition): @@ -116,7 +119,7 @@ def from_callable_or_type( has_required_args = True elif isinstance(field_out, SubparsersSpecification): # Handle subparsers. - subparsers_from_prefix[field_out.prefix] = field_out + subparsers_from_prefix[field_out.intern_prefix] = field_out subparsers = add_subparsers_to_leaves(subparsers, field_out) elif isinstance(field_out, ParserSpecification): # Handle nested parsers. @@ -128,7 +131,9 @@ def from_callable_or_type( # Include nested subparsers. if nested_parser.subparsers is not None: - subparsers_from_prefix.update(nested_parser.subparsers_from_prefix) + subparsers_from_prefix.update( + nested_parser.subparsers_from_intern_prefix + ) subparsers = add_subparsers_to_leaves( subparsers, nested_parser.subparsers ) @@ -138,17 +143,17 @@ def from_callable_or_type( k, v, ) in nested_parser.helptext_from_nested_class_field_name.items(): - helptext_from_nested_class_field_name[ - _strings.make_field_name([field.name, k]) + helptext_from_intern_prefixed_field_name[ + _strings.make_field_name([field.intern_name, k]) ] = v - class_field_name = _strings.make_field_name([field.name]) + class_field_name = _strings.make_field_name([field.intern_name]) if field.helptext is not None: - helptext_from_nested_class_field_name[ + helptext_from_intern_prefixed_field_name[ class_field_name ] = field.helptext else: - helptext_from_nested_class_field_name[ + helptext_from_intern_prefixed_field_name[ class_field_name ] = _docstrings.get_callable_description(nested_parser.f) @@ -158,10 +163,10 @@ def from_callable_or_type( len(nested_parser.args) >= 1 and _markers._OPTIONAL_GROUP in nested_parser.args[0].field.markers ): - current_helptext = helptext_from_nested_class_field_name[ + current_helptext = helptext_from_intern_prefixed_field_name[ class_field_name ] - helptext_from_nested_class_field_name[class_field_name] = ( + helptext_from_intern_prefixed_field_name[class_field_name] = ( ("" if current_helptext is None else current_helptext + "\n\n") + "Default: " + str(field.default) @@ -175,10 +180,11 @@ def from_callable_or_type( else _docstrings.get_callable_description(f) ), args=args, - helptext_from_nested_class_field_name=helptext_from_nested_class_field_name, + helptext_from_nested_class_field_name=helptext_from_intern_prefixed_field_name, subparsers=subparsers, - subparsers_from_prefix=subparsers_from_prefix, - prefix=prefix, + subparsers_from_intern_prefix=subparsers_from_prefix, + intern_prefix=intern_prefix, + extern_prefix=extern_prefix, has_required_args=has_required_args, consolidate_subcommand_args=consolidate_subcommand_args, ) @@ -246,13 +252,13 @@ def format_group_name(prefix: str) -> str: for arg in self.args: if ( arg.lowered.help is not argparse.SUPPRESS - and arg.dest_prefix not in group_from_prefix + and arg.intern_prefix not in group_from_prefix ): description = self.helptext_from_nested_class_field_name.get( - arg.dest_prefix + arg.intern_prefix ) - group_from_prefix[arg.dest_prefix] = parser.add_argument_group( - format_group_name(arg.dest_prefix), + group_from_prefix[arg.intern_prefix] = parser.add_argument_group( + format_group_name(arg.intern_prefix), description=description, ) @@ -262,8 +268,8 @@ def format_group_name(prefix: str) -> str: arg.add_argument(positional_group) continue - if arg.dest_prefix in group_from_prefix: - arg.add_argument(group_from_prefix[arg.dest_prefix]) + if arg.intern_prefix in group_from_prefix: + arg.add_argument(group_from_prefix[arg.intern_prefix]) else: # Suppressed argument: still need to add them, but they won't show up in # the helptext so it doesn't matter which group. @@ -275,7 +281,8 @@ def handle_field( field: _fields.FieldDefinition, type_from_typevar: Dict[TypeVar, TypeForm[Any]], parent_classes: Set[Type[Any]], - prefix: str, + intern_prefix: str, + extern_prefix: str, subcommand_prefix: str, ) -> Union[ _arguments.ArgumentDefinition, @@ -286,7 +293,7 @@ def handle_field( if isinstance(field.type_or_callable, TypeVar): raise _instantiators.UnsupportedTypeAnnotationError( - f"Field {field.name} has an unbound TypeVar: {field.type_or_callable}." + f"Field {field.intern_name} has an unbound TypeVar: {field.type_or_callable}." ) if _markers.Fixed not in field.markers: @@ -295,7 +302,8 @@ def handle_field( field, type_from_typevar=type_from_typevar, parent_classes=parent_classes, - prefix=_strings.make_field_name([prefix, field.name]), + intern_prefix=_strings.make_field_name([intern_prefix, field.intern_name]), + extern_prefix=_strings.make_field_name([extern_prefix, field.extern_name]), ) if subparsers_attempt is not None: if ( @@ -328,14 +336,19 @@ def handle_field( description=None, parent_classes=parent_classes, default_instance=field.default, - prefix=_strings.make_field_name([prefix, field.name]), + intern_prefix=_strings.make_field_name( + [intern_prefix, field.intern_name] + ), + extern_prefix=_strings.make_field_name( + [extern_prefix, field.extern_name] + ), subcommand_prefix=subcommand_prefix, ) # (3) Handle primitive or fixed types. These produce a single argument! return _arguments.ArgumentDefinition( - dest_prefix=prefix, - name_prefix=prefix, + intern_prefix=intern_prefix, + extern_prefix=extern_prefix, subcommand_prefix=subcommand_prefix, field=field, type_from_typevar=type_from_typevar, @@ -349,7 +362,7 @@ class SubparsersSpecification: name: str description: Optional[str] parser_from_name: Dict[str, ParserSpecification] - prefix: str + intern_prefix: str required: bool default_instance: Any options: Tuple[Union[TypeForm[Any], Callable], ...] @@ -359,7 +372,8 @@ def from_field( field: _fields.FieldDefinition, type_from_typevar: Dict[TypeVar, TypeForm[Any]], parent_classes: Set[Type[Any]], - prefix: str, + intern_prefix: str, + extern_prefix: str, ) -> Optional[SubparsersSpecification]: # Union of classes should create subparsers. typ = _resolver.unwrap_annotated(field.type_or_callable)[0] @@ -414,7 +428,10 @@ def from_field( subcommand_type_from_name: Dict[str, type] = {} for option in options: subcommand_name = _strings.subparser_name_from_type( - prefix, type(None) if option is none_proxy else cast(type, option) + "" + if _markers.OmitSubcommandPrefixes in field.markers + else extern_prefix, + type(None) if option is none_proxy else cast(type, option), ) option_unwrapped, found_subcommand_configs = _resolver.unwrap_annotated( option, _confstruct._SubcommandConfiguration @@ -441,17 +458,20 @@ def from_field( # This should never be triggered because union types are expanded when # a bad default value is provided. assert default_name is not None, ( - f"`{prefix}` was provided a default value of type" + f"`{extern_prefix}` was provided a default value of type" f" {type(field.default)} but no matching subcommand was found. A" " type may be missing in the Union type declaration for" - f" `{prefix}`, which currently expects {options}." + f" `{extern_prefix}`, which currently expects {options}." ) # Add subcommands for each option. parser_from_name: Dict[str, ParserSpecification] = {} for option in options: subcommand_name = _strings.subparser_name_from_type( - prefix, type(None) if option is none_proxy else cast(type, option) + "" + if _markers.OmitSubcommandPrefixes in field.markers + else extern_prefix, + type(None) if option is none_proxy else cast(type, option), ) # Get a subcommand config: either pulled from the type annotations or the @@ -482,15 +502,16 @@ def from_field( description=subcommand_config.description, parent_classes=parent_classes, default_instance=subcommand_config.default, - prefix=prefix, - subcommand_prefix=prefix, + intern_prefix=intern_prefix, + extern_prefix=extern_prefix, + subcommand_prefix=intern_prefix, ) # Apply prefix to helptext in nested classes in subparsers. subparser = dataclasses.replace( subparser, helptext_from_nested_class_field_name={ - _strings.make_field_name([prefix, k]): v + _strings.make_field_name([intern_prefix, k]): v for k, v in subparser.helptext_from_nested_class_field_name.items() }, ) @@ -530,13 +551,13 @@ def from_field( ) return SubparsersSpecification( - name=field.name, + name=field.intern_name, # If we wanted, we could add information about the default instance # automatically, as is done for normal fields. But for now we just rely on # the user to include it in the docstring. description=description, parser_from_name=parser_from_name, - prefix=prefix, + intern_prefix=intern_prefix, required=required, default_instance=field.default, options=tuple(options), @@ -553,7 +574,7 @@ def apply( # Add subparsers to every node in previous level of the tree. argparse_subparsers = parent_parser.add_subparsers( - dest=_strings.make_subparser_dest(self.prefix), + dest=_strings.make_subparser_dest(self.intern_prefix), description=self.description, required=self.required, title=title, diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 931025837..d5a0fb01f 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -52,6 +52,11 @@ def resolve_generic_types( class, and a mapping from typevars to concrete types.""" origin_cls = get_origin(cls) + annotations = () + if origin_cls is Annotated: + annotations = get_args(cls)[1:] + cls = get_args(cls)[0] + origin_cls = get_origin(cls) type_from_typevar = {} if ( @@ -76,7 +81,10 @@ def resolve_generic_types( typevar_values = get_args(base) type_from_typevar.update(dict(zip(typevars, typevar_values))) - return cls, type_from_typevar + if len(annotations) == 0: + return cls, type_from_typevar + else: + return Annotated.__class_getitem__((cls, *annotations)), type_from_typevar # type: ignore @_unsafe_cache.unsafe_cache(maxsize=1024) diff --git a/tyro/_strings.py b/tyro/_strings.py index 2d495e10d..83684d422 100644 --- a/tyro/_strings.py +++ b/tyro/_strings.py @@ -107,6 +107,7 @@ def get_name(cls: Type) -> str: if orig is not None and hasattr(orig, "__name__"): parts = [orig.__name__] # type: ignore parts.extend(map(get_name, get_args(cls))) + parts = [hyphen_separated_from_camel_case(part) for part in parts] return get_delimeter().join(parts) elif hasattr(cls, "__name__"): return hyphen_separated_from_camel_case(cls.__name__) diff --git a/tyro/_subcommand_matching.py b/tyro/_subcommand_matching.py index 951524cd1..83ed38e08 100644 --- a/tyro/_subcommand_matching.py +++ b/tyro/_subcommand_matching.py @@ -81,7 +81,7 @@ def make( return _TypeTree( typ, { - field.name: _TypeTree.make(field.type_or_callable, field.default) + field.intern_name: _TypeTree.make(field.type_or_callable, field.default) for field in field_list }, ) diff --git a/tyro/conf/_markers.py b/tyro/conf/_markers.py index 3fbb717cc..6d04a38e7 100644 --- a/tyro/conf/_markers.py +++ b/tyro/conf/_markers.py @@ -84,8 +84,8 @@ """ OmitSubcommandPrefixes = Annotated[T, None] -"""Make flags used for keyword arguments in subcommands shorter by omitting the -subcommand-specific portion of the prefix. +"""Make CLI inputs used for subcommands shorter by omitting the subcommand-specific +portion of the prefix. If we have a structure with the field: @@ -93,6 +93,10 @@ By default, `--cmd.arg` may be generated as a flag for each dataclass in the union. If subcommand prefixes are omitted, we would instead simply have `--arg`. + +By default, `cmd:nested-type-a` and `cmd:nested-type-b` may be generated as subcommand. +If subcommand prefixes are omitted, we would instead simply have `nested-type-a` and +`nested-type-b`. """ OmitArgPrefixes = Annotated[T, None] From 6140aa14e9971d909ffe0237187c7cb3563c3c01 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 26 Nov 2023 02:44:36 -0800 Subject: [PATCH 349/491] Account for indent in subcommand helptext wrapping (#95) * Account for indent in subcommand helptext wrapping * Run format * Don't account for indent in help position --- tests/test_helptext.py | 68 +++++++++++++++++++ .../test_helptext_generated.py | 68 +++++++++++++++++++ tyro/_argparse_formatter.py | 7 +- 3 files changed, 140 insertions(+), 3 deletions(-) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 47d682edd..56fa060cc 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -685,3 +685,71 @@ class MultipleSubparsers: assert "--b.arg_x" in helptext assert "--b.no_arg_flag" in helptext assert "--b.arg_flag" in helptext + + +def test_subparsers_wrapping() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: int + + @dataclasses.dataclass + class CheckoutCompletion: + """Help message.""" + + y: int + + help = get_helptext(Union[A, CheckoutCompletion]) # type: ignore + assert help.count("checkout-completion") == 3 + + +def test_subparsers_wrapping1() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: int + + @dataclasses.dataclass + class CheckoutCompletio: + """Help message.""" + + y: int + + help = get_helptext(Union[A, CheckoutCompletio]) # type: ignore + assert help.count("checkout-completio") == 3 + + +def test_subparsers_wrapping2() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: int + + @dataclasses.dataclass + class CheckoutCompletionn: + """Help message.""" + + y: int + + help = get_helptext(Union[A, CheckoutCompletionn]) # type: ignore + assert help.count("checkout-completionn") == 3 + + +def test_subparsers_wrapping3() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: int + + @dataclasses.dataclass + class CmdCheckout012: + """Help message.""" + + y: int + + help = get_helptext(Union[A, CmdCheckout012]) # type: ignore + assert help.count("cmd-checkout012") == 3 diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py index 7defaf199..097999859 100644 --- a/tests/test_py311_generated/test_helptext_generated.py +++ b/tests/test_py311_generated/test_helptext_generated.py @@ -680,3 +680,71 @@ class MultipleSubparsers: assert "--b.arg_x" in helptext assert "--b.no_arg_flag" in helptext assert "--b.arg_flag" in helptext + + +def test_subparsers_wrapping() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: int + + @dataclasses.dataclass + class CheckoutCompletion: + """Help message.""" + + y: int + + help = get_helptext(A | CheckoutCompletion) # type: ignore + assert help.count("checkout-completion") == 3 + + +def test_subparsers_wrapping1() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: int + + @dataclasses.dataclass + class CheckoutCompletio: + """Help message.""" + + y: int + + help = get_helptext(A | CheckoutCompletio) # type: ignore + assert help.count("checkout-completio") == 3 + + +def test_subparsers_wrapping2() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: int + + @dataclasses.dataclass + class CheckoutCompletionn: + """Help message.""" + + y: int + + help = get_helptext(A | CheckoutCompletionn) # type: ignore + assert help.count("checkout-completionn") == 3 + + +def test_subparsers_wrapping3() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: int + + @dataclasses.dataclass + class CmdCheckout012: + """Help message.""" + + y: int + + help = get_helptext(A | CmdCheckout012) # type: ignore + assert help.count("cmd-checkout012") == 3 diff --git a/tyro/_argparse_formatter.py b/tyro/_argparse_formatter.py index 5cbb57410..b68bf93ac 100644 --- a/tyro/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -1006,7 +1006,7 @@ def _format_action(self, action: argparse.Action): invocation = self.formatter._format_action_invocation(action) indent = self.formatter._current_indent help_position = min( - self.formatter._action_max_length + 4 + indent, + self.formatter._action_max_length + 4, self.formatter._max_help_position, ) if self.formatter._fixed_help_position: @@ -1037,7 +1037,8 @@ def _format_action(self, action: argparse.Action): if ( action.help - and len(_strings.strip_ansi_sequences(invocation)) < help_position - 1 + and len(_strings.strip_ansi_sequences(invocation)) + indent + < help_position - 1 and not self.formatter._fixed_help_position ): table = Table(show_header=False, box=None, padding=0) @@ -1065,7 +1066,7 @@ def _format_action(self, action: argparse.Action): Padding( # Unescape % signs, which need special handling in argparse. helptext, - pad=(0, 0, 0, help_position), + pad=(0, 0, 0, help_position - indent), ) ) From 8918cc33766a8499f610b0e8eab5637c52111dbf Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 26 Nov 2023 15:08:11 -0800 Subject: [PATCH 350/491] Add support for mixed unions (#96) * Add support for mixed unions * Resolve mixed union edge cases * Test coverage * Use older python syntax * Coverage --- tests/test_collections.py | 5 ++ tests/test_mixed_unions.py | 29 ++++--- tests/test_new_style_annotations_min_py39.py | 5 ++ .../test_collections_generated.py | 5 ++ .../test_mixed_unions_generated.py | 28 +++++-- ...ew_style_annotations_min_py39_generated.py | 5 ++ tyro/_calling.py | 25 +++++-- tyro/_cli.py | 2 +- tyro/_fields.py | 31 +++++++- tyro/_parsers.py | 14 +++- tyro/_resolver.py | 75 ++----------------- tyro/_strings.py | 4 +- tyro/_subcommand_matching.py | 2 +- 13 files changed, 124 insertions(+), 106 deletions(-) diff --git a/tests/test_collections.py b/tests/test_collections.py index 4f2d245a9..6f8a56a5f 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -487,3 +487,8 @@ def test_list_narrowing_direct() -> None: "four", 5, ] + + +def test_tuple_direct() -> None: + assert tyro.cli(Tuple[int, ...], args="1 2".split(" ")) == (1, 2) # type: ignore + assert tyro.cli(Tuple[int, int], args="1 2".split(" ")) == (1, 2) # type: ignore diff --git a/tests/test_mixed_unions.py b/tests/test_mixed_unions.py index 47fb35833..2da3329f7 100644 --- a/tests/test_mixed_unions.py +++ b/tests/test_mixed_unions.py @@ -1,16 +1,13 @@ """Tests for unsupported union types. -Unions like `int | str` or `SomeDataclassA | SomeDataclassB` are generally OK (note that -the latter will produce a pair of subcommands), but when we write things like -`int | SomeDataclassA` handling gets more complicated; see docstring for -`narrow_union_type()` in _resolvers.py. - -Hopefully we can fix/improve this in the future! +Unions like `int | str` or `SomeDataclassA | SomeDataclassB` are OK (note that the latter +will produce a pair of subcommands); when we write things like `int | SomeDataclassA` +handling gets more complicated but should still be supported! """ import dataclasses -from typing import Union +from typing import Any, Dict, List, Tuple, Union import pytest @@ -78,10 +75,24 @@ class DefaultSubparser: bc: Union[int, str, DefaultHTTPServer, DefaultSMTPServer] = 5 assert ( - tyro.cli(DefaultSubparser, args=["--x", "1", "--bc", "5"]) + tyro.cli(DefaultSubparser, args=["--x", "1", "bc:int", "5"]) == tyro.cli(DefaultSubparser, args=["--x", "1"]) == DefaultSubparser(x=1, bc=5) ) assert tyro.cli( - DefaultSubparser, args=["--x", "1", "--bc", "five"] + DefaultSubparser, args=["--x", "1", "bc:str", "five"] ) == DefaultSubparser(x=1, bc="five") + + +def test_with_fancy_types() -> None: + @dataclasses.dataclass + class Args: + y: int + + def main(x: Union[Tuple[int, ...], List[str], Args, Dict[str, int]]) -> Any: + return x + + assert tyro.cli(main, args="x:tuple-int-ellipsis 1 2 3".split(" ")) == (1, 2, 3) + assert tyro.cli(main, args="x:list-str 1 2 3".split(" ")) == ["1", "2", "3"] + assert tyro.cli(main, args="x:args --x.y 5".split(" ")) == Args(5) + assert tyro.cli(main, args="x:dict-str-int 1 2 3 4".split(" ")) == {"1": 2, "3": 4} diff --git a/tests/test_new_style_annotations_min_py39.py b/tests/test_new_style_annotations_min_py39.py index 9d04bb029..3dc584e75 100644 --- a/tests/test_new_style_annotations_min_py39.py +++ b/tests/test_new_style_annotations_min_py39.py @@ -62,3 +62,8 @@ def main( ] with pytest.raises(SystemExit): tyro.cli(main, args=["--help"]) + + +def test_tuple_direct() -> None: + assert tyro.cli(tuple[int, ...], args="1 2".split(" ")) == (1, 2) # type: ignore + assert tyro.cli(tuple[int, int], args="1 2".split(" ")) == (1, 2) # type: ignore diff --git a/tests/test_py311_generated/test_collections_generated.py b/tests/test_py311_generated/test_collections_generated.py index 2df1b6517..d4566bffd 100644 --- a/tests/test_py311_generated/test_collections_generated.py +++ b/tests/test_py311_generated/test_collections_generated.py @@ -486,3 +486,8 @@ def test_list_narrowing_direct() -> None: "four", 5, ] + + +def test_tuple_direct() -> None: + assert tyro.cli(Tuple[int, ...], args="1 2".split(" ")) == (1, 2) # type: ignore + assert tyro.cli(Tuple[int, int], args="1 2".split(" ")) == (1, 2) # type: ignore diff --git a/tests/test_py311_generated/test_mixed_unions_generated.py b/tests/test_py311_generated/test_mixed_unions_generated.py index 15272ec95..1b10b2945 100644 --- a/tests/test_py311_generated/test_mixed_unions_generated.py +++ b/tests/test_py311_generated/test_mixed_unions_generated.py @@ -1,15 +1,13 @@ """Tests for unsupported union types. -Unions like `int | str` or `SomeDataclassA | SomeDataclassB` are generally OK (note that -the latter will produce a pair of subcommands), but when we write things like -`int | SomeDataclassA` handling gets more complicated; see docstring for -`narrow_union_type()` in _resolvers.py. - -Hopefully we can fix/improve this in the future! +Unions like `int | str` or `SomeDataclassA | SomeDataclassB` are OK (note that the latter +will produce a pair of subcommands); when we write things like `int | SomeDataclassA` +handling gets more complicated but should still be supported! """ import dataclasses +from typing import Any, Dict, List, Tuple import pytest @@ -77,10 +75,24 @@ class DefaultSubparser: bc: int | str | DefaultHTTPServer | DefaultSMTPServer = 5 assert ( - tyro.cli(DefaultSubparser, args=["--x", "1", "--bc", "5"]) + tyro.cli(DefaultSubparser, args=["--x", "1", "bc:int", "5"]) == tyro.cli(DefaultSubparser, args=["--x", "1"]) == DefaultSubparser(x=1, bc=5) ) assert tyro.cli( - DefaultSubparser, args=["--x", "1", "--bc", "five"] + DefaultSubparser, args=["--x", "1", "bc:str", "five"] ) == DefaultSubparser(x=1, bc="five") + + +def test_with_fancy_types() -> None: + @dataclasses.dataclass + class Args: + y: int + + def main(x: Tuple[int, ...] | List[str] | Args | Dict[str, int]) -> Any: + return x + + assert tyro.cli(main, args="x:tuple-int-ellipsis 1 2 3".split(" ")) == (1, 2, 3) + assert tyro.cli(main, args="x:list-str 1 2 3".split(" ")) == ["1", "2", "3"] + assert tyro.cli(main, args="x:args --x.y 5".split(" ")) == Args(5) + assert tyro.cli(main, args="x:dict-str-int 1 2 3 4".split(" ")) == {"1": 2, "3": 4} diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py index d1e01c056..62b205ff5 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py @@ -62,3 +62,8 @@ def main( ] with pytest.raises(SystemExit): tyro.cli(main, args=["--help"]) + + +def test_tuple_direct() -> None: + assert tyro.cli(tuple[int, ...], args="1 2".split(" ")) == (1, 2) # type: ignore + assert tyro.cli(tuple[int, int], args="1 2".split(" ")) == (1, 2) # type: ignore diff --git a/tyro/_calling.py b/tyro/_calling.py index f9bbfbb30..a12718f18 100644 --- a/tyro/_calling.py +++ b/tyro/_calling.py @@ -38,7 +38,7 @@ def call_from_args( # Resolve the type of `f`, generate a field list. f, type_from_typevar, field_list = _fields.field_list_from_callable( - f=f, default_instance=default_instance + f=f, default_instance=default_instance, support_single_arg_types=True ) positional_args: List[Any] = [] @@ -225,14 +225,23 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore if unwrapped_f in (tuple, list, set): - assert len(positional_args) == 0 - # When tuples are used as nested structures (eg Tuple[SomeDataclass]), we - # use keyword arguments. - assert len(positional_args) == 0 - return unwrapped_f(kwargs.values()), consumed_keywords # type: ignore + if len(positional_args) > 0: + # Triggered when support_single_arg_types=True is used. + assert len(kwargs) == 0 + assert len(positional_args) == 1 + return positional_args[0], consumed_keywords # type: ignore + else: + assert len(positional_args) == 0 + return unwrapped_f(kwargs.values()), consumed_keywords # type: ignore elif unwrapped_f is dict: - assert len(positional_args) == 0 - return kwargs, consumed_keywords # type: ignore + if len(positional_args) > 0: + # Triggered when support_single_arg_types=True is used. + assert len(kwargs) == 0 + assert len(positional_args) == 1 + return unwrapped_f(*positional_args), consumed_keywords # type: ignore + else: + assert len(positional_args) == 0 + return kwargs, consumed_keywords # type: ignore else: if field_name_prefix == "": # Don't catch any errors for the "root" field. If main() in tyro.cli(main) diff --git a/tyro/_cli.py b/tyro/_cli.py index beb1bef0b..7c364db54 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -304,7 +304,7 @@ def _cli_impl( dataclasses.field(), ) f = dataclasses.make_dataclass( - cls_name="", + cls_name="dummy", fields=[(_strings.dummy_field_name, cast(type, f), dummy_field)], frozen=True, ) diff --git a/tyro/_fields.py b/tyro/_fields.py index d313c1980..f8269ebc9 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -62,7 +62,7 @@ class FieldDefinition: stripped.""" default: Any helptext: Optional[str] - markers: FrozenSet[_markers._Marker] + markers: FrozenSet[Any] custom_constructor: bool argconf: _confstruct._ArgConfiguration @@ -254,6 +254,7 @@ def is_nested_type( def field_list_from_callable( f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance, + support_single_arg_types: bool, ) -> Tuple[ Union[Callable, TypeForm[Any]], Dict[TypeVar, TypeForm], List[FieldDefinition] ]: @@ -273,7 +274,30 @@ def field_list_from_callable( field_list = _try_field_list_from_callable(f, default_instance) if isinstance(field_list, UnsupportedNestedTypeMessage): - raise _instantiators.UnsupportedTypeAnnotationError(field_list.message) + if support_single_arg_types: + return ( + f, + type_from_typevar, + [ + FieldDefinition( + intern_name="value", + extern_name="value", # Doesn't matter. + type_or_callable=f, + default=default_instance, + helptext="", + custom_constructor=False, + markers=frozenset( + (_markers.Positional, _markers._PositionalCall) + ), + argconf=_confstruct._ArgConfiguration( + None, None, None, None, None, None + ), + call_argname="", + ) + ], + ) + else: + raise _instantiators.UnsupportedTypeAnnotationError(field_list.message) # Recursively apply markers. _, parent_markers = _resolver.unwrap_annotated(f, _markers._Marker) @@ -352,7 +376,8 @@ def _try_field_list_from_callable( default_instance = found_subcommand_configs[0].default # Unwrap generics. - f, _ = _resolver.resolve_generic_types(f) + f, type_from_typevar = _resolver.resolve_generic_types(f) + f = _resolver.apply_type_from_typevar(f, type_from_typevar) f = _resolver.narrow_subtypes(f, default_instance) f = _resolver.narrow_collection_types(f, default_instance) f_origin = _resolver.unwrap_origin_strip_extras(cast(TypeForm, f)) diff --git a/tyro/_parsers.py b/tyro/_parsers.py index ec311c18e..671cbdb48 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -68,6 +68,7 @@ def from_callable_or_type( intern_prefix: str, extern_prefix: str, subcommand_prefix: str = "", + support_single_arg_types: bool = False, ) -> ParserSpecification: """Create a parser definition from a callable or type.""" @@ -79,7 +80,9 @@ def from_callable_or_type( # Resolve the type of `f`, generate a field list. f, type_from_typevar, field_list = _fields.field_list_from_callable( - f=f, default_instance=default_instance + f=f, + default_instance=default_instance, + support_single_arg_types=support_single_arg_types, ) # Cycle detection. @@ -343,6 +346,7 @@ def handle_field( [extern_prefix, field.extern_name] ), subcommand_prefix=subcommand_prefix, + support_single_arg_types=False, ) # (3) Handle primitive or fixed types. These produce a single argument! @@ -412,10 +416,11 @@ def from_field( ) ) - # Exit if we don't contain nested types. - if not all( + # Exit if we don't contain any nested types. + if not any( [ - _fields.is_nested_type(cast(type, o), _fields.MISSING_NONPROP) + o is not none_proxy + and _fields.is_nested_type(cast(type, o), _fields.MISSING_NONPROP) for o in options ] ): @@ -505,6 +510,7 @@ def from_field( intern_prefix=intern_prefix, extern_prefix=extern_prefix, subcommand_prefix=intern_prefix, + support_single_arg_types=True, ) # Apply prefix to helptext in nested classes in subparsers. diff --git a/tyro/_resolver.py b/tyro/_resolver.py index d5a0fb01f..5e9eed1cb 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -265,47 +265,12 @@ def apply_type_from_typevar( def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCallable: """Narrow union types. - This is a shim for failing more gracefully when we we're given one of two errors: - (A) A Union type that doesn't match the default value. - (B) An unsupported Union type, which mixes "nested" types (like dataclasses) with - non-"nested" types (like strings). + This is a shim for failing more gracefully when we we're given a Union type that + doesn't match the default value. - -- - For (A): - - We raise a warning, then add the type of the default value to the union. - Loosely motivated by: https://github.com/brentyi/tyro/issues/20 - - -- - For (B): - - When do we want to narrow Union types? - - Unions over nested types: no. - typ = NestedA | NestedB - => NestedA | NestedB can be converted to two subcommands. - - Unions over nested and not nested types: no. - typ = int | str - => int | str can be instantiated as a union. - - Unions over mixed nested / not nested types: if the default is a nested - type, strip out the non-nested ones. If the default is a non-nested - type, strip out the nested ones. - - typ = NestedA | int, default_instance = NestedA() - => NestedA - - typ = NestedA | int, default_instance = 5 - => int - - typ = NestedA | NestedB | int, default_instance = NestedA() - => NestedA - - This is a hack to get around the fact that we don't currently support - mixing nested types (eg `SomeDataclass`) and non-nested ones (eg `int` or - `int | str`) in unions. This should be supported in the future, but will - likely require a big code refactor.""" + In this case, we raise a warning, then add the type of the default value to the + union. Loosely motivated by: https://github.com/brentyi/tyro/issues/20 + """ if get_origin(typ) is not Union: return typ @@ -325,32 +290,4 @@ def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCalla except TypeError: pass - # (B) - is_nested = tuple( - map( - lambda option: _fields.is_nested_type( - option, - _fields.MISSING_NONPROP, - ), - options, - ) - ) - if type(None) in options: - none_index = options.index(type(None)) - is_nested_no_none = is_nested[:none_index] + is_nested[none_index + 1 :] - else: - is_nested_no_none = is_nested - - if all(is_nested_no_none) or not any(is_nested_no_none): - # Either all types are nested or none of them are. - return typ - else: - is_default_nested = _fields.is_nested_type(type(default_instance), default_instance) # type: ignore - out = Union.__getitem__( # type: ignore - tuple( - option - for option, nested in zip(get_args(typ), is_nested) - if nested is is_default_nested - ) - ) - return out # type: ignore + return typ diff --git a/tyro/_strings.py b/tyro/_strings.py index 83684d422..91a950662 100644 --- a/tyro/_strings.py +++ b/tyro/_strings.py @@ -112,9 +112,7 @@ def get_name(cls: Type) -> str: elif hasattr(cls, "__name__"): return hyphen_separated_from_camel_case(cls.__name__) else: - raise AssertionError( - f"Tried to interpret {cls} as a subcommand, but could not infer name" - ) + return hyphen_separated_from_camel_case(str(cls)) if len(type_from_typevar) == 0: return get_name(cls), prefix_name # type: ignore diff --git a/tyro/_subcommand_matching.py b/tyro/_subcommand_matching.py index 83ed38e08..8ba0fbf3d 100644 --- a/tyro/_subcommand_matching.py +++ b/tyro/_subcommand_matching.py @@ -73,7 +73,7 @@ def make( """From an object instance, return a data structure representing the types in the object.""" try: typ, _type_from_typevar, field_list = _fields.field_list_from_callable( - typ, default_instance=default_instance + typ, default_instance=default_instance, support_single_arg_types=False ) except _instantiators.UnsupportedTypeAnnotationError: return _TypeTree(typ, {}) From 1afc1303bdf714454c0f3383ab7b4508d7522e48 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 1 Dec 2023 01:51:33 -0800 Subject: [PATCH 351/491] Add test from #100 --- tests/test_base_configs_nested.py | 180 ++++++++++++++++++ .../test_base_configs_nested_generated.py | 179 +++++++++++++++++ 2 files changed, 359 insertions(+) create mode 100644 tests/test_base_configs_nested.py create mode 100644 tests/test_py311_generated/test_base_configs_nested_generated.py diff --git a/tests/test_base_configs_nested.py b/tests/test_base_configs_nested.py new file mode 100644 index 000000000..bb7cf0809 --- /dev/null +++ b/tests/test_base_configs_nested.py @@ -0,0 +1,180 @@ +from dataclasses import dataclass +from typing import Callable, Tuple, TYPE_CHECKING +from typing_extensions import Literal + +from torch import nn + +import tyro + + +@dataclass(frozen=True) +class AdamOptimizer: + learning_rate: float = 1e-3 + betas: Tuple[float, float] = (0.9, 0.999) + + +@dataclass(frozen=True) +class SGDOptimizer: + learning_rate: float = 1e-3 + momentum: float = 0.9 + + +@dataclass(frozen=True) +class DataConfig: + test: int = 0 + + +@dataclass(frozen=True) +class ExperimentConfig: + # Dataset to run experiment on. + dataset: Literal["mnist", "imagenet-50"] + + # Optimizer parameters. + optimizer: AdamOptimizer + + # Model size. + num_layers: int + units: int + + # Batch size. + batch_size: int + + # Total number of training steps. + train_steps: int + + # Random seed. This is helpful for making sure that our experiments are all + # reproducible! + seed: int + + # Activation to use. Not specifiable via the commandline. + activation: Callable[[], nn.Module] + + +DataConfigDataParserUnion = tyro.extras.subcommand_type_from_defaults( + { + "small-data": DataConfig( + test=2221, + ), + "big-data": DataConfig( + test=2, + ), + }, + prefix_names=False, # Omit prefixes in subcommands themselves. +) + + +# Note that we could also define this library using separate YAML files (similar to +# `config_path`/`config_name` in Hydra), but staying in Python enables seamless type +# checking + IDE support. +ExperimentConfigDataParserUnion = tyro.extras.subcommand_type_from_defaults( + { + "small": ExperimentConfig( + dataset="mnist", + optimizer=AdamOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + seed=0, + activation=nn.ReLU, + ), + "big": ExperimentConfig( + dataset="imagenet-50", + optimizer=AdamOptimizer(), + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=0, + activation=nn.GELU, + ), + }, + prefix_names=False, # Omit prefixes in subcommands themselves. +) + + +if TYPE_CHECKING: + AnnotatedDataParserUnion = DataConfig + AnnotatedExperimentParserUnion = ExperimentConfig +else: + AnnotatedDataParserUnion = tyro.conf.OmitSubcommandPrefixes[ + DataConfigDataParserUnion + ] # Omit prefixes of flags in subcommands. + AnnotatedExperimentParserUnion = tyro.conf.OmitSubcommandPrefixes[ + ExperimentConfigDataParserUnion + ] # Omit prefixes of flags in subcommands. + + +@dataclass +class BaseConfig: + # The name of the experiment to run. + experiment: str + + # The path to the output directory. + output_dir: str + + # The experiment configuration. + experiment_config: AnnotatedExperimentParserUnion + + # The experiment configuration. + data_config: AnnotatedDataParserUnion = DataConfig() + + +def test_base_configs_nested() -> None: + def main(cfg: BaseConfig) -> BaseConfig: + return cfg + + assert tyro.cli( + main, + args="--cfg.experiment test --cfg.output-dir test small".split(" "), + ) == BaseConfig( + "test", + "test", + ExperimentConfig( + dataset="mnist", + optimizer=AdamOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + seed=0, + activation=nn.ReLU, + ), + DataConfig(0), + ) + assert tyro.cli( + main, + args="--cfg.experiment test --cfg.output-dir test small small-data".split(" "), + ) == BaseConfig( + "test", + "test", + ExperimentConfig( + dataset="mnist", + optimizer=AdamOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + seed=0, + activation=nn.ReLU, + ), + DataConfig(2221), + ) + assert tyro.cli( + main, + args="--cfg.experiment test --cfg.output-dir test small big-data".split(" "), + ) == BaseConfig( + "test", + "test", + ExperimentConfig( + dataset="mnist", + optimizer=AdamOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + seed=0, + activation=nn.ReLU, + ), + DataConfig(2), + ) diff --git a/tests/test_py311_generated/test_base_configs_nested_generated.py b/tests/test_py311_generated/test_base_configs_nested_generated.py new file mode 100644 index 000000000..ba83318e2 --- /dev/null +++ b/tests/test_py311_generated/test_base_configs_nested_generated.py @@ -0,0 +1,179 @@ +from dataclasses import dataclass +from typing import TYPE_CHECKING, Callable, Literal, Tuple + +from torch import nn + +import tyro + + +@dataclass(frozen=True) +class AdamOptimizer: + learning_rate: float = 1e-3 + betas: Tuple[float, float] = (0.9, 0.999) + + +@dataclass(frozen=True) +class SGDOptimizer: + learning_rate: float = 1e-3 + momentum: float = 0.9 + + +@dataclass(frozen=True) +class DataConfig: + test: int = 0 + + +@dataclass(frozen=True) +class ExperimentConfig: + # Dataset to run experiment on. + dataset: Literal["mnist", "imagenet-50"] + + # Optimizer parameters. + optimizer: AdamOptimizer + + # Model size. + num_layers: int + units: int + + # Batch size. + batch_size: int + + # Total number of training steps. + train_steps: int + + # Random seed. This is helpful for making sure that our experiments are all + # reproducible! + seed: int + + # Activation to use. Not specifiable via the commandline. + activation: Callable[[], nn.Module] + + +DataConfigDataParserUnion = tyro.extras.subcommand_type_from_defaults( + { + "small-data": DataConfig( + test=2221, + ), + "big-data": DataConfig( + test=2, + ), + }, + prefix_names=False, # Omit prefixes in subcommands themselves. +) + + +# Note that we could also define this library using separate YAML files (similar to +# `config_path`/`config_name` in Hydra), but staying in Python enables seamless type +# checking + IDE support. +ExperimentConfigDataParserUnion = tyro.extras.subcommand_type_from_defaults( + { + "small": ExperimentConfig( + dataset="mnist", + optimizer=AdamOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + seed=0, + activation=nn.ReLU, + ), + "big": ExperimentConfig( + dataset="imagenet-50", + optimizer=AdamOptimizer(), + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=0, + activation=nn.GELU, + ), + }, + prefix_names=False, # Omit prefixes in subcommands themselves. +) + + +if TYPE_CHECKING: + AnnotatedDataParserUnion = DataConfig + AnnotatedExperimentParserUnion = ExperimentConfig +else: + AnnotatedDataParserUnion = tyro.conf.OmitSubcommandPrefixes[ + DataConfigDataParserUnion + ] # Omit prefixes of flags in subcommands. + AnnotatedExperimentParserUnion = tyro.conf.OmitSubcommandPrefixes[ + ExperimentConfigDataParserUnion + ] # Omit prefixes of flags in subcommands. + + +@dataclass +class BaseConfig: + # The name of the experiment to run. + experiment: str + + # The path to the output directory. + output_dir: str + + # The experiment configuration. + experiment_config: AnnotatedExperimentParserUnion + + # The experiment configuration. + data_config: AnnotatedDataParserUnion = DataConfig() + + +def test_base_configs_nested() -> None: + def main(cfg: BaseConfig) -> BaseConfig: + return cfg + + assert tyro.cli( + main, + args="--cfg.experiment test --cfg.output-dir test small".split(" "), + ) == BaseConfig( + "test", + "test", + ExperimentConfig( + dataset="mnist", + optimizer=AdamOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + seed=0, + activation=nn.ReLU, + ), + DataConfig(0), + ) + assert tyro.cli( + main, + args="--cfg.experiment test --cfg.output-dir test small small-data".split(" "), + ) == BaseConfig( + "test", + "test", + ExperimentConfig( + dataset="mnist", + optimizer=AdamOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + seed=0, + activation=nn.ReLU, + ), + DataConfig(2221), + ) + assert tyro.cli( + main, + args="--cfg.experiment test --cfg.output-dir test small big-data".split(" "), + ) == BaseConfig( + "test", + "test", + ExperimentConfig( + dataset="mnist", + optimizer=AdamOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + seed=0, + activation=nn.ReLU, + ), + DataConfig(2), + ) From e7ca5a2e5a4a3d25f9c4119b12bdc55cdd127d3e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 11 Dec 2023 19:11:10 +0000 Subject: [PATCH 352/491] Tooling: ruff, pyright --- .github/workflows/lint.yml | 17 ----------- .github/workflows/pyright.yml | 29 ++++++++++++++++++ .github/workflows/ruff.yml | 27 +++++++++++++++++ README.md | 5 ++-- docs/source/conf.py | 2 +- docs/source/index.md | 9 ++++-- pyproject.toml | 22 +++++++++++++- tests/test_attrs.py | 1 + tests/test_collections.py | 4 +-- tests/test_conf.py | 2 +- tests/test_flax_min_py38.py | 4 ++- tests/test_helptext.py | 12 ++++---- tests/test_nested.py | 8 +++-- tests/test_nested_in_containers.py | 30 ++++++++----------- tests/test_new_style_annotations_min_py310.py | 2 +- tests/test_new_style_annotations_min_py39.py | 2 +- tests/test_py311_generated/_generate.py | 5 ++-- .../test_attrs_generated.py | 1 + .../test_collections_generated.py | 4 +-- .../test_conf_generated.py | 2 +- .../test_flax_min_py38_generated.py | 4 ++- .../test_helptext_generated.py | 16 ++++++---- .../test_nested_generated.py | 19 ++++-------- .../test_nested_in_containers_generated.py | 30 ++++++++----------- ...w_style_annotations_min_py310_generated.py | 2 +- ...ew_style_annotations_min_py39_generated.py | 2 +- .../test_pydantic_generated.py | 1 + .../test_union_from_mapping_generated.py | 9 +++--- tests/test_pydantic.py | 1 + tests/test_union_from_mapping.py | 9 +++--- tyro/_argparse_formatter.py | 10 +++---- tyro/_arguments.py | 7 ++++- tyro/_cli.py | 6 +++- tyro/_fields.py | 7 ++--- tyro/_instantiators.py | 3 +- tyro/_parsers.py | 8 ++--- tyro/_resolver.py | 11 ++++--- tyro/extras/_serialization.py | 9 +----- 38 files changed, 199 insertions(+), 143 deletions(-) delete mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/pyright.yml create mode 100644 .github/workflows/ruff.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 4f5170c1a..000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: lint - -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - black-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - name: Black Code Formatter - uses: lgeiger/black-action@master - with: - args: ". --check" diff --git a/.github/workflows/pyright.yml b/.github/workflows/pyright.yml new file mode 100644 index 000000000..3f96b89d3 --- /dev/null +++ b/.github/workflows/pyright.yml @@ -0,0 +1,29 @@ +name: pyright + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + pyright: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -e ".[dev]" + pip install -r docs/requirements.txt + - name: Run pyright + run: | + pyright . diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 000000000..001833420 --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,27 @@ +name: ruff + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: "Set up Python 3.10" + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -e ".[dev]" + - name: Ruff check + run: | + ruff check --output-format github + - name: Ruff format + run: | + ruff format --diff diff --git a/README.md b/README.md index 4b8c19b73..c44f76d5a 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,9 @@

build - mypy - lint + mypy + pyright + ruff codecov diff --git a/docs/source/conf.py b/docs/source/conf.py index 5b8366b4b..ce1ed1118 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -390,7 +390,7 @@ def docstring(app, what, name, obj, options, lines): md = "\n".join(lines) rst = m2r2.convert(md) lines.clear() - lines += rst.splitlines() + lines += rst.splitlines() # type: ignore def setup(app): diff --git a/docs/source/index.md b/docs/source/index.md index d274ce0f7..e4a3dfef6 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -1,6 +1,6 @@ # tyro -|build| |nbsp| |mypy| |nbsp| |lint| |nbsp| |coverage| |nbsp| |versions| +|build| |nbsp| |ruff| |nbsp| |mypy| |nbsp| |pyright| |nbsp| |coverage| |nbsp| |versions| :code:`tyro` is a tool for generating command-line interfaces and configuration objects in Python. @@ -140,10 +140,13 @@ To get started, we recommend browsing the examples to the left. .. |build| image:: https://github.com/brentyi/tyro/workflows/build/badge.svg :alt: Build status icon :target: https://github.com/brentyi/tyro -.. |mypy| image:: https://github.com/brentyi/tyro/workflows/mypy/badge.svg?branch=main +.. |mypy| image:: https://github.com/brentyi/tyro/workflows/mypy/badge.svg :alt: Mypy status icon :target: https://github.com/brentyi/tyro -.. |lint| image:: https://github.com/brentyi/tyro/workflows/lint/badge.svg +.. |pyright| image:: https://github.com/brentyi/tyro/workflows/pyright/badge.svg + :alt: Mypy status icon + :target: https://github.com/brentyi/tyro +.. |ruff| image:: https://github.com/brentyi/tyro/workflows/ruff/badge.svg :alt: Lint status icon :target: https://github.com/brentyi/tyro .. |coverage| image:: https://codecov.io/gh/brentyi/tyro/branch/main/graph/badge.svg diff --git a/pyproject.toml b/pyproject.toml index c426672f6..a77b185e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ dev = [ "attrs>=21.4.0", "torch>=1.10.0", "pyright>=1.1.264", + "ruff>=0.1.7", "mypy>=1.4.1", "numpy>=1.20.0", # As of 7/27/2023, flax install fails for Python 3.7 without pinning to an @@ -104,8 +105,27 @@ exclude_lines = [ ] [tool.ruff] +select = [ + "E", # pycodestyle errors. + "F", # Pyflakes rules. + "PLC", # Pylint convention warnings. + "PLE", # Pylint errors. + "PLR", # Pylint refactor recommendations. + "PLW", # Pylint warnings. +] ignore = [ - "E501", # Ignore line length errors. + "E741", # Ambiguous variable name. (l, O, or I) + "E501", # Line too long. + "PLR2004", # Magic value used in comparison. + "PLR0915", # Too many statements. + "PLR0913", # Too many arguments. + "PLC0414", # Import alias does not rename variable. (this is used for exporting names) + "PLC1901", # Use falsey strings. + "PLR5501", # Use `elif` instead of `else if`. + "PLR0911", # Too many return statements. + "PLR0912", # Too many branches. + "PLW0603", # Global statement updates are discouraged. + "PLW2901", # For loop variable overwritten. ] [tool.pytest.ini_options] diff --git a/tests/test_attrs.py b/tests/test_attrs.py index c8a2ed445..2cd154a73 100644 --- a/tests/test_attrs.py +++ b/tests/test_attrs.py @@ -8,6 +8,7 @@ from attrs import define, field import tyro +import tyro._strings def test_attrs_basic() -> None: diff --git a/tests/test_collections.py b/tests/test_collections.py index 6f8a56a5f..ce1719dc0 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -395,7 +395,7 @@ def main( Union[Tuple[int, int], Tuple[str, str]], ] ] - ] = None + ] = None, ) -> Any: return x @@ -426,7 +426,7 @@ def test_double_dict_no_annotation() -> None: def main( x: Dict[str, Any] = { "wow": {"int": 5, "str": "5"}, - } + }, ): return x diff --git a/tests/test_conf.py b/tests/test_conf.py index 492f0501e..929fcfc80 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -501,7 +501,7 @@ def test_suppress_auto_fixed() -> None: class Struct: a: int = 5 - def b(x): + def b(self, x): return 5 def main(x: tyro.conf.SuppressFixed[Any] = Struct()): diff --git a/tests/test_flax_min_py38.py b/tests/test_flax_min_py38.py index 4448c556b..6f944693a 100644 --- a/tests/test_flax_min_py38.py +++ b/tests/test_flax_min_py38.py @@ -1,4 +1,6 @@ """Tests initializing flax modules directly via tyro.""" +from typing import cast + import jax import pytest from flax import linen as nn @@ -45,7 +47,7 @@ def test_ok(): x = jnp.zeros((10, 4)) params = network.init(jax.random.PRNGKey(0), x) - assert network.apply(params, x).shape == (10, 3) + assert cast(jax.Array, network.apply(params, x)).shape == (10, 3) helptext = get_helptext(Classifier) assert "parent" not in helptext diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 56fa060cc..fe9f6cc3c 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -5,7 +5,7 @@ from collections.abc import Callable from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar, Union, cast -import torch.nn as nn +from torch import nn from helptext_utils import get_helptext from typing_extensions import Annotated, Literal @@ -463,7 +463,7 @@ def main( Literal[0, 1, 2, 3], Literal["hey,there", "hello"], List[int], - ] + ], ) -> None: pass @@ -477,7 +477,7 @@ def main( x: Tuple[ Literal[0, 1, 2, 3], Union[int, str], - ] + ], ) -> None: pass @@ -490,7 +490,7 @@ def main( x: Union[ Literal[0, 1, 2, 3], Union[Tuple[int, int], Tuple[str]], - ] + ], ) -> None: pass @@ -504,7 +504,7 @@ def main( Literal[0, 1, 2, 3], Union[Tuple[int, int], Tuple[str, str, str]], Literal[True], - ] + ], ) -> None: pass @@ -514,7 +514,7 @@ def main( def test_metavar_5() -> None: def main( - x: List[Union[Tuple[int, int], Tuple[str, str]]] = [(1, 1), (2, 2)] + x: List[Union[Tuple[int, int], Tuple[str, str]]] = [(1, 1), (2, 2)], ) -> None: pass diff --git a/tests/test_nested.py b/tests/test_nested.py index 2db8df02b..7d94ee6a6 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -280,7 +280,8 @@ class Subparser: bc: Union[HTTPServer, SMTPServer] assert tyro.cli( - Union[HTTPServer, SMTPServer], args=["http-server", "--y", "3"] # type: ignore + Union[HTTPServer, SMTPServer], + args=["http-server", "--y", "3"], # type: ignore ) == HTTPServer(y=3) @@ -403,7 +404,8 @@ class DefaultSubparser: # Should give us a bunch of warnings! with pytest.warns(UserWarning): assert tyro.cli(DefaultSubparser, args=["--x", "1"]) == DefaultSubparser( - 1, 5 # type: ignore + 1, + 5, # type: ignore ) @@ -896,7 +898,7 @@ def main( "num_epochs": 20, "batch_size": 64, } - ) + ), ): return x diff --git a/tests/test_nested_in_containers.py b/tests/test_nested_in_containers.py index a3eab0e1e..819998176 100644 --- a/tests/test_nested_in_containers.py +++ b/tests/test_nested_in_containers.py @@ -166,7 +166,7 @@ def main( "red": Color(255, 0, 0), "green": Color(0, 255, 0), "blue": Color(0, 0, 255), - } + }, ) -> Any: return x @@ -182,7 +182,7 @@ def main( 0: Color(255, 0, 0), 1: Color(0, 255, 0), 2: Color(0, 0, 255), - } + }, ) -> Any: return x @@ -201,7 +201,7 @@ def main( ColorType.RED: Color(255, 0, 0), ColorType.GREEN: Color(0, 255, 0), ColorType.BLUE: Color(0, 0, 255), - } + }, ) -> Any: return x @@ -218,7 +218,7 @@ def main( "red": (Color(255, 0, 0), 9), "green": (Color(0, 255, 0), 10), "blue": (Color(0, 0, 255), 12), - } + }, ) -> Any: return x @@ -263,7 +263,7 @@ def main( x: Tuple[GenericColor[float], GenericColor[int]] = ( GenericColor(0.5, 0.2, 0.3), GenericColor[int](25, 2, 3), # The subscript should be optional. - ) + ), ) -> Any: return x @@ -290,7 +290,7 @@ def main( x: Tuple[GenericColor, ...] = ( GenericColor[float](0.5, 0.2, 0.3), GenericColor[int](25, 2, 3), - ) + ), ) -> Any: return x @@ -317,16 +317,14 @@ def main( x: Dict[str, GenericColor] = { "float": GenericColor(0.5, 0.2, 0.3), "int": GenericColor[int](25, 2, 3), - } + }, ) -> Any: return x assert tyro.cli( main, args="--x.float.g 0.1".split(" "), - )[ - "float" - ] == GenericColor(0.5, 0.1, 0.3) + )["float"] == GenericColor(0.5, 0.1, 0.3) assert tyro.cli( main, args="--x.int.g 0".split(" "), @@ -348,16 +346,14 @@ def main( "float": GenericColor(0.5, 0.2, 0.3), "int": GenericColor[int](25, 2, 3), } - } + }, ) -> Any: return x assert tyro.cli( main, args="--x.hello.float.g 0.1".split(" "), - )["hello"][ - "float" - ] == GenericColor(0.5, 0.1, 0.3) + )["hello"]["float"] == GenericColor(0.5, 0.1, 0.3) assert tyro.cli( main, args="--x.hello.int.g 0".split(" "), @@ -373,13 +369,11 @@ def main( "a": Color(5, 2, 3), "b": Color(25, 2, 3), } - } + }, ) -> Any: return x assert tyro.cli( main, args="--x.hello.a.g 1".split(" "), - )["hello"][ - "a" - ] == Color(5, 1, 3) + )["hello"]["a"] == Color(5, 1, 3) diff --git a/tests/test_new_style_annotations_min_py310.py b/tests/test_new_style_annotations_min_py310.py index 94d0fc6f2..dc17e3db4 100644 --- a/tests/test_new_style_annotations_min_py310.py +++ b/tests/test_new_style_annotations_min_py310.py @@ -48,7 +48,7 @@ def main( Literal[3, 4], tuple[int, int] | tuple[str, str], ] - ] = None + ] = None, ) -> Any: return x diff --git a/tests/test_new_style_annotations_min_py39.py b/tests/test_new_style_annotations_min_py39.py index 3dc584e75..9b8908a35 100644 --- a/tests/test_new_style_annotations_min_py39.py +++ b/tests/test_new_style_annotations_min_py39.py @@ -48,7 +48,7 @@ def main( Union[tuple[int, int], tuple[str, str]], ] ] - ] = None + ] = None, ) -> Any: return x diff --git a/tests/test_py311_generated/_generate.py b/tests/test_py311_generated/_generate.py index 6b80bb616..54980a3c1 100644 --- a/tests/test_py311_generated/_generate.py +++ b/tests/test_py311_generated/_generate.py @@ -39,6 +39,5 @@ ) out_path.write_text(content) - subprocess.run(["isort", str(out_path)]) - subprocess.run(["black", str(out_path)]) - subprocess.run(["ruff", "--fix", str(out_path)]) + subprocess.run(["ruff", "format", str(out_path)], check=True) + subprocess.run(["ruff", "--fix", str(out_path)], check=True) diff --git a/tests/test_py311_generated/test_attrs_generated.py b/tests/test_py311_generated/test_attrs_generated.py index c8a2ed445..2cd154a73 100644 --- a/tests/test_py311_generated/test_attrs_generated.py +++ b/tests/test_py311_generated/test_attrs_generated.py @@ -8,6 +8,7 @@ from attrs import define, field import tyro +import tyro._strings def test_attrs_basic() -> None: diff --git a/tests/test_py311_generated/test_collections_generated.py b/tests/test_py311_generated/test_collections_generated.py index d4566bffd..ba494b85c 100644 --- a/tests/test_py311_generated/test_collections_generated.py +++ b/tests/test_py311_generated/test_collections_generated.py @@ -394,7 +394,7 @@ def main( Tuple[int, int] | Tuple[str, str], ] ] - ] = None + ] = None, ) -> Any: return x @@ -425,7 +425,7 @@ def test_double_dict_no_annotation() -> None: def main( x: Dict[str, Any] = { "wow": {"int": 5, "str": "5"}, - } + }, ): return x diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index 6d0025441..d851fa5eb 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -496,7 +496,7 @@ def test_suppress_auto_fixed() -> None: class Struct: a: int = 5 - def b(x): + def b(self, x): return 5 def main(x: tyro.conf.SuppressFixed[Any] = Struct()): diff --git a/tests/test_py311_generated/test_flax_min_py38_generated.py b/tests/test_py311_generated/test_flax_min_py38_generated.py index 4448c556b..6f944693a 100644 --- a/tests/test_py311_generated/test_flax_min_py38_generated.py +++ b/tests/test_py311_generated/test_flax_min_py38_generated.py @@ -1,4 +1,6 @@ """Tests initializing flax modules directly via tyro.""" +from typing import cast + import jax import pytest from flax import linen as nn @@ -45,7 +47,7 @@ def test_ok(): x = jnp.zeros((10, 4)) params = network.init(jax.random.PRNGKey(0), x) - assert network.apply(params, x).shape == (10, 3) + assert cast(jax.Array, network.apply(params, x)).shape == (10, 3) helptext = get_helptext(Classifier) assert "parent" not in helptext diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py index 097999859..aded14844 100644 --- a/tests/test_py311_generated/test_helptext_generated.py +++ b/tests/test_py311_generated/test_helptext_generated.py @@ -16,8 +16,8 @@ cast, ) -import torch.nn as nn from helptext_utils import get_helptext +from torch import nn def test_helptext() -> None: @@ -469,7 +469,7 @@ def main(x: Literal[0, 1, 2, 3] | Tuple[int, int]) -> None: def test_metavar_1() -> None: def main( - x: Literal[0, 1, 2, 3] | Literal["hey,there", "hello"] | List[int] + x: Literal[0, 1, 2, 3] | Literal["hey,there", "hello"] | List[int], ) -> None: pass @@ -483,7 +483,7 @@ def main( x: Tuple[ Literal[0, 1, 2, 3], int | str, - ] + ], ) -> None: pass @@ -492,7 +492,9 @@ def main( def test_metavar_3() -> None: - def main(x: Literal[0, 1, 2, 3] | Tuple[int, int] | Tuple[str]) -> None: + def main( + x: Literal[0, 1, 2, 3] | Tuple[int, int] | Tuple[str], + ) -> None: pass helptext = get_helptext(main) @@ -501,7 +503,7 @@ def main(x: Literal[0, 1, 2, 3] | Tuple[int, int] | Tuple[str]) -> None: def test_metavar_4() -> None: def main( - x: Literal[0, 1, 2, 3] | Tuple[int, int] | Tuple[str, str, str] | Literal[True] + x: Literal[0, 1, 2, 3] | Tuple[int, int] | Tuple[str, str, str] | Literal[True], ) -> None: pass @@ -510,7 +512,9 @@ def main( def test_metavar_5() -> None: - def main(x: List[Tuple[int, int] | Tuple[str, str]] = [(1, 1), (2, 2)]) -> None: + def main( + x: List[Tuple[int, int] | Tuple[str, str]] = [(1, 1), (2, 2)], + ) -> None: pass helptext = get_helptext(main) diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index fed6d61dc..7574214d6 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -1,14 +1,5 @@ import dataclasses -from typing import ( - Annotated, - Any, - Generic, - Literal, - Mapping, - Optional, - Tuple, - TypeVar, -) +from typing import Annotated, Any, Generic, Literal, Mapping, Optional, Tuple, TypeVar import pytest from frozendict import frozendict # type: ignore @@ -288,7 +279,8 @@ class Subparser: bc: HTTPServer | SMTPServer assert tyro.cli( - HTTPServer | SMTPServer, args=["http-server", "--y", "3"] # type: ignore + HTTPServer | SMTPServer, + args=["http-server", "--y", "3"], # type: ignore ) == HTTPServer(y=3) @@ -411,7 +403,8 @@ class DefaultSubparser: # Should give us a bunch of warnings! with pytest.warns(UserWarning): assert tyro.cli(DefaultSubparser, args=["--x", "1"]) == DefaultSubparser( - 1, 5 # type: ignore + 1, + 5, # type: ignore ) @@ -904,7 +897,7 @@ def main( "num_epochs": 20, "batch_size": 64, } - ) + ), ): return x diff --git a/tests/test_py311_generated/test_nested_in_containers_generated.py b/tests/test_py311_generated/test_nested_in_containers_generated.py index a3eab0e1e..819998176 100644 --- a/tests/test_py311_generated/test_nested_in_containers_generated.py +++ b/tests/test_py311_generated/test_nested_in_containers_generated.py @@ -166,7 +166,7 @@ def main( "red": Color(255, 0, 0), "green": Color(0, 255, 0), "blue": Color(0, 0, 255), - } + }, ) -> Any: return x @@ -182,7 +182,7 @@ def main( 0: Color(255, 0, 0), 1: Color(0, 255, 0), 2: Color(0, 0, 255), - } + }, ) -> Any: return x @@ -201,7 +201,7 @@ def main( ColorType.RED: Color(255, 0, 0), ColorType.GREEN: Color(0, 255, 0), ColorType.BLUE: Color(0, 0, 255), - } + }, ) -> Any: return x @@ -218,7 +218,7 @@ def main( "red": (Color(255, 0, 0), 9), "green": (Color(0, 255, 0), 10), "blue": (Color(0, 0, 255), 12), - } + }, ) -> Any: return x @@ -263,7 +263,7 @@ def main( x: Tuple[GenericColor[float], GenericColor[int]] = ( GenericColor(0.5, 0.2, 0.3), GenericColor[int](25, 2, 3), # The subscript should be optional. - ) + ), ) -> Any: return x @@ -290,7 +290,7 @@ def main( x: Tuple[GenericColor, ...] = ( GenericColor[float](0.5, 0.2, 0.3), GenericColor[int](25, 2, 3), - ) + ), ) -> Any: return x @@ -317,16 +317,14 @@ def main( x: Dict[str, GenericColor] = { "float": GenericColor(0.5, 0.2, 0.3), "int": GenericColor[int](25, 2, 3), - } + }, ) -> Any: return x assert tyro.cli( main, args="--x.float.g 0.1".split(" "), - )[ - "float" - ] == GenericColor(0.5, 0.1, 0.3) + )["float"] == GenericColor(0.5, 0.1, 0.3) assert tyro.cli( main, args="--x.int.g 0".split(" "), @@ -348,16 +346,14 @@ def main( "float": GenericColor(0.5, 0.2, 0.3), "int": GenericColor[int](25, 2, 3), } - } + }, ) -> Any: return x assert tyro.cli( main, args="--x.hello.float.g 0.1".split(" "), - )["hello"][ - "float" - ] == GenericColor(0.5, 0.1, 0.3) + )["hello"]["float"] == GenericColor(0.5, 0.1, 0.3) assert tyro.cli( main, args="--x.hello.int.g 0".split(" "), @@ -373,13 +369,11 @@ def main( "a": Color(5, 2, 3), "b": Color(25, 2, 3), } - } + }, ) -> Any: return x assert tyro.cli( main, args="--x.hello.a.g 1".split(" "), - )["hello"][ - "a" - ] == Color(5, 1, 3) + )["hello"]["a"] == Color(5, 1, 3) diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py index 94d0fc6f2..dc17e3db4 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py @@ -48,7 +48,7 @@ def main( Literal[3, 4], tuple[int, int] | tuple[str, str], ] - ] = None + ] = None, ) -> Any: return x diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py index 62b205ff5..0fe423b59 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py39_generated.py @@ -48,7 +48,7 @@ def main( tuple[int, int] | tuple[str, str], ] ] - ] = None + ] = None, ) -> Any: return x diff --git a/tests/test_py311_generated/test_pydantic_generated.py b/tests/test_py311_generated/test_pydantic_generated.py index 18e2a51bf..62ecbff09 100644 --- a/tests/test_py311_generated/test_pydantic_generated.py +++ b/tests/test_py311_generated/test_pydantic_generated.py @@ -7,6 +7,7 @@ from pydantic import BaseModel, Field import tyro +import tyro._strings def test_pydantic() -> None: diff --git a/tests/test_py311_generated/test_union_from_mapping_generated.py b/tests/test_py311_generated/test_union_from_mapping_generated.py index 841710e78..0b8b6b3c9 100644 --- a/tests/test_py311_generated/test_union_from_mapping_generated.py +++ b/tests/test_py311_generated/test_union_from_mapping_generated.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Optional +from typing import TYPE_CHECKING, Optional import tyro @@ -30,9 +30,10 @@ def test_union_from_mapping_in_function(): "three": A(3), } - # Hack for mypy. Not needed for pyright. - ConfigUnion = A - ConfigUnion = tyro.extras.subcommand_type_from_defaults(base_configs) # type: ignore + if TYPE_CHECKING: + ConfigUnion = A + else: + ConfigUnion = tyro.extras.subcommand_type_from_defaults(base_configs) def main(config: ConfigUnion, flag: bool = False) -> Optional[A]: if flag: diff --git a/tests/test_pydantic.py b/tests/test_pydantic.py index 18e2a51bf..62ecbff09 100644 --- a/tests/test_pydantic.py +++ b/tests/test_pydantic.py @@ -7,6 +7,7 @@ from pydantic import BaseModel, Field import tyro +import tyro._strings def test_pydantic() -> None: diff --git a/tests/test_union_from_mapping.py b/tests/test_union_from_mapping.py index 841710e78..0b8b6b3c9 100644 --- a/tests/test_union_from_mapping.py +++ b/tests/test_union_from_mapping.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Optional +from typing import TYPE_CHECKING, Optional import tyro @@ -30,9 +30,10 @@ def test_union_from_mapping_in_function(): "three": A(3), } - # Hack for mypy. Not needed for pyright. - ConfigUnion = A - ConfigUnion = tyro.extras.subcommand_type_from_defaults(base_configs) # type: ignore + if TYPE_CHECKING: + ConfigUnion = A + else: + ConfigUnion = tyro.extras.subcommand_type_from_defaults(base_configs) def main(config: ConfigUnion, flag: bool = False) -> Optional[A]: if flag: diff --git a/tyro/_argparse_formatter.py b/tyro/_argparse_formatter.py index b68bf93ac..3ad28ff88 100644 --- a/tyro/_argparse_formatter.py +++ b/tyro/_argparse_formatter.py @@ -66,9 +66,7 @@ def set_accent_color(accent_color: Optional[str]) -> None: THEME.helptext = Style(dim=True) THEME.helptext_required = Style(color="bright_red", bold=True) THEME.helptext_default = Style( - color="cyan" - if accent_color != "cyan" - else "magenta" + color="cyan" if accent_color != "cyan" else "magenta" # Another option: make default color match accent color. This is maybe more # visually consistent, but harder to read. # color=accent_color if accent_color is not None else "cyan", @@ -139,7 +137,8 @@ def _recursive_arg_search( and callable(arg.lowered.action) ): option_strings = arg.lowered.action( - option_strings, dest="" # dest should not matter. + option_strings, + dest="", # dest should not matter. ).option_strings arguments.append( @@ -369,7 +368,6 @@ def consume_optional(start_index): # Manually track unused arguments to assist with error messages # later. if not self._parsing_known_args: - global global_unrecognized_args global_unrecognized_args.append(option_string) # @@ -918,7 +916,7 @@ def format_help(self): return out @override - class _Section(object): + class _Section(object): # type: ignore def __init__(self, formatter, parent, heading=None): self.formatter = formatter self.parent = parent diff --git a/tyro/_arguments.py b/tyro/_arguments.py index 1c14b9a35..bafdda39a 100644 --- a/tyro/_arguments.py +++ b/tyro/_arguments.py @@ -255,7 +255,12 @@ def _rule_handle_boolean_flags( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> LoweredArgumentDefinition: - if _resolver.apply_type_from_typevar(arg.field.type_or_callable, arg.type_from_typevar) is not bool: # type: ignore + if ( + _resolver.apply_type_from_typevar( + arg.field.type_or_callable, arg.type_from_typevar + ) + is not bool + ): # type: ignore return lowered if ( diff --git a/tyro/_cli.py b/tyro/_cli.py index 7c364db54..bfa5dc75f 100644 --- a/tyro/_cli.py +++ b/tyro/_cli.py @@ -270,7 +270,11 @@ def _cli_impl( return_parser: bool, return_unknown_args: bool, **deprecated_kwargs, -) -> Union[OutT, argparse.ArgumentParser, Tuple[OutT, List[str]],]: +) -> Union[ + OutT, + argparse.ArgumentParser, + Tuple[OutT, List[str]], +]: """Helper for stitching the `tyro` pipeline together.""" if "default_instance" in deprecated_kwargs: warnings.warn( diff --git a/tyro/_fields.py b/tyro/_fields.py index f8269ebc9..4535a52e2 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -87,7 +87,7 @@ def make( helptext: Optional[str], call_argname_override: Optional[Any] = None, *, - markers: Tuple[_markers._Marker, ...] = (), + markers: Tuple[_markers.Marker, ...] = (), ): # Try to extract argconf overrides from type. _, argconfs = _resolver.unwrap_annotated( @@ -877,7 +877,7 @@ def _field_list_from_params( while not done: done = True if hasattr(f, "__wrapped__"): - f = f.__wrapped__ + f = f.__wrapped__ # type: ignore done = False if isinstance(f, functools.partial): f = f.func @@ -1022,8 +1022,7 @@ def _get_dataclass_field_default( # The only time this matters is when we our dataclass has a `__post_init__` # function that mutates the dataclass. We choose here to use the default values # before this method is called. - dataclasses.is_dataclass(field.type) - and field.default_factory is field.type + dataclasses.is_dataclass(field.type) and field.default_factory is field.type ): return field.default_factory() diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 55ec07183..628f754e4 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -178,7 +178,6 @@ def instantiator(strings: List[str]) -> None: # Note that other inputs should be caught by `choices` before the # instantiator runs. assert strings == ["None"] - return None return instantiator, InstantiatorMetadata( nargs=1, @@ -682,7 +681,7 @@ def _instantiator_from_literal( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], markers: FrozenSet[_markers.Marker], -) -> Tuple[Instantiator, InstantiatorMetadata]: +) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: choices = get_args(typ) str_choices = tuple(x.name if isinstance(x, enum.Enum) else str(x) for x in choices) return ( diff --git a/tyro/_parsers.py b/tyro/_parsers.py index 671cbdb48..f4f79c8fc 100644 --- a/tyro/_parsers.py +++ b/tyro/_parsers.py @@ -393,9 +393,7 @@ def from_field( options = [ ( # Cast seems unnecessary but needed in mypy... (1.4.1) - cast(Callable, none_proxy) - if o is type(None) - else o + cast(Callable, none_proxy) if o is type(None) else o ) for o in options ] @@ -551,9 +549,7 @@ def from_field( description = ( # We use `None` instead of an empty string to prevent a line break from # being created where the description would be. - " ".join(description_parts) - if len(description_parts) > 0 - else None + " ".join(description_parts) if len(description_parts) > 0 else None ) return SubparsersSpecification( diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 5e9eed1cb..99eb0cb66 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -42,7 +42,7 @@ def unwrap_origin_strip_extras(typ: TypeOrCallable) -> TypeOrCallable: def is_dataclass(cls: Union[TypeForm, Callable]) -> bool: """Same as `dataclasses.is_dataclass`, but also handles generic aliases.""" - return dataclasses.is_dataclass(unwrap_origin_strip_extras(cls)) + return dataclasses.is_dataclass(unwrap_origin_strip_extras(cls)) # type: ignore def resolve_generic_types( @@ -94,7 +94,7 @@ def resolved_fields(cls: TypeForm) -> List[dataclasses.Field]: assert dataclasses.is_dataclass(cls) fields = [] - annotations = get_type_hints(cls, include_extras=True) + annotations = get_type_hints(cast(Callable, cls), include_extras=True) for field in getattr(cls, "__dataclass_fields__").values(): # Avoid mutating original field. field = copy.copy(field) @@ -193,7 +193,8 @@ def narrow_collection_types( def unwrap_annotated( - typ: TypeOrCallable, search_type: TypeForm[MetadataType] = Any # type: ignore + typ: TypeOrCallable, + search_type: TypeForm[MetadataType] = Any, # type: ignore ) -> Tuple[TypeOrCallable, Tuple[MetadataType, ...]]: """Helper for parsing typing.Annotated types. @@ -256,7 +257,9 @@ def apply_type_from_typevar( if isinstance(typ, new) or get_origin(typ) is new: # type: ignore typ = old.__getitem__(args) # type: ignore - return typ.copy_with(tuple(apply_type_from_typevar(x, type_from_typevar) for x in args)) # type: ignore + return typ.copy_with( # type: ignore + tuple(apply_type_from_typevar(x, type_from_typevar) for x in args) + ) return typ diff --git a/tyro/extras/_serialization.py b/tyro/extras/_serialization.py index 3acc40f00..2e2adc9c6 100644 --- a/tyro/extras/_serialization.py +++ b/tyro/extras/_serialization.py @@ -94,13 +94,10 @@ class DataclassLoader(yaml.Loader): f" {contained_type_names}" ) - loader: yaml.Loader - node: yaml.Node - def make_dataclass_constructor(typ: Type[Any]): return lambda loader, node: typ(**loader.construct_mapping(node)) - def make_enum_constructor(typ: Type[Any]): + def make_enum_constructor(typ: Type[enum.Enum]): return lambda loader, node: typ[loader.construct_python_str(node)] for typ, name in zip(contained_types, contained_type_names): @@ -141,10 +138,6 @@ def ignore_aliases(self, data): f" {contained_type_names}" ) - dumper: yaml.Dumper - data: Any - field: dataclasses.Field - def make_representer(name: str): def representer(dumper: DataclassDumper, data: Any) -> yaml.Node: if dataclasses.is_dataclass(data): From 152b889ef293aaa03f74843b378d1a37bd630700 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 12 Dec 2023 13:18:41 +0000 Subject: [PATCH 353/491] Improve PEP 585/604 annotation errors for older versions of Python (#103) * Improve PEP 585/604 annotation errors for older versions of Python * Coverage --- tests/conftest.py | 8 ++--- tests/test_base_configs_nested.py | 4 +-- tests/test_errors_new_annotations.py | 35 +++++++++++++++++++ tests/test_helptext.py | 2 +- .../test_conf_generated.py | 24 +++++++------ tyro/_fields.py | 15 +++----- tyro/_instantiators.py | 11 ++---- tyro/_resolver.py | 32 ++++++++++++++++- 8 files changed, 92 insertions(+), 39 deletions(-) create mode 100644 tests/test_errors_new_annotations.py diff --git a/tests/conftest.py b/tests/conftest.py index 1cf00b0bf..63f1e1b87 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,14 +2,14 @@ collect_ignore_glob = [] -if not (sys.version_info.major == 3 and sys.version_info.minor >= 8): +if not sys.version_info >= (3, 8): collect_ignore_glob.append("*_min_py38.py") -if not (sys.version_info.major == 3 and sys.version_info.minor >= 9): +if not sys.version_info >= (3, 9): collect_ignore_glob.append("*_min_py39.py") -if not (sys.version_info.major == 3 and sys.version_info.minor >= 10): +if not sys.version_info >= (3, 10): collect_ignore_glob.append("*_min_py310.py") -if not (sys.version_info.major == 3 and sys.version_info.minor >= 11): +if not sys.version_info >= (3, 11): collect_ignore_glob.append("test_py311_generated/*.py") diff --git a/tests/test_base_configs_nested.py b/tests/test_base_configs_nested.py index bb7cf0809..1933617ad 100644 --- a/tests/test_base_configs_nested.py +++ b/tests/test_base_configs_nested.py @@ -1,8 +1,8 @@ from dataclasses import dataclass -from typing import Callable, Tuple, TYPE_CHECKING -from typing_extensions import Literal +from typing import TYPE_CHECKING, Callable, Tuple from torch import nn +from typing_extensions import Literal import tyro diff --git a/tests/test_errors_new_annotations.py b/tests/test_errors_new_annotations.py new file mode 100644 index 000000000..75b69f58f --- /dev/null +++ b/tests/test_errors_new_annotations.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import sys + +import pytest + +import tyro + + +@pytest.mark.skipif( + sys.version_info >= (3, 10), reason="No error for newer versions of Python." +) +def test_new_union_error() -> None: + """PEP 604 allows `|` to be used as a type annotation in Python >=3.10.""" + + def main(x: int | str) -> None: + ... + + with pytest.raises(TypeError) as e: + tyro.cli(main) + assert "You may be using a Union in the form of `X | Y`" in e.value.args[0] + + +@pytest.mark.skipif( + sys.version_info >= (3, 9), reason="No error for newer versions of Python." +) +def test_new_collection_error() -> None: + """PEP 585 allows standard collections to be used as generics in Python >=3.9.""" + + def main(x: list[int]) -> None: + ... + + with pytest.raises(TypeError) as e: + tyro.cli(main) + assert "You may be using a standard collection as a generic" in e.value.args[0] diff --git a/tests/test_helptext.py b/tests/test_helptext.py index fe9f6cc3c..ec4d3446d 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -5,8 +5,8 @@ from collections.abc import Callable from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar, Union, cast -from torch import nn from helptext_utils import get_helptext +from torch import nn from typing_extensions import Annotated, Literal diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index d851fa5eb..dd87eca3e 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -164,9 +164,10 @@ class B: @dataclasses.dataclass class Nested2: - subcommand: Annotated[ - A, tyro.conf.subcommand("command-a", default=A(7)) - ] | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] + subcommand: ( + Annotated[A, tyro.conf.subcommand("command-a", default=A(7))] + | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] + ) @dataclasses.dataclass class Nested1: @@ -271,9 +272,10 @@ class B: @dataclasses.dataclass class Nested2(Generic[T]): - subcommand: Annotated[ - T, tyro.conf.subcommand("command-a", default=A(7)) - ] | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] + subcommand: ( + Annotated[T, tyro.conf.subcommand("command-a", default=A(7))] + | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] + ) @dataclasses.dataclass class Nested1: @@ -325,11 +327,11 @@ class B: @dataclasses.dataclass class Nested: - subcommand: Annotated[ - B, tyro.conf.subcommand("one", default=default_one) - ] | Annotated[B, tyro.conf.subcommand("two")] | Annotated[ - B, tyro.conf.subcommand("three", default=default_three) - ] + subcommand: ( + Annotated[B, tyro.conf.subcommand("one", default=default_one)] + | Annotated[B, tyro.conf.subcommand("two")] + | Annotated[B, tyro.conf.subcommand("three", default=default_three)] + ) # Match by hash. def main_one(x: Nested = Nested(default_one)) -> None: diff --git a/tyro/_fields.py b/tyro/_fields.py index 4535a52e2..08740f2bf 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -31,14 +31,7 @@ import docstring_parser import typing_extensions -from typing_extensions import ( - NotRequired, - Required, - get_args, - get_origin, - get_type_hints, - is_typeddict, -) +from typing_extensions import NotRequired, Required, get_args, get_origin, is_typeddict from . import conf # Avoid circular import. from . import ( @@ -460,7 +453,7 @@ def _field_list_from_typeddict( total = getattr(cls, "__total__", True) assert isinstance(total, bool) assert not valid_default_instance or isinstance(default_instance, dict) - for name, typ in get_type_hints(cls, include_extras=True).items(): + for name, typ in _resolver.get_type_hints(cls, include_extras=True).items(): typ_origin = get_origin(typ) if valid_default_instance and name in cast(dict, default_instance): default = cast(dict, default_instance)[name] @@ -517,7 +510,7 @@ def _field_list_from_namedtuple( field_defaults = getattr(cls, "_field_defaults") # Note that _field_types is removed in Python 3.9. - for name, typ in get_type_hints(cls, include_extras=True).items(): + for name, typ in _resolver.get_type_hints(cls, include_extras=True).items(): # Get default, with priority for `default_instance`. default = field_defaults.get(name, MISSING_NONPROP) if hasattr(default_instance, name): @@ -898,7 +891,7 @@ def _field_list_from_params( # This will throw a type error for torch.device, typing.Dict, etc. try: - hints = get_type_hints(f, include_extras=True) + hints = _resolver.get_type_hints(f, include_extras=True) except TypeError: return UnsupportedNestedTypeMessage(f"Could not get hints for {f}!") diff --git a/tyro/_instantiators.py b/tyro/_instantiators.py index 628f754e4..84690b3f3 100644 --- a/tyro/_instantiators.py +++ b/tyro/_instantiators.py @@ -55,14 +55,7 @@ overload, ) -from typing_extensions import ( - Annotated, - Final, - Literal, - get_args, - get_origin, - get_type_hints, -) +from typing_extensions import Annotated, Final, Literal, get_args, get_origin from . import _resolver @@ -128,7 +121,7 @@ def is_type_string_converter(typ: Union[Callable, TypeForm[Any]]) -> bool: # One day this should be fixed with `__text_signature__`. return True - type_annotations = get_type_hints(typ) + type_annotations = _resolver.get_type_hints_with_nicer_errors(typ) # Some checks we can do if the signature is available! for i, param in enumerate(signature.parameters.values()): annotation = type_annotations.get(param.name, param.annotation) diff --git a/tyro/_resolver.py b/tyro/_resolver.py index 99eb0cb66..c9007e7df 100644 --- a/tyro/_resolver.py +++ b/tyro/_resolver.py @@ -1,5 +1,4 @@ """Utilities for resolving types and forward references.""" - import collections.abc import copy import dataclasses @@ -294,3 +293,34 @@ def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCalla pass return typ + + +def get_type_hints_with_nicer_errors( + obj: Callable[..., Any], include_extras: bool = False +) -> Dict[str, Any]: + try: + return get_type_hints(obj, include_extras=include_extras) + except TypeError as e: # pragma: no cover + common_message = f"\n-----\n\nError calling typing.get_type_hints() on {obj}! " + message = e.args[0] + if message.startswith( + "unsupported operand type(s) for |" + ) and sys.version_info < (3, 10): + # PEP 604. Requires Python 3.10. + raise TypeError( + e.args[0] + + common_message + + "You may be using a Union in the form of `X | Y`; to support Python versions that lower than 3.10, you need to use `typing.Union[X, Y]` instead of `X | Y` and `typing.Optional[X]` instead of `X | None`.", + *e.args[1:], + ) + elif message == "'type' object is not subscriptable" and ( + sys.version_info < (3, 9) + ): + # PEP 585. Requires Python 3.9. + raise TypeError( + e.args[0] + + common_message + + "You may be using a standard collection as a generic, like `list[T]` or `tuple[T1, T2]`. To support Python versions lower than 3.9, you need to using `typing.List[T]` and `typing.Tuple[T1, T2]`.", + *e.args[1:], + ) + raise e From f9283b760b1d6755096533c9c86b51db6da09174 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 19 Dec 2023 13:47:58 -0800 Subject: [PATCH 354/491] Update README badge URLs --- README.md | 8 ++++---- docs/source/index.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c44f76d5a..3bc8a2482 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,10 @@

- build - mypy - pyright - ruff + build + mypy + pyright + ruff codecov diff --git a/docs/source/index.md b/docs/source/index.md index e4a3dfef6..c622118b8 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -137,16 +137,16 @@ To get started, we recommend browsing the examples to the left. -.. |build| image:: https://github.com/brentyi/tyro/workflows/build/badge.svg +.. |build| image:: https://github.com/brentyi/tyro/actions/workflows/build.yml/badge.svg :alt: Build status icon :target: https://github.com/brentyi/tyro -.. |mypy| image:: https://github.com/brentyi/tyro/workflows/mypy/badge.svg +.. |mypy| image:: https://github.com/brentyi/tyro/actions/workflows/mypy.yml/badge.svg :alt: Mypy status icon :target: https://github.com/brentyi/tyro -.. |pyright| image:: https://github.com/brentyi/tyro/workflows/pyright/badge.svg +.. |pyright| image:: https://github.com/brentyi/tyro/actions/workflows/pyright.yml/badge.svg :alt: Mypy status icon :target: https://github.com/brentyi/tyro -.. |ruff| image:: https://github.com/brentyi/tyro/workflows/ruff/badge.svg +.. |ruff| image:: https://github.com/brentyi/tyro/actions/workflows/ruff.yml/badge.svg :alt: Lint status icon :target: https://github.com/brentyi/tyro .. |coverage| image:: https://codecov.io/gh/brentyi/tyro/branch/main/graph/badge.svg From ed0eaacafb903a58a690b3940eb73e6fa92ae9cf Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 19 Dec 2023 16:06:02 -0800 Subject: [PATCH 355/491] Minor patch for pyright upgrade --- tests/test_dcargs.py | 3 +-- tests/test_py311_generated/_generate.py | 1 + .../test_attrs_generated.py | 2 +- .../test_conf_generated.py | 24 +++++++++---------- .../test_dcargs_generated.py | 3 +-- .../test_dict_namedtuple_generated.py | 2 +- .../test_nested_generated.py | 11 ++++++++- .../test_pydantic_generated.py | 2 +- ...t_unsupported_but_should_work_generated.py | 2 +- 9 files changed, 28 insertions(+), 22 deletions(-) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index bb200374a..c07f1778c 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -306,7 +306,6 @@ class A: assert tyro.cli(A, args=["--x", "3"]) -# Hack for mypy. Not needed for pyright. Choices = int Choices = tyro.extras.literal_type_from_choices([0, 1, 2]) # type: ignore @@ -314,7 +313,7 @@ class A: def test_dynamic_literal() -> None: @dataclasses.dataclass class A: - x: Choices + x: Choices # type: ignore assert tyro.cli(A, args=["--x", "1"]) == A(x=1) with pytest.raises(SystemExit): diff --git a/tests/test_py311_generated/_generate.py b/tests/test_py311_generated/_generate.py index 54980a3c1..4e572529a 100644 --- a/tests/test_py311_generated/_generate.py +++ b/tests/test_py311_generated/_generate.py @@ -39,5 +39,6 @@ ) out_path.write_text(content) + subprocess.run(["isort", "--profile=black", str(out_path)], check=True) subprocess.run(["ruff", "format", str(out_path)], check=True) subprocess.run(["ruff", "--fix", str(out_path)], check=True) diff --git a/tests/test_py311_generated/test_attrs_generated.py b/tests/test_py311_generated/test_attrs_generated.py index 2cd154a73..42b0563ab 100644 --- a/tests/test_py311_generated/test_attrs_generated.py +++ b/tests/test_py311_generated/test_attrs_generated.py @@ -5,10 +5,10 @@ import attr import pytest +import tyro._strings from attrs import define, field import tyro -import tyro._strings def test_attrs_basic() -> None: diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index dd87eca3e..d851fa5eb 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -164,10 +164,9 @@ class B: @dataclasses.dataclass class Nested2: - subcommand: ( - Annotated[A, tyro.conf.subcommand("command-a", default=A(7))] - | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] - ) + subcommand: Annotated[ + A, tyro.conf.subcommand("command-a", default=A(7)) + ] | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] @dataclasses.dataclass class Nested1: @@ -272,10 +271,9 @@ class B: @dataclasses.dataclass class Nested2(Generic[T]): - subcommand: ( - Annotated[T, tyro.conf.subcommand("command-a", default=A(7))] - | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] - ) + subcommand: Annotated[ + T, tyro.conf.subcommand("command-a", default=A(7)) + ] | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] @dataclasses.dataclass class Nested1: @@ -327,11 +325,11 @@ class B: @dataclasses.dataclass class Nested: - subcommand: ( - Annotated[B, tyro.conf.subcommand("one", default=default_one)] - | Annotated[B, tyro.conf.subcommand("two")] - | Annotated[B, tyro.conf.subcommand("three", default=default_three)] - ) + subcommand: Annotated[ + B, tyro.conf.subcommand("one", default=default_one) + ] | Annotated[B, tyro.conf.subcommand("two")] | Annotated[ + B, tyro.conf.subcommand("three", default=default_three) + ] # Match by hash. def main_one(x: Nested = Nested(default_one)) -> None: diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py index ee6f8f311..25b1b7c13 100644 --- a/tests/test_py311_generated/test_dcargs_generated.py +++ b/tests/test_py311_generated/test_dcargs_generated.py @@ -308,7 +308,6 @@ class A: assert tyro.cli(A, args=["--x", "3"]) -# Hack for mypy. Not needed for pyright. Choices = int Choices = tyro.extras.literal_type_from_choices([0, 1, 2]) # type: ignore @@ -316,7 +315,7 @@ class A: def test_dynamic_literal() -> None: @dataclasses.dataclass class A: - x: Choices + x: Choices # type: ignore assert tyro.cli(A, args=["--x", "1"]) == A(x=1) with pytest.raises(SystemExit): diff --git a/tests/test_py311_generated/test_dict_namedtuple_generated.py b/tests/test_py311_generated/test_dict_namedtuple_generated.py index 44e560990..18be741cf 100644 --- a/tests/test_py311_generated/test_dict_namedtuple_generated.py +++ b/tests/test_py311_generated/test_dict_namedtuple_generated.py @@ -17,9 +17,9 @@ ) import pytest +import tyro._strings import tyro -import tyro._strings def test_basic_dict() -> None: diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index 7574214d6..2b72d6cde 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -1,5 +1,14 @@ import dataclasses -from typing import Annotated, Any, Generic, Literal, Mapping, Optional, Tuple, TypeVar +from typing import ( + Annotated, + Any, + Generic, + Literal, + Mapping, + Optional, + Tuple, + TypeVar, +) import pytest from frozendict import frozendict # type: ignore diff --git a/tests/test_py311_generated/test_pydantic_generated.py b/tests/test_py311_generated/test_pydantic_generated.py index 62ecbff09..92e37883a 100644 --- a/tests/test_py311_generated/test_pydantic_generated.py +++ b/tests/test_py311_generated/test_pydantic_generated.py @@ -4,10 +4,10 @@ from typing import cast import pytest +import tyro._strings from pydantic import BaseModel, Field import tyro -import tyro._strings def test_pydantic() -> None: diff --git a/tests/test_py311_generated/test_unsupported_but_should_work_generated.py b/tests/test_py311_generated/test_unsupported_but_should_work_generated.py index 7341d3251..370833c46 100644 --- a/tests/test_py311_generated/test_unsupported_but_should_work_generated.py +++ b/tests/test_py311_generated/test_unsupported_but_should_work_generated.py @@ -7,9 +7,9 @@ import omegaconf import pytest +import tyro._strings import tyro -import tyro._strings def test_omegaconf_missing(): From d076c44a12c88aa2f80bd33754d042c9c1c4cd31 Mon Sep 17 00:00:00 2001 From: Kevin Chen <79341521+kevinddchen@users.noreply.github.com> Date: Thu, 21 Dec 2023 00:24:36 -0800 Subject: [PATCH 356/491] Fix default values in Pydantic (#109) * add test * fix pydantic v2 * fix pydantic v1 * fixup * no cover pydantic < 2 * ruff ruff * Jump through some hoops to add type checking to pydantic logic * Remove unused type ignore * Add test for nested pydantic w/ default * Add support for pydantic.v1 module * Attempt to appease type checkers --------- Co-authored-by: Brent Yi --- pyproject.toml | 2 +- tests/test_pydantic.py | 68 +++++++++++++++++++++- tyro/_fields.py | 127 +++++++++++++++++++++++++++++++++-------- 3 files changed, 170 insertions(+), 27 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a77b185e4..785853708 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ dev = [ # As of 7/27/2023, flax install fails for Python 3.7 without pinning to an # old version. But doing so breaks other Python versions. "flax>=0.6.9;python_version>='3.8'", - "pydantic>=2.3.0", + "pydantic>=2.5.2", "coverage[toml]>=6.5.0" ] diff --git a/tests/test_pydantic.py b/tests/test_pydantic.py index 62ecbff09..ab5cca63d 100644 --- a/tests/test_pydantic.py +++ b/tests/test_pydantic.py @@ -4,10 +4,10 @@ from typing import cast import pytest -from pydantic import BaseModel, Field +import tyro._strings +from pydantic import BaseModel, Field, v1 import tyro -import tyro._strings def test_pydantic() -> None: @@ -29,6 +29,25 @@ class ManyTypesA(BaseModel): ) == ManyTypesA(i=5, s="hello", f=3.0, p=pathlib.Path("~")) +def test_pydantic_v1() -> None: + class ManyTypesA(v1.BaseModel): + i: int + s: str = "hello" + f: float = v1.Field(default_factory=lambda: 3.0) + p: pathlib.Path + + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( + ManyTypesA, + args=[ + "--i", + "5", + "--p", + "~", + ], + ) == ManyTypesA(i=5, s="hello", f=3.0, p=pathlib.Path("~")) + + def test_pydantic_helptext() -> None: class Helptext(BaseModel): """This docstring should be printed as a description.""" @@ -107,3 +126,48 @@ class AnnotatedAsPositional(BaseModel): result = tyro.cli(AnnotatedAsPositional, args=["myname"]) assert isinstance(result, AnnotatedAsPositional) + + +def test_pydantic_default_instance() -> None: + class Inside(BaseModel): + x: int = 1 + + class Outside(BaseModel): + i: Inside = Inside(x=2) + + assert tyro.cli(Outside, args=[]).i.x == 2, ( + "Expected x value from the default instance", + ) + assert tyro.cli(Outside, args=["--i.x", "3"]).i.x == 3 + + +def test_pydantic_nested_default_instance() -> None: + class Inside(BaseModel): + x: int = 1 + + class Middle(BaseModel): + i: Inside + + class Outside(BaseModel): + m: Middle = Middle(i=Inside(x=2)) + + assert tyro.cli(Outside, args=[]).m.i.x == 2, ( + "Expected x value from the default instance", + ) + assert tyro.cli(Outside, args=["--m.i.x", "3"]).m.i.x == 3 + + +def test_pydantic_v1_nested_default_instance() -> None: + class Inside(v1.BaseModel): + x: int = 1 + + class Middle(v1.BaseModel): + i: Inside + + class Outside(v1.BaseModel): + m: Middle = Middle(i=Inside(x=2)) + + assert tyro.cli(Outside, args=[]).m.i.x == 2, ( + "Expected x value from the default instance", + ) + assert tyro.cli(Outside, args=["--m.i.x", "3"]).m.i.x == 3 diff --git a/tyro/_fields.py b/tyro/_fields.py index 08740f2bf..1ba9b1682 100644 --- a/tyro/_fields.py +++ b/tyro/_fields.py @@ -15,6 +15,7 @@ import typing import warnings from typing import ( + TYPE_CHECKING, Any, Callable, Dict, @@ -578,15 +579,25 @@ def _field_list_from_dataclass( # Support attrs and pydantic if they're installed. - try: import pydantic except ImportError: - pydantic = None # type: ignore + if not TYPE_CHECKING: + pydantic = None # type: ignore + +try: + from pydantic import v1 as pydantic_v1 +except ImportError: + if not TYPE_CHECKING: + pydantic_v1 = None # type: ignore def _is_pydantic(cls: TypeForm[Any]) -> bool: - return pydantic is not None and issubclass(cls, pydantic.BaseModel) + if pydantic is not None and issubclass(cls, pydantic.BaseModel): + return True + if pydantic_v1 is not None and issubclass(cls, pydantic_v1.BaseModel): + return True + return False def _field_list_from_pydantic( @@ -597,44 +608,52 @@ def _field_list_from_pydantic( # Handle pydantic models. field_list = [] pydantic_version = int(getattr(pydantic, "__version__", "1.0.0").partition(".")[0]) - if pydantic_version < 2: # pragma: no cover + if pydantic_version < 2 or ( + pydantic_v1 is not None and issubclass(cls, pydantic_v1.BaseModel) + ): # Pydantic 1.xx. - for pd_field in cls.__fields__.values(): # type: ignore - helptext = pd_field.field_info.description + # We do a conditional cast because the pydantic.v1 module won't + # actually exist in legacy versions of pydantic. + if TYPE_CHECKING: + cls_cast = cast(pydantic_v1.BaseModel, cls) + else: + cls_cast = cls + for pd1_field in cls_cast.__fields__.values(): + helptext = pd1_field.field_info.description if helptext is None: - helptext = _docstrings.get_field_docstring(cls, pd_field.name) + helptext = _docstrings.get_field_docstring(cls, pd1_field.name) + + default = _get_pydantic_v1_field_default( + pd1_field.name, pd1_field, default_instance + ) field_list.append( FieldDefinition.make( - name=pd_field.name, - type_or_callable=pd_field.outer_type_, - default=( - MISSING_NONPROP if pd_field.required else pd_field.get_default() - ), + name=pd1_field.name, + type_or_callable=pd1_field.outer_type_, + default=default, helptext=helptext, ) ) else: # Pydantic 2.xx. - for name, pd_field in cls.model_fields.items(): # type: ignore - helptext = pd_field.description + for name, pd2_field in cast(pydantic.BaseModel, cls).model_fields.items(): + helptext = pd2_field.description if helptext is None: helptext = _docstrings.get_field_docstring(cls, name) + default = _get_pydantic_v2_field_default(name, pd2_field, default_instance) + field_list.append( FieldDefinition.make( name=name, - type_or_callable=pd_field.annotation, + type_or_callable=pd2_field.annotation, # type: ignore markers=tuple( meta - for meta in pd_field.metadata + for meta in pd2_field.metadata if isinstance(meta, _markers._Marker) ), - default=( - MISSING_NONPROP - if pd_field.is_required() - else pd_field.get_default(call_default_factory=True) - ), + default=default, helptext=helptext, ) ) @@ -973,8 +992,8 @@ def _ensure_dataclass_instance_used_as_default_is_frozen( def _get_dataclass_field_default( field: dataclasses.Field, parent_default_instance: Any -) -> Optional[Any]: - """Helper for getting the default instance for a field.""" +) -> Any: + """Helper for getting the default instance for a dataclass field.""" # If the dataclass's parent is explicitly marked MISSING, mark this field as missing # as well. if parent_default_instance is MISSING_PROP: @@ -985,7 +1004,7 @@ def _get_dataclass_field_default( parent_default_instance not in MISSING_SINGLETONS and parent_default_instance is not None ): - # Populate default from some parent, eg `default_instance` in `tyro.cli()`. + # Populate default from some parent, eg `default=` in `tyro.cli()`. if hasattr(parent_default_instance, field.name): return getattr(parent_default_instance, field.name) else: @@ -1022,3 +1041,63 @@ def _get_dataclass_field_default( # Otherwise, no default. This is different from MISSING, because MISSING propagates # to children. We could revisit this design to make it clearer. return MISSING_NONPROP + + +def _get_pydantic_v1_field_default( + name: str, + field: pydantic_v1.fields.ModelField, + parent_default_instance: DefaultInstance, +) -> Any: + """Helper for getting the default instance for a Pydantic field.""" + + # Try grabbing default from parent instance. + if ( + parent_default_instance not in MISSING_SINGLETONS + and parent_default_instance is not None + ): + # Populate default from some parent, eg `default=` in `tyro.cli()`. + if hasattr(parent_default_instance, name): + return getattr(parent_default_instance, name) + else: + warnings.warn( + f"Could not find field {name} in default instance" + f" {parent_default_instance}, which has" + f" type {type(parent_default_instance)},", + stacklevel=2, + ) + + if not field.required: + return field.get_default() + + # Otherwise, no default. + return MISSING_NONPROP + + +def _get_pydantic_v2_field_default( + name: str, + field: pydantic.fields.FieldInfo, + parent_default_instance: DefaultInstance, +) -> Any: + """Helper for getting the default instance for a Pydantic field.""" + + # Try grabbing default from parent instance. + if ( + parent_default_instance not in MISSING_SINGLETONS + and parent_default_instance is not None + ): + # Populate default from some parent, eg `default=` in `tyro.cli()`. + if hasattr(parent_default_instance, name): + return getattr(parent_default_instance, name) + else: + warnings.warn( + f"Could not find field {name} in default instance" + f" {parent_default_instance}, which has" + f" type {type(parent_default_instance)},", + stacklevel=2, + ) + + if not field.is_required(): + return field.get_default(call_default_factory=True) + + # Otherwise, no default. + return MISSING_NONPROP From 401bd0b7c49723c0f00ecf6843e7196e2b751a20 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 21 Dec 2023 21:22:51 -0800 Subject: [PATCH 357/491] Support + test for `typing.NewType` (#106) * Cleanup * Add NewType support + test * Add example from #110 * Fix Python<3.10, test for NewType in subcommand * More subcommand testing * Address some CI errors * More tests, address some edge cases for generics * Appease mypy --- .flake8 | 6 -- examples/02_nesting/02_subcommands.py | 7 +- pyproject.toml | 5 - {tyro => src/tyro}/__init__.py | 0 {tyro => src/tyro}/_argparse_formatter.py | 0 {tyro => src/tyro}/_arguments.py | 0 {tyro => src/tyro}/_calling.py | 4 +- {tyro => src/tyro}/_cli.py | 0 {tyro => src/tyro}/_deprecated.py | 0 {tyro => src/tyro}/_docstrings.py | 0 {tyro => src/tyro}/_fields.py | 26 +++-- {tyro => src/tyro}/_instantiators.py | 10 +- {tyro => src/tyro}/_parsers.py | 2 +- {tyro => src/tyro}/_resolver.py | 60 +++++++++--- {tyro => src/tyro}/_singleton.py | 0 {tyro => src/tyro}/_strings.py | 0 {tyro => src/tyro}/_subcommand_matching.py | 2 + {tyro => src/tyro}/_typing.py | 0 {tyro => src/tyro}/_unsafe_cache.py | 0 {tyro => src/tyro}/conf/__init__.py | 0 {tyro => src/tyro}/conf/_confstruct.py | 0 {tyro => src/tyro}/conf/_markers.py | 0 {tyro => src/tyro}/extras/__init__.py | 0 {tyro => src/tyro}/extras/_base_configs.py | 0 {tyro => src/tyro}/extras/_choices_type.py | 0 {tyro => src/tyro}/extras/_serialization.py | 0 .../tyro}/extras/_subcommand_cli_from_dict.py | 0 {tyro => src/tyro}/py.typed | 0 tests/test_generics_and_serialization.py | 25 ++++- tests/test_nested.py | 95 ++++++++++++++++++- .../test_attrs_generated.py | 2 +- .../test_dict_namedtuple_generated.py | 2 +- .../test_errors_new_annotations_generated.py | 35 +++++++ ...st_generics_and_serialization_generated.py | 25 ++++- .../test_nested_generated.py | 94 ++++++++++++++++++ .../test_pydantic_generated.py | 78 ++++++++++++++- .../test_pydantic_with_newtype_generated.py | 36 +++++++ ...t_unsupported_but_should_work_generated.py | 2 +- tests/test_pydantic.py | 11 ++- tests/test_pydantic_with_newtype.py | 37 ++++++++ 40 files changed, 515 insertions(+), 49 deletions(-) delete mode 100644 .flake8 rename {tyro => src/tyro}/__init__.py (100%) rename {tyro => src/tyro}/_argparse_formatter.py (100%) rename {tyro => src/tyro}/_arguments.py (100%) rename {tyro => src/tyro}/_calling.py (99%) rename {tyro => src/tyro}/_cli.py (100%) rename {tyro => src/tyro}/_deprecated.py (100%) rename {tyro => src/tyro}/_docstrings.py (100%) rename {tyro => src/tyro}/_fields.py (98%) rename {tyro => src/tyro}/_instantiators.py (98%) rename {tyro => src/tyro}/_parsers.py (99%) rename {tyro => src/tyro}/_resolver.py (87%) rename {tyro => src/tyro}/_singleton.py (100%) rename {tyro => src/tyro}/_strings.py (100%) rename {tyro => src/tyro}/_subcommand_matching.py (96%) rename {tyro => src/tyro}/_typing.py (100%) rename {tyro => src/tyro}/_unsafe_cache.py (100%) rename {tyro => src/tyro}/conf/__init__.py (100%) rename {tyro => src/tyro}/conf/_confstruct.py (100%) rename {tyro => src/tyro}/conf/_markers.py (100%) rename {tyro => src/tyro}/extras/__init__.py (100%) rename {tyro => src/tyro}/extras/_base_configs.py (100%) rename {tyro => src/tyro}/extras/_choices_type.py (100%) rename {tyro => src/tyro}/extras/_serialization.py (100%) rename {tyro => src/tyro}/extras/_subcommand_cli_from_dict.py (100%) rename {tyro => src/tyro}/py.typed (100%) create mode 100644 tests/test_py311_generated/test_errors_new_annotations_generated.py create mode 100644 tests/test_py311_generated/test_pydantic_with_newtype_generated.py create mode 100644 tests/test_pydantic_with_newtype.py diff --git a/.flake8 b/.flake8 deleted file mode 100644 index a0bebef2b..000000000 --- a/.flake8 +++ /dev/null @@ -1,6 +0,0 @@ -[flake8] -# E203: whitespace before : -# E501: line too long ( characters) -# W503: line break before binary operator -; ignore = E203,E501,D100,D101,D102,D103,W503 -ignore = E203,E501,W503 diff --git a/examples/02_nesting/02_subcommands.py b/examples/02_nesting/02_subcommands.py index 7948b2ab1..6eb4d661d 100644 --- a/examples/02_nesting/02_subcommands.py +++ b/examples/02_nesting/02_subcommands.py @@ -16,18 +16,21 @@ from __future__ import annotations import dataclasses -from typing import Union +from typing import NewType, Union import tyro @dataclasses.dataclass(frozen=True) -class Checkout: +class Checkout_: """Checkout a branch.""" branch: str +Checkout = NewType("Checkout", Checkout_) + + @dataclasses.dataclass(frozen=True) class Commit: """Commit changes.""" diff --git a/pyproject.toml b/pyproject.toml index 785853708..47e95e964 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,11 +54,6 @@ dev = [ [project.urls] "GitHub" = "https://github.com/brentyi/tyro" -[tool.setuptools.packages.find] -exclude = [ - "tests*" -] - [tool.setuptools.package-data] tyro = ["py.typed"] diff --git a/tyro/__init__.py b/src/tyro/__init__.py similarity index 100% rename from tyro/__init__.py rename to src/tyro/__init__.py diff --git a/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py similarity index 100% rename from tyro/_argparse_formatter.py rename to src/tyro/_argparse_formatter.py diff --git a/tyro/_arguments.py b/src/tyro/_arguments.py similarity index 100% rename from tyro/_arguments.py rename to src/tyro/_arguments.py diff --git a/tyro/_calling.py b/src/tyro/_calling.py similarity index 99% rename from tyro/_calling.py rename to src/tyro/_calling.py index a12718f18..d1cfd6ee7 100644 --- a/tyro/_calling.py +++ b/src/tyro/_calling.py @@ -220,7 +220,9 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: # `tuple`. unwrapped_f = f unwrapped_f = _resolver.unwrap_origin_strip_extras(unwrapped_f) - unwrapped_f = _resolver.narrow_subtypes(unwrapped_f, default_instance) + unwrapped_f = _resolver.unwrap_newtype_and_narrow_subtypes( + unwrapped_f, default_instance + ) unwrapped_f = _resolver.unwrap_origin_strip_extras(unwrapped_f) unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore diff --git a/tyro/_cli.py b/src/tyro/_cli.py similarity index 100% rename from tyro/_cli.py rename to src/tyro/_cli.py diff --git a/tyro/_deprecated.py b/src/tyro/_deprecated.py similarity index 100% rename from tyro/_deprecated.py rename to src/tyro/_deprecated.py diff --git a/tyro/_docstrings.py b/src/tyro/_docstrings.py similarity index 100% rename from tyro/_docstrings.py rename to src/tyro/_docstrings.py diff --git a/tyro/_fields.py b/src/tyro/_fields.py similarity index 98% rename from tyro/_fields.py rename to src/tyro/_fields.py index 1ba9b1682..3c9c05a4f 100644 --- a/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -1,4 +1,4 @@ -"""Abstractions for pulling out 'field' definitions, which specify inputs, types, and +"""Abstractions for pulling out 'field' definitions, which specify inputs, types, and # type: ignore defaults, from general callables.""" from __future__ import annotations @@ -32,7 +32,14 @@ import docstring_parser import typing_extensions -from typing_extensions import NotRequired, Required, get_args, get_origin, is_typeddict +from typing_extensions import ( + Annotated, + NotRequired, + Required, + get_args, + get_origin, + is_typeddict, +) from . import conf # Avoid circular import. from . import ( @@ -262,7 +269,7 @@ def field_list_from_callable( """ # Resolve generic types. f, type_from_typevar = _resolver.resolve_generic_types(f) - f = _resolver.narrow_subtypes(f, default_instance) + f = _resolver.unwrap_newtype_and_narrow_subtypes(f, default_instance) # Try to generate field list. field_list = _try_field_list_from_callable(f, default_instance) @@ -372,7 +379,7 @@ def _try_field_list_from_callable( # Unwrap generics. f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.apply_type_from_typevar(f, type_from_typevar) - f = _resolver.narrow_subtypes(f, default_instance) + f = _resolver.unwrap_newtype_and_narrow_subtypes(f, default_instance) f = _resolver.narrow_collection_types(f, default_instance) f_origin = _resolver.unwrap_origin_strip_extras(cast(TypeForm, f)) @@ -647,12 +654,11 @@ def _field_list_from_pydantic( field_list.append( FieldDefinition.make( name=name, - type_or_callable=pd2_field.annotation, # type: ignore - markers=tuple( - meta - for meta in pd2_field.metadata - if isinstance(meta, _markers._Marker) - ), + type_or_callable=Annotated.__class_getitem__( # type: ignore + (pd2_field.annotation,) + tuple(pd2_field.metadata) + ) + if len(pd2_field.metadata) > 0 + else pd2_field.annotation, default=default, helptext=helptext, ) diff --git a/tyro/_instantiators.py b/src/tyro/_instantiators.py similarity index 98% rename from tyro/_instantiators.py rename to src/tyro/_instantiators.py index 84690b3f3..0a7322f3e 100644 --- a/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -195,6 +195,14 @@ def instantiator(strings: List[str]) -> None: if container_out is not None: return container_out + # Unwrap NewType + set metavar based on NewType name. + # `isinstance(x, NewType)` doesn't work because NewType isn't a class until + # Python 3.10, so we instead do a duck typing-style check. + metavar = getattr(typ, "__name__", "").upper() + typ, maybe_newtype_name = _resolver.unwrap_newtype(typ) + if maybe_newtype_name is not None: + metavar = maybe_newtype_name.upper() + # Validate that typ is a `(arg: str) -> T` type converter, as expected by argparse. if typ in _builtin_set: pass @@ -246,7 +254,7 @@ def instantiator_base_case(strings: List[str]) -> Any: return instantiator_base_case, InstantiatorMetadata( nargs=1, metavar=( - typ.__name__.upper() + metavar if auto_choices is None else "{" + ",".join(map(str, auto_choices)) + "}" ), diff --git a/tyro/_parsers.py b/src/tyro/_parsers.py similarity index 99% rename from tyro/_parsers.py rename to src/tyro/_parsers.py index f4f79c8fc..2fd59a045 100644 --- a/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -322,7 +322,7 @@ def handle_field( if _fields.is_nested_type(field.type_or_callable, field.default): field = dataclasses.replace( field, - type_or_callable=_resolver.narrow_subtypes( + type_or_callable=_resolver.unwrap_newtype_and_narrow_subtypes( field.type_or_callable, field.default, ), diff --git a/tyro/_resolver.py b/src/tyro/_resolver.py similarity index 87% rename from tyro/_resolver.py rename to src/tyro/_resolver.py index c9007e7df..d8cd62430 100644 --- a/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -12,6 +12,7 @@ Dict, FrozenSet, List, + Optional, Set, Tuple, TypeVar, @@ -33,10 +34,11 @@ def unwrap_origin_strip_extras(typ: TypeOrCallable) -> TypeOrCallable: # TODO: Annotated[] handling should be revisited... typ, _ = unwrap_annotated(typ) origin = get_origin(typ) - if origin is None: - return typ - else: - return origin + + if origin is not None: + typ = origin + + return typ def is_dataclass(cls: Union[TypeForm, Callable]) -> bool: @@ -50,12 +52,15 @@ def resolve_generic_types( """If the input is a class: no-op. If it's a generic alias: returns the origin class, and a mapping from typevars to concrete types.""" - origin_cls = get_origin(cls) - annotations = () - if origin_cls is Annotated: - annotations = get_args(cls)[1:] - cls = get_args(cls)[0] - origin_cls = get_origin(cls) + annotations: Tuple[Any, ...] = () + if get_origin(cls) is Annotated: + # ^We need this `if` statement for an obscure edge case: when `cls` is a + # function with `__tyro_markers__` set, we don't want/need to return + # Annotated[func, markers]. + cls, annotations = unwrap_annotated(cls) + + # We'll ignore NewType when getting the origin + args for generics. + origin_cls = get_origin(unwrap_newtype(cls)[0]) type_from_typevar = {} if ( @@ -65,7 +70,7 @@ def resolve_generic_types( and hasattr(origin_cls.__parameters__, "__len__") ): typevars = origin_cls.__parameters__ - typevar_values = get_args(cls) + typevar_values = get_args(unwrap_newtype(cls)[0]) assert len(typevars) == len(typevar_values) cls = origin_cls type_from_typevar.update(dict(zip(typevars, typevar_values))) @@ -83,7 +88,9 @@ def resolve_generic_types( if len(annotations) == 0: return cls, type_from_typevar else: - return Annotated.__class_getitem__((cls, *annotations)), type_from_typevar # type: ignore + return Annotated.__class_getitem__( # type: ignore + (cls, *annotations) + ), type_from_typevar @_unsafe_cache.unsafe_cache(maxsize=1024) @@ -136,8 +143,31 @@ def type_from_typevar_constraints(typ: TypeOrCallable) -> TypeOrCallable: return typ +TypeOrCallableOrNone = TypeVar("TypeOrCallableOrNone", Callable, TypeForm[Any], None) + + +def unwrap_newtype( + typ: TypeOrCallableOrNone, +) -> Tuple[TypeOrCallableOrNone, Optional[str]]: + # We'll unwrap NewType annotations here; this is needed before issubclass + # checks! + # + # `isinstance(x, NewType)` doesn't work because NewType isn't a class until + # Python 3.10, so we instead do a duck typing-style check. + return_name = None + while hasattr(typ, "__name__") and hasattr(typ, "__supertype__"): + if return_name is None: + return_name = getattr(typ, "__name__") + typ = getattr(typ, "__supertype__") + + return typ, return_name + + @_unsafe_cache.unsafe_cache(maxsize=1024) -def narrow_subtypes(typ: TypeOrCallable, default_instance: Any) -> TypeOrCallable: +def unwrap_newtype_and_narrow_subtypes( + typ: TypeOrCallable, + default_instance: Any, +) -> TypeOrCallable: """Type narrowing: if we annotate as Animal but specify a default instance of Cat, we should parse as Cat. @@ -145,6 +175,10 @@ def narrow_subtypes(typ: TypeOrCallable, default_instance: Any) -> TypeOrCallabl individual arguments/fields. (if a field is annotated as Union[int, str], and a string default is passed in, we don't want to narrow the type to always be strings!)""" + + typ, unused_name = unwrap_newtype(typ) + del unused_name + try: potential_subclass = type(default_instance) diff --git a/tyro/_singleton.py b/src/tyro/_singleton.py similarity index 100% rename from tyro/_singleton.py rename to src/tyro/_singleton.py diff --git a/tyro/_strings.py b/src/tyro/_strings.py similarity index 100% rename from tyro/_strings.py rename to src/tyro/_strings.py diff --git a/tyro/_subcommand_matching.py b/src/tyro/_subcommand_matching.py similarity index 96% rename from tyro/_subcommand_matching.py rename to src/tyro/_subcommand_matching.py index 8ba0fbf3d..94cad1c36 100644 --- a/tyro/_subcommand_matching.py +++ b/src/tyro/_subcommand_matching.py @@ -97,9 +97,11 @@ def _get_type_options(typ: _typing.TypeForm) -> Tuple[_typing.TypeForm, ...]: # Check against supertypes. for self_type in self_types: self_type = _resolver.unwrap_annotated(self_type)[0] + self_type, _ = _resolver.unwrap_newtype(self_type) ok = False for super_type in super_types: super_type = _resolver.unwrap_annotated(super_type)[0] + self_type, _ = _resolver.unwrap_newtype(self_type) if issubclass(self_type, super_type): ok = True if not ok: diff --git a/tyro/_typing.py b/src/tyro/_typing.py similarity index 100% rename from tyro/_typing.py rename to src/tyro/_typing.py diff --git a/tyro/_unsafe_cache.py b/src/tyro/_unsafe_cache.py similarity index 100% rename from tyro/_unsafe_cache.py rename to src/tyro/_unsafe_cache.py diff --git a/tyro/conf/__init__.py b/src/tyro/conf/__init__.py similarity index 100% rename from tyro/conf/__init__.py rename to src/tyro/conf/__init__.py diff --git a/tyro/conf/_confstruct.py b/src/tyro/conf/_confstruct.py similarity index 100% rename from tyro/conf/_confstruct.py rename to src/tyro/conf/_confstruct.py diff --git a/tyro/conf/_markers.py b/src/tyro/conf/_markers.py similarity index 100% rename from tyro/conf/_markers.py rename to src/tyro/conf/_markers.py diff --git a/tyro/extras/__init__.py b/src/tyro/extras/__init__.py similarity index 100% rename from tyro/extras/__init__.py rename to src/tyro/extras/__init__.py diff --git a/tyro/extras/_base_configs.py b/src/tyro/extras/_base_configs.py similarity index 100% rename from tyro/extras/_base_configs.py rename to src/tyro/extras/_base_configs.py diff --git a/tyro/extras/_choices_type.py b/src/tyro/extras/_choices_type.py similarity index 100% rename from tyro/extras/_choices_type.py rename to src/tyro/extras/_choices_type.py diff --git a/tyro/extras/_serialization.py b/src/tyro/extras/_serialization.py similarity index 100% rename from tyro/extras/_serialization.py rename to src/tyro/extras/_serialization.py diff --git a/tyro/extras/_subcommand_cli_from_dict.py b/src/tyro/extras/_subcommand_cli_from_dict.py similarity index 100% rename from tyro/extras/_subcommand_cli_from_dict.py rename to src/tyro/extras/_subcommand_cli_from_dict.py diff --git a/tyro/py.typed b/src/tyro/py.typed similarity index 100% rename from tyro/py.typed rename to src/tyro/py.typed diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 5af75ee2f..6f42315a2 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -2,7 +2,7 @@ import dataclasses import enum import io -from typing import Generic, List, Tuple, Type, TypeVar, Union +from typing import Generic, List, NewType, Tuple, Type, TypeVar, Union import pytest import yaml @@ -30,6 +30,29 @@ class TupleGenericVariable(Generic[ScalarType]): ) == TupleGenericVariable((1, 2, 3)) +def test_tuple_generic_variable_newtype() -> None: + @dataclasses.dataclass + class TupleGenericVariable(Generic[ScalarType]): + xyz: Tuple[ScalarType, ...] + + SpecialInt = NewType("SpecialInt", int) + assert tyro.cli( + TupleGenericVariable[SpecialInt], args=["--xyz", "1", "2", "3"] + ) == TupleGenericVariable((SpecialInt(1), SpecialInt(2), SpecialInt(3))) + + +def test_tuple_generic_variable_more_newtype() -> None: + @dataclasses.dataclass + class TupleGenericVariable(Generic[ScalarType]): + xyz: Tuple[ScalarType, ...] + + SpecialInt = NewType("SpecialInt", int) + SpecialTuple = NewType("SpecialTuple", TupleGenericVariable[SpecialInt]) + assert tyro.cli(SpecialTuple, args=["--xyz", "1", "2", "3"]) == SpecialTuple( + TupleGenericVariable((SpecialInt(1), SpecialInt(2), SpecialInt(3))) + ) + + def test_tuple_generic_helptext() -> None: @dataclasses.dataclass class TupleGenericVariableHelptext(Generic[ScalarType]): diff --git a/tests/test_nested.py b/tests/test_nested.py index 7d94ee6a6..aa78b554b 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Any, Generic, Mapping, Optional, Tuple, TypeVar, Union +from typing import Any, Generic, Mapping, NewType, Optional, Tuple, TypeVar, Union import pytest from frozendict import frozendict # type: ignore @@ -170,6 +170,49 @@ class OptionalNested: ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) +def test_optional_nested_newtype() -> None: + @dataclasses.dataclass + class OptionalNestedChild: + y: int + z: int + + SpecialOptionalNestedChild = NewType( + "SpecialOptionalNestedChild", OptionalNestedChild + ) + + @dataclasses.dataclass + class OptionalNested: + x: int + b: Optional[SpecialOptionalNestedChild] = None + + assert tyro.cli(OptionalNested, args=["--x", "1"]) == OptionalNested(x=1, b=None) + with pytest.raises(SystemExit): + tyro.cli( + OptionalNested, + args=["--x", "1", "b:special-optional-nested-child", "--b.y", "3"], + ) + with pytest.raises(SystemExit): + tyro.cli( + OptionalNested, + args=["--x", "1", "b:special-optional-nested-child", "--b.z", "3"], + ) + + assert tyro.cli( + OptionalNested, + args=[ + "--x", + "1", + "b:special-optional-nested-child", + "--b.y", + "2", + "--b.z", + "3", + ], + ) == OptionalNested( + x=1, b=SpecialOptionalNestedChild(OptionalNestedChild(y=2, z=3)) + ) + + def test_optional_nested_multiple() -> None: """Adapted from: https://github.com/brentyi/tyro/issues/60""" @@ -329,6 +372,56 @@ class DefaultSubparser: tyro.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) +def test_subparser_with_default_and_newtype() -> None: + @dataclasses.dataclass + class DefaultHTTPServer_: + y: int + + DefaultHTTPServer__ = NewType("DefaultHTTPServer__", DefaultHTTPServer_) + DefaultHTTPServer = NewType("DefaultHTTPServer", DefaultHTTPServer__) + + def make_http_server(y: int) -> DefaultHTTPServer: + return DefaultHTTPServer(DefaultHTTPServer__(DefaultHTTPServer_(y))) + + @dataclasses.dataclass + class DefaultSMTPServer: + z: int + + @dataclasses.dataclass + class DefaultSubparser: + x: int + bc: Union[DefaultHTTPServer, DefaultSMTPServer] = dataclasses.field( + default_factory=lambda: make_http_server(5) + ) + + assert ( + tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "5"] + ) + == tyro.cli(DefaultSubparser, args=["--x", "1"]) + == DefaultSubparser(x=1, bc=make_http_server(y=5)) + ) + assert tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-smtp-server", "--bc.z", "3"] + ) == DefaultSubparser(x=1, bc=DefaultSMTPServer(z=3)) + assert ( + tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "8"] + ) + == tyro.cli( + DefaultSubparser, + args=[], + default=DefaultSubparser(x=1, bc=make_http_server(y=8)), + ) + == DefaultSubparser(x=1, bc=make_http_server(y=8)) + ) + + with pytest.raises(SystemExit): + tyro.cli(DefaultSubparser, args=["--x", "1", "b", "--bc.z", "3"]) + with pytest.raises(SystemExit): + tyro.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) + + def test_subparser_with_default_alternate() -> None: @dataclasses.dataclass class DefaultInstanceHTTPServer: diff --git a/tests/test_py311_generated/test_attrs_generated.py b/tests/test_py311_generated/test_attrs_generated.py index 42b0563ab..2cd154a73 100644 --- a/tests/test_py311_generated/test_attrs_generated.py +++ b/tests/test_py311_generated/test_attrs_generated.py @@ -5,10 +5,10 @@ import attr import pytest -import tyro._strings from attrs import define, field import tyro +import tyro._strings def test_attrs_basic() -> None: diff --git a/tests/test_py311_generated/test_dict_namedtuple_generated.py b/tests/test_py311_generated/test_dict_namedtuple_generated.py index 18be741cf..44e560990 100644 --- a/tests/test_py311_generated/test_dict_namedtuple_generated.py +++ b/tests/test_py311_generated/test_dict_namedtuple_generated.py @@ -17,9 +17,9 @@ ) import pytest -import tyro._strings import tyro +import tyro._strings def test_basic_dict() -> None: diff --git a/tests/test_py311_generated/test_errors_new_annotations_generated.py b/tests/test_py311_generated/test_errors_new_annotations_generated.py new file mode 100644 index 000000000..75b69f58f --- /dev/null +++ b/tests/test_py311_generated/test_errors_new_annotations_generated.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import sys + +import pytest + +import tyro + + +@pytest.mark.skipif( + sys.version_info >= (3, 10), reason="No error for newer versions of Python." +) +def test_new_union_error() -> None: + """PEP 604 allows `|` to be used as a type annotation in Python >=3.10.""" + + def main(x: int | str) -> None: + ... + + with pytest.raises(TypeError) as e: + tyro.cli(main) + assert "You may be using a Union in the form of `X | Y`" in e.value.args[0] + + +@pytest.mark.skipif( + sys.version_info >= (3, 9), reason="No error for newer versions of Python." +) +def test_new_collection_error() -> None: + """PEP 585 allows standard collections to be used as generics in Python >=3.9.""" + + def main(x: list[int]) -> None: + ... + + with pytest.raises(TypeError) as e: + tyro.cli(main) + assert "You may be using a standard collection as a generic" in e.value.args[0] diff --git a/tests/test_py311_generated/test_generics_and_serialization_generated.py b/tests/test_py311_generated/test_generics_and_serialization_generated.py index 223601cd8..a0897ef27 100644 --- a/tests/test_py311_generated/test_generics_and_serialization_generated.py +++ b/tests/test_py311_generated/test_generics_and_serialization_generated.py @@ -2,7 +2,7 @@ import dataclasses import enum import io -from typing import Annotated, Generic, List, Tuple, Type, TypeVar +from typing import Annotated, Generic, List, NewType, Tuple, Type, TypeVar import pytest import yaml @@ -29,6 +29,29 @@ class TupleGenericVariable(Generic[ScalarType]): ) == TupleGenericVariable((1, 2, 3)) +def test_tuple_generic_variable_newtype() -> None: + @dataclasses.dataclass + class TupleGenericVariable(Generic[ScalarType]): + xyz: Tuple[ScalarType, ...] + + SpecialInt = NewType("SpecialInt", int) + assert tyro.cli( + TupleGenericVariable[SpecialInt], args=["--xyz", "1", "2", "3"] + ) == TupleGenericVariable((SpecialInt(1), SpecialInt(2), SpecialInt(3))) + + +def test_tuple_generic_variable_more_newtype() -> None: + @dataclasses.dataclass + class TupleGenericVariable(Generic[ScalarType]): + xyz: Tuple[ScalarType, ...] + + SpecialInt = NewType("SpecialInt", int) + SpecialTuple = NewType("SpecialTuple", TupleGenericVariable[SpecialInt]) + assert tyro.cli(SpecialTuple, args=["--xyz", "1", "2", "3"]) == SpecialTuple( + TupleGenericVariable((SpecialInt(1), SpecialInt(2), SpecialInt(3))) + ) + + def test_tuple_generic_helptext() -> None: @dataclasses.dataclass class TupleGenericVariableHelptext(Generic[ScalarType]): diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index 2b72d6cde..c72222f22 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -5,6 +5,7 @@ Generic, Literal, Mapping, + NewType, Optional, Tuple, TypeVar, @@ -178,6 +179,49 @@ class OptionalNested: ) == OptionalNested(x=1, b=OptionalNestedChild(y=2, z=3)) +def test_optional_nested_newtype() -> None: + @dataclasses.dataclass + class OptionalNestedChild: + y: int + z: int + + SpecialOptionalNestedChild = NewType( + "SpecialOptionalNestedChild", OptionalNestedChild + ) + + @dataclasses.dataclass + class OptionalNested: + x: int + b: Optional[SpecialOptionalNestedChild] = None + + assert tyro.cli(OptionalNested, args=["--x", "1"]) == OptionalNested(x=1, b=None) + with pytest.raises(SystemExit): + tyro.cli( + OptionalNested, + args=["--x", "1", "b:special-optional-nested-child", "--b.y", "3"], + ) + with pytest.raises(SystemExit): + tyro.cli( + OptionalNested, + args=["--x", "1", "b:special-optional-nested-child", "--b.z", "3"], + ) + + assert tyro.cli( + OptionalNested, + args=[ + "--x", + "1", + "b:special-optional-nested-child", + "--b.y", + "2", + "--b.z", + "3", + ], + ) == OptionalNested( + x=1, b=SpecialOptionalNestedChild(OptionalNestedChild(y=2, z=3)) + ) + + def test_optional_nested_multiple() -> None: """Adapted from: https://github.com/brentyi/tyro/issues/60""" @@ -337,6 +381,56 @@ class DefaultSubparser: tyro.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) +def test_subparser_with_default_and_newtype() -> None: + @dataclasses.dataclass + class DefaultHTTPServer_: + y: int + + DefaultHTTPServer__ = NewType("DefaultHTTPServer__", DefaultHTTPServer_) + DefaultHTTPServer = NewType("DefaultHTTPServer", DefaultHTTPServer__) + + def make_http_server(y: int) -> DefaultHTTPServer: + return DefaultHTTPServer(DefaultHTTPServer__(DefaultHTTPServer_(y))) + + @dataclasses.dataclass + class DefaultSMTPServer: + z: int + + @dataclasses.dataclass + class DefaultSubparser: + x: int + bc: DefaultHTTPServer | DefaultSMTPServer = dataclasses.field( + default_factory=lambda: make_http_server(5) + ) + + assert ( + tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "5"] + ) + == tyro.cli(DefaultSubparser, args=["--x", "1"]) + == DefaultSubparser(x=1, bc=make_http_server(y=5)) + ) + assert tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-smtp-server", "--bc.z", "3"] + ) == DefaultSubparser(x=1, bc=DefaultSMTPServer(z=3)) + assert ( + tyro.cli( + DefaultSubparser, args=["--x", "1", "bc:default-http-server", "--bc.y", "8"] + ) + == tyro.cli( + DefaultSubparser, + args=[], + default=DefaultSubparser(x=1, bc=make_http_server(y=8)), + ) + == DefaultSubparser(x=1, bc=make_http_server(y=8)) + ) + + with pytest.raises(SystemExit): + tyro.cli(DefaultSubparser, args=["--x", "1", "b", "--bc.z", "3"]) + with pytest.raises(SystemExit): + tyro.cli(DefaultSubparser, args=["--x", "1", "c", "--bc.y", "3"]) + + def test_subparser_with_default_alternate() -> None: @dataclasses.dataclass class DefaultInstanceHTTPServer: diff --git a/tests/test_py311_generated/test_pydantic_generated.py b/tests/test_py311_generated/test_pydantic_generated.py index 92e37883a..e60b8d3fa 100644 --- a/tests/test_py311_generated/test_pydantic_generated.py +++ b/tests/test_py311_generated/test_pydantic_generated.py @@ -1,13 +1,13 @@ import contextlib import io import pathlib -from typing import cast +from typing import Annotated, cast import pytest -import tyro._strings -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, v1 import tyro +import tyro._strings def test_pydantic() -> None: @@ -29,6 +29,25 @@ class ManyTypesA(BaseModel): ) == ManyTypesA(i=5, s="hello", f=3.0, p=pathlib.Path("~")) +def test_pydantic_v1() -> None: + class ManyTypesA(v1.BaseModel): + i: int + s: str = "hello" + f: float = v1.Field(default_factory=lambda: 3.0) + p: pathlib.Path + + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( + ManyTypesA, + args=[ + "--i", + "5", + "--p", + "~", + ], + ) == ManyTypesA(i=5, s="hello", f=3.0, p=pathlib.Path("~")) + + def test_pydantic_helptext() -> None: class Helptext(BaseModel): """This docstring should be printed as a description.""" @@ -107,3 +126,56 @@ class AnnotatedAsPositional(BaseModel): result = tyro.cli(AnnotatedAsPositional, args=["myname"]) assert isinstance(result, AnnotatedAsPositional) + + +def test_pydantic_alias() -> None: + class AliasCfg(BaseModel): + alias: Annotated[str, tyro.conf.arg(aliases=["-a"])] + + assert tyro.cli(AliasCfg, args=["--alias", "3"]) == AliasCfg(alias="3") + assert tyro.cli(AliasCfg, args=["-a", "3"]) == AliasCfg(alias="3") + + +def test_pydantic_default_instance() -> None: + class Inside(BaseModel): + x: int = 1 + + class Outside(BaseModel): + i: Inside = Inside(x=2) + + assert tyro.cli(Outside, args=[]).i.x == 2, ( + "Expected x value from the default instance", + ) + assert tyro.cli(Outside, args=["--i.x", "3"]).i.x == 3 + + +def test_pydantic_nested_default_instance() -> None: + class Inside(BaseModel): + x: int = 1 + + class Middle(BaseModel): + i: Inside + + class Outside(BaseModel): + m: Middle = Middle(i=Inside(x=2)) + + assert tyro.cli(Outside, args=[]).m.i.x == 2, ( + "Expected x value from the default instance", + ) + assert tyro.cli(Outside, args=["--m.i.x", "3"]).m.i.x == 3 + + +def test_pydantic_v1_nested_default_instance() -> None: + class Inside(v1.BaseModel): + x: int = 1 + + class Middle(v1.BaseModel): + i: Inside + + class Outside(v1.BaseModel): + m: Middle = Middle(i=Inside(x=2)) + + assert tyro.cli(Outside, args=[]).m.i.x == 2, ( + "Expected x value from the default instance", + ) + assert tyro.cli(Outside, args=["--m.i.x", "3"]).m.i.x == 3 diff --git a/tests/test_py311_generated/test_pydantic_with_newtype_generated.py b/tests/test_py311_generated/test_pydantic_with_newtype_generated.py new file mode 100644 index 000000000..1ac21745c --- /dev/null +++ b/tests/test_py311_generated/test_pydantic_with_newtype_generated.py @@ -0,0 +1,36 @@ +from typing import Annotated, NewType, Tuple + +import pydantic +from pydantic import BaseModel + +import tyro + +Microliter = NewType("Microliter", int) + + +class Measurements(BaseModel): + single: Microliter = pydantic.Field(10) + renamed_single: Annotated[ + Microliter, tyro.conf.arg(name="other_single") + ] = pydantic.Field(10) + pair: Tuple[Microliter, Microliter] = pydantic.Field((20, 30)) + + +IncorrectMeasurements = NewType("IncorrectMeasurements", Measurements) + + +def test_pydantic_with_newtype(): + assert tyro.cli( + IncorrectMeasurements, args="--single 1 --pair 2 3".split(" ") + ) == Measurements( + single=Microliter(1), + renamed_single=Microliter(10), + pair=(Microliter(2), Microliter(3)), + ) + assert tyro.cli( + IncorrectMeasurements, args="--single 1 --other-single 5".split(" ") + ) == Measurements( + single=Microliter(1), + renamed_single=Microliter(5), + pair=(Microliter(20), Microliter(30)), + ) diff --git a/tests/test_py311_generated/test_unsupported_but_should_work_generated.py b/tests/test_py311_generated/test_unsupported_but_should_work_generated.py index 370833c46..7341d3251 100644 --- a/tests/test_py311_generated/test_unsupported_but_should_work_generated.py +++ b/tests/test_py311_generated/test_unsupported_but_should_work_generated.py @@ -7,9 +7,9 @@ import omegaconf import pytest -import tyro._strings import tyro +import tyro._strings def test_omegaconf_missing(): diff --git a/tests/test_pydantic.py b/tests/test_pydantic.py index ab5cca63d..8882404ae 100644 --- a/tests/test_pydantic.py +++ b/tests/test_pydantic.py @@ -4,10 +4,11 @@ from typing import cast import pytest -import tyro._strings from pydantic import BaseModel, Field, v1 +from typing_extensions import Annotated import tyro +import tyro._strings def test_pydantic() -> None: @@ -128,6 +129,14 @@ class AnnotatedAsPositional(BaseModel): assert isinstance(result, AnnotatedAsPositional) +def test_pydantic_alias() -> None: + class AliasCfg(BaseModel): + alias: Annotated[str, tyro.conf.arg(aliases=["-a"])] + + assert tyro.cli(AliasCfg, args=["--alias", "3"]) == AliasCfg(alias="3") + assert tyro.cli(AliasCfg, args=["-a", "3"]) == AliasCfg(alias="3") + + def test_pydantic_default_instance() -> None: class Inside(BaseModel): x: int = 1 diff --git a/tests/test_pydantic_with_newtype.py b/tests/test_pydantic_with_newtype.py new file mode 100644 index 000000000..9ec0252fe --- /dev/null +++ b/tests/test_pydantic_with_newtype.py @@ -0,0 +1,37 @@ +from typing import NewType, Tuple + +import pydantic +from pydantic import BaseModel +from typing_extensions import Annotated + +import tyro + +Microliter = NewType("Microliter", int) + + +class Measurements(BaseModel): + single: Microliter = pydantic.Field(10) + renamed_single: Annotated[ + Microliter, tyro.conf.arg(name="other_single") + ] = pydantic.Field(10) + pair: Tuple[Microliter, Microliter] = pydantic.Field((20, 30)) + + +IncorrectMeasurements = NewType("IncorrectMeasurements", Measurements) + + +def test_pydantic_with_newtype(): + assert tyro.cli( + IncorrectMeasurements, args="--single 1 --pair 2 3".split(" ") + ) == Measurements( + single=Microliter(1), + renamed_single=Microliter(10), + pair=(Microliter(2), Microliter(3)), + ) + assert tyro.cli( + IncorrectMeasurements, args="--single 1 --other-single 5".split(" ") + ) == Measurements( + single=Microliter(1), + renamed_single=Microliter(5), + pair=(Microliter(20), Microliter(30)), + ) From d901888217efcd5e781f097f253ab0e4ac655bd0 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 21 Dec 2023 21:23:32 -0800 Subject: [PATCH 358/491] mro() => __mro__ (closes #104) --- src/tyro/_docstrings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tyro/_docstrings.py b/src/tyro/_docstrings.py index 9f0fe4e1b..f69c01078 100644 --- a/src/tyro/_docstrings.py +++ b/src/tyro/_docstrings.py @@ -117,7 +117,7 @@ def get_class_tokenization_with_field( ) -> Optional[_ClassTokenization]: # Search for token in this class + all parents. found_field: bool = False - classes_to_search = cls.mro() + classes_to_search = cls.__mro__ tokenization = None for search_cls in classes_to_search: # Inherited generics seem challenging for now. From 975f07098e5143bf2dda73290c8555bf0f9e8d65 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 21 Dec 2023 21:34:17 -0800 Subject: [PATCH 359/491] Add `default_instance` support for attrs, fix docs (#111) --- docs/source/conf.py | 2 +- examples/02_nesting/02_subcommands.py | 7 ++---- src/tyro/_fields.py | 17 ++++++++++--- tests/test_attrs.py | 24 +++++++++++++++++++ .../test_attrs_generated.py | 24 +++++++++++++++++++ 5 files changed, 65 insertions(+), 9 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index ce1ed1118..53f889d5d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -213,7 +213,7 @@ # -- Extension configuration -------------------------------------------------- # -- Options for autoapi extension -------------------------------------------- -autoapi_dirs = ["../../tyro"] +autoapi_dirs = ["../../src/tyro"] autoapi_root = "api" autoapi_options = [ "members", diff --git a/examples/02_nesting/02_subcommands.py b/examples/02_nesting/02_subcommands.py index 6eb4d661d..7948b2ab1 100644 --- a/examples/02_nesting/02_subcommands.py +++ b/examples/02_nesting/02_subcommands.py @@ -16,21 +16,18 @@ from __future__ import annotations import dataclasses -from typing import NewType, Union +from typing import Union import tyro @dataclasses.dataclass(frozen=True) -class Checkout_: +class Checkout: """Checkout a branch.""" branch: str -Checkout = NewType("Checkout", Checkout_) - - @dataclasses.dataclass(frozen=True) class Commit: """Commit changes.""" diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 3c9c05a4f..8f088c7e8 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -685,8 +685,19 @@ def _field_list_from_attrs( field_list = [] for attr_field in attr.fields(cls): # Default handling. + name = attr_field.name default = attr_field.default - if default is attr.NOTHING: + if default_instance not in MISSING_SINGLETONS: + if hasattr(default_instance, name): + default = getattr(default_instance, name) + else: + warnings.warn( + f"Could not find field {name} in default instance" + f" {default_instance}, which has" + f" type {type(default_instance)},", + stacklevel=2, + ) + elif default is attr.NOTHING: default = MISSING_NONPROP elif isinstance(default, attr.Factory): # type: ignore default = default.factory() # type: ignore @@ -694,10 +705,10 @@ def _field_list_from_attrs( assert attr_field.type is not None field_list.append( FieldDefinition.make( - name=attr_field.name, + name=name, type_or_callable=attr_field.type, default=default, - helptext=_docstrings.get_field_docstring(cls, attr_field.name), + helptext=_docstrings.get_field_docstring(cls, name), ) ) return field_list diff --git a/tests/test_attrs.py b/tests/test_attrs.py index 2cd154a73..a38bb95fd 100644 --- a/tests/test_attrs.py +++ b/tests/test_attrs.py @@ -101,3 +101,27 @@ class Helptext: assert "Documentation 1" in helptext assert "Documentation 2" in helptext assert "Documentation 3" in helptext + + +def test_attrs_default_instance() -> None: + @attr.s + class ManyTypesB: + i: int = attr.ib() + s: str = attr.ib() + f: float = attr.ib(default=1.0) + + assert tyro.cli( + ManyTypesB, + args=[ + "--i", + "5", + "--s", + "5", + ], + default=ManyTypesB(i=5, s="5", f=2.0), + ) == ManyTypesB(i=5, s="5", f=2.0) + assert tyro.cli( + ManyTypesB, + args=["--i", "5"], + default=ManyTypesB(i=5, s="5", f=2.0), + ) == ManyTypesB(i=5, s="5", f=2.0) diff --git a/tests/test_py311_generated/test_attrs_generated.py b/tests/test_py311_generated/test_attrs_generated.py index 2cd154a73..a38bb95fd 100644 --- a/tests/test_py311_generated/test_attrs_generated.py +++ b/tests/test_py311_generated/test_attrs_generated.py @@ -101,3 +101,27 @@ class Helptext: assert "Documentation 1" in helptext assert "Documentation 2" in helptext assert "Documentation 3" in helptext + + +def test_attrs_default_instance() -> None: + @attr.s + class ManyTypesB: + i: int = attr.ib() + s: str = attr.ib() + f: float = attr.ib(default=1.0) + + assert tyro.cli( + ManyTypesB, + args=[ + "--i", + "5", + "--s", + "5", + ], + default=ManyTypesB(i=5, s="5", f=2.0), + ) == ManyTypesB(i=5, s="5", f=2.0) + assert tyro.cli( + ManyTypesB, + args=["--i", "5"], + default=ManyTypesB(i=5, s="5", f=2.0), + ) == ManyTypesB(i=5, s="5", f=2.0) From 98dbc5d5fe4a8cdeefb7f5516fa543b45642853b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 21 Dec 2023 21:37:36 -0800 Subject: [PATCH 360/491] Bump version Signed-off-by: Brent Yi --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 47e95e964..3fdeb881f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.6.0" +version = "0.6.1" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } From db3ef1c3c19244014655a40ab9caeca1ffd88e4f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 26 Dec 2023 10:44:34 -0800 Subject: [PATCH 361/491] More comprehensive helptext tests, details - Tests for optional groups - Tests for tyro.conf.UseAppendAction - Improve helptext for tyro.conf.arg(constructor=...), special-case JSON constructors (closes #102) - Tests for tyro.conf.arg(constructor=...) --- src/tyro/_argparse_formatter.py | 4 +- src/tyro/_arguments.py | 69 +++++++----- src/tyro/_cli.py | 6 +- src/tyro/_fields.py | 3 +- src/tyro/_parsers.py | 8 +- src/tyro/_resolver.py | 7 +- tests/helptext_utils.py | 23 ++-- tests/test_dict_namedtuple.py | 12 +-- tests/test_helptext.py | 101 ++++++++++++++++++ tests/test_nested_in_containers.py | 12 ++- .../test_dict_namedtuple_generated.py | 12 +-- .../test_helptext_generated.py | 101 ++++++++++++++++++ 12 files changed, 295 insertions(+), 63 deletions(-) diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index 3ad28ff88..d7dc169b7 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -66,7 +66,9 @@ def set_accent_color(accent_color: Optional[str]) -> None: THEME.helptext = Style(dim=True) THEME.helptext_required = Style(color="bright_red", bold=True) THEME.helptext_default = Style( - color="cyan" if accent_color != "cyan" else "magenta" + color="cyan" + if accent_color != "cyan" + else "magenta" # Another option: make default color match accent color. This is maybe more # visually consistent, but harder to read. # color=accent_color if accent_color is not None else "cyan", diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index bafdda39a..db74fac90 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -7,6 +7,7 @@ import enum import functools import itertools +import json import shlex from typing import ( TYPE_CHECKING, @@ -22,6 +23,7 @@ Tuple, TypeVar, Union, + cast, ) import rich.markup @@ -428,36 +430,58 @@ def _rule_generate_helptext( if primary_help is not None and primary_help != "": help_parts.append(_rich_tag_if_enabled(primary_help, "helptext")) - default = lowered.default - if lowered.is_fixed() or lowered.action == "append": - # Cases where we'll be missing the lowered default. Use field default instead. - assert default in _fields.MISSING_SINGLETONS or default is None - default = arg.field.default - if not lowered.required: + # Get the default value. + # Note: lowered.default is the stringified version! See + # `_rule_convert_defaults_to_strings()`. + default = lowered.default + if lowered.is_fixed() or lowered.action == "append": + # Cases where we'll be missing the lowered default. Use field default instead. + assert default in _fields.MISSING_SINGLETONS or default is None + default = arg.field.default + + # Get the default value label. + if arg.field.argconf.constructor_factory is not None: + default_label = ( + str(default) + if arg.field.type_or_callable is not json.loads + else json.dumps(arg.field.default) + ) + elif hasattr(default, "__iter__"): + # For tuple types, we might have default as (0, 1, 2, 3). + # For list types, we might have default as [0, 1, 2, 3]. + # For set types, we might have default as {0, 1, 2, 3}. + # + # In all cases, we want to display (default: 0 1 2 3), for consistency with + # the format that argparse expects when we set nargs. + assert default is not None + default_label = " ".join(map(shlex.quote, map(str, default))) + else: + default_label = str(default) + # Include default value in helptext. We intentionally don't use the % template # because the types of all arguments are set to strings, which will cause the # default to be casted to a string and introduce extra quotation marks. if lowered.instantiator is None: # Intentionally not quoted via shlex, since this can't actually be passed # in via the commandline. - default_text = f"(fixed to: {str(arg.field.default)})" + default_text = f"(fixed to: {default_label})" elif lowered.action == "append" and ( - arg.field.default in _fields.MISSING_SINGLETONS - or len(arg.field.default) == 0 + default in _fields.MISSING_SINGLETONS or len(cast(tuple, default)) == 0 ): default_text = "(repeatable)" - elif lowered.action == "append" and len(arg.field.default) > 0: + elif lowered.action == "append" and len(cast(tuple, default)) > 0: assert default is not None # Just for type checker. - default_parts = map(shlex.quote, map(str, default)) - default_text = f"(repeatable, appends: {' '.join(default_parts)})" - elif arg.field.default is _fields.EXCLUDE_FROM_CALL: + default_text = f"(repeatable, appends to: {default_label})" + elif default is _fields.EXCLUDE_FROM_CALL: default_text = "(unset by default)" elif ( _markers._OPTIONAL_GROUP in arg.field.markers and default in _fields.MISSING_SINGLETONS ): # Argument in an optional group, but with no default. This is typically used + # Note: lowered.default is the stringified version! See + # `_rule_convert_defaults_to_strings()`. # when general (non-argument, non-dataclass) object arguments are given a # default, or when we use `tyro.conf.arg(constructor=...)`. # @@ -466,22 +490,11 @@ def _rule_generate_helptext( # default should be passed in or none at all. default_text = "(optional)" elif _markers._OPTIONAL_GROUP in arg.field.markers: - # Argument in an optional group, but which also have a default. - assert default is not None # Just for type checker. - default_parts = map(shlex.quote, map(str, default)) - default_text = f"(default if used: {' '.join(default_parts)})" - elif lowered.nargs is not None and hasattr(default, "__iter__"): - # For tuple types, we might have default as (0, 1, 2, 3). - # For list types, we might have default as [0, 1, 2, 3]. - # For set types, we might have default as {0, 1, 2, 3}. - # - # In all cases, we want to display (default: 0 1 2 3), for consistency with - # the format that argparse expects when we set nargs. - assert default is not None # Just for type checker. - default_parts = map(shlex.quote, map(str, default)) - default_text = f"(default: {' '.join(default_parts)})" + # Argument in an optional group, but which also has a default. + default_text = f"(default if used: {default_label})" else: - default_text = f"(default: {shlex.quote(str(default))})" + default_text = f"(default: {default_label})" + help_parts.append(_rich_tag_if_enabled(default_text, "helptext_default")) else: help_parts.append(_rich_tag_if_enabled("(required)", "helptext_required")) diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index bfa5dc75f..7c364db54 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -270,11 +270,7 @@ def _cli_impl( return_parser: bool, return_unknown_args: bool, **deprecated_kwargs, -) -> Union[ - OutT, - argparse.ArgumentParser, - Tuple[OutT, List[str]], -]: +) -> Union[OutT, argparse.ArgumentParser, Tuple[OutT, List[str]],]: """Helper for stitching the `tyro` pipeline together.""" if "default_instance" in deprecated_kwargs: warnings.warn( diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 8f088c7e8..268c4fffc 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -1051,7 +1051,8 @@ def _get_dataclass_field_default( # The only time this matters is when we our dataclass has a `__post_init__` # function that mutates the dataclass. We choose here to use the default values # before this method is called. - dataclasses.is_dataclass(field.type) and field.default_factory is field.type + dataclasses.is_dataclass(field.type) + and field.default_factory is field.type ): return field.default_factory() diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 2fd59a045..620caaa2c 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -393,7 +393,9 @@ def from_field( options = [ ( # Cast seems unnecessary but needed in mypy... (1.4.1) - cast(Callable, none_proxy) if o is type(None) else o + cast(Callable, none_proxy) + if o is type(None) + else o ) for o in options ] @@ -549,7 +551,9 @@ def from_field( description = ( # We use `None` instead of an empty string to prevent a line break from # being created where the description would be. - " ".join(description_parts) if len(description_parts) > 0 else None + " ".join(description_parts) + if len(description_parts) > 0 + else None ) return SubparsersSpecification( diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index d8cd62430..a8d1ca0a0 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -88,9 +88,10 @@ def resolve_generic_types( if len(annotations) == 0: return cls, type_from_typevar else: - return Annotated.__class_getitem__( # type: ignore - (cls, *annotations) - ), type_from_typevar + return ( + Annotated.__class_getitem__((cls, *annotations)), # type: ignore + type_from_typevar, + ) @_unsafe_cache.unsafe_cache(maxsize=1024) diff --git a/tests/helptext_utils.py b/tests/helptext_utils.py index 58553e1f5..9857b76ac 100644 --- a/tests/helptext_utils.py +++ b/tests/helptext_utils.py @@ -2,7 +2,7 @@ import contextlib import io import os -from typing import Callable, List +from typing import Any, Callable, List import pytest @@ -12,11 +12,14 @@ def get_helptext( - f: Callable, args: List[str] = ["--help"], use_underscores: bool = False + f: Callable, + args: List[str] = ["--help"], + use_underscores: bool = False, + default: Any = None, ) -> str: target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): - tyro.cli(f, args=args, use_underscores=use_underscores) + tyro.cli(f, args=args, use_underscores=use_underscores, default=default) # Check tyro.extras.get_parser(). parser = tyro.extras.get_parser(f, use_underscores=use_underscores) @@ -33,20 +36,24 @@ def get_helptext( # Completion scripts; just smoke test for now. with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): - tyro.cli(f, args=["--tyro-print-completion", "bash"]) + tyro.cli(f, default=default, args=["--tyro-print-completion", "bash"]) with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): - tyro.cli(f, args=["--tyro-print-completion", "zsh"]) + tyro.cli(f, default=default, args=["--tyro-print-completion", "zsh"]) with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): - tyro.cli(f, args=["--tyro-write-completion", "bash", os.devnull]) + tyro.cli( + f, default=default, args=["--tyro-write-completion", "bash", os.devnull] + ) with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): - tyro.cli(f, args=["--tyro-write-completion", "zsh", os.devnull]) + tyro.cli( + f, default=default, args=["--tyro-write-completion", "zsh", os.devnull] + ) # Check helptext with vs without formatting. This can help catch text wrapping bugs # caused by ANSI sequences. target2 = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target2): tyro._arguments.USE_RICH = False - tyro.cli(f, args=args, use_underscores=use_underscores) + tyro.cli(f, default=default, args=args, use_underscores=use_underscores) tyro._arguments.USE_RICH = True if target2.getvalue() != tyro._strings.strip_ansi_sequences(target.getvalue()): diff --git a/tests/test_dict_namedtuple.py b/tests/test_dict_namedtuple.py index 7c3ab1cc7..6ddc804f9 100644 --- a/tests/test_dict_namedtuple.py +++ b/tests/test_dict_namedtuple.py @@ -535,7 +535,7 @@ def test_nested_dict_annotations() -> None: def test_functional_typeddict(): """Source: https://github.com/brentyi/tyro/issues/87""" NerfMLPHiddenLayers_0 = TypedDict( - "NerfMLPHiddenLayers0", + "NerfMLPHiddenLayers_0", { "hidden_layers.0": int, "hidden_layers.1": int, @@ -548,7 +548,7 @@ def test_functional_typeddict(): }, ) NerfMLPHiddenLayers_1 = TypedDict( - "NerfMLPHiddenLayers1", + "NerfMLPHiddenLayers_1", { "hidden_layers.0": NotRequired[int], "hidden_layers.1": NotRequired[int], @@ -561,7 +561,7 @@ def test_functional_typeddict(): }, ) NerfMLPHiddenLayers_2 = TypedDict( - "NerfMLPHiddenLayers2", + "NerfMLPHiddenLayers_2", { "hidden_layers.0": int, "hidden_layers.1": int, @@ -587,7 +587,7 @@ def test_functional_typeddict(): def test_functional_typeddict_with_default(): """Source: https://github.com/brentyi/tyro/issues/87""" NerfMLPHiddenLayers_0 = TypedDict( - "NerfMLPHiddenLayers0", + "NerfMLPHiddenLayers_0", { "hidden_layers.0": int, "hidden_layers.1": int, @@ -600,7 +600,7 @@ def test_functional_typeddict_with_default(): }, ) NerfMLPHiddenLayers_1 = TypedDict( - "NerfMLPHiddenLayers1", + "NerfMLPHiddenLayers_1", { "hidden_layers.0": NotRequired[int], "hidden_layers.1": NotRequired[int], @@ -613,7 +613,7 @@ def test_functional_typeddict_with_default(): }, ) NerfMLPHiddenLayers_2 = TypedDict( - "NerfMLPHiddenLayers2", + "NerfMLPHiddenLayers_2", { "hidden_layers.0": int, "hidden_layers.1": int, diff --git a/tests/test_helptext.py b/tests/test_helptext.py index ec4d3446d..07500f400 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -1,5 +1,6 @@ import dataclasses import enum +import json import os import pathlib from collections.abc import Callable @@ -9,6 +10,8 @@ from torch import nn from typing_extensions import Annotated, Literal +import tyro + def test_helptext() -> None: @dataclasses.dataclass @@ -753,3 +756,101 @@ class CmdCheckout012: help = get_helptext(Union[A, CmdCheckout012]) # type: ignore assert help.count("cmd-checkout012") == 3 + + +def test_tuple_default() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: Tuple[str, str] = ("hello", "world") + + help = get_helptext(A) + assert "STR STR" in help + assert "hello world" in help + assert "('hello', 'world')" not in help + + +def test_argconf_constructor() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: Annotated[ + Tuple[str, str], tyro.conf.arg(constructor=lambda x: ("a", "b")) + ] = ("hello", "world") + + help = get_helptext(A) + assert "STR STR" not in help + # Unlike case above, should not be converted to 'hello world'. + assert "('hello', 'world')" in help # JSON special case. + + +def test_argconf_constructor_json_special_case() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: Annotated[ + Tuple[str, str], tyro.conf.arg(constructor=json.loads, metavar="JSON") + ] = ("hello", "world") + y: Annotated[ + Tuple[int, int], tyro.conf.arg(constructor=json.loads, metavar="JSON") + ] = (3, 5) + + help = get_helptext(A) + assert "STR STR" not in help + assert "JSON" in help + assert '["hello", "world"]' in help # JSON special case. + assert "[3, 5]" in help, help # JSON special case. + + +def test_optional_group() -> None: + def f( + x: Annotated[ + Tuple[str, str], tyro.conf.arg(constructor=json.loads, metavar="JSON") + ] = ("hello", "world"), + y: int = 3, + ) -> int: + del x, y + return 5 + + help = get_helptext(f, default=3) + assert 'default if used: ["hello", "world"]' in help + assert "default if used: 3" in help + + +def test_append_fixed() -> None: + def f( + x: tyro.conf.UseAppendAction[Tuple[str, str]], + y: int = 3, + ) -> int: + del x, y + return 5 + + help = get_helptext(f) + assert "repeatable" not in help, help + + +def test_append_good() -> None: + def f( + x: tyro.conf.UseAppendAction[Tuple[str, ...]], + y: int = 3, + ) -> int: + del x, y + return 5 + + help = get_helptext(f) + assert "repeatable" in help, help + + +def test_append_with_default() -> None: + def f( + x: tyro.conf.UseAppendAction[Tuple[str, ...]] = ("hello world", "hello"), + y: int = 3, + ) -> int: + del x, y + return 5 + + help = get_helptext(f) + assert "repeatable, appends to: 'hello world' hello" in help, help diff --git a/tests/test_nested_in_containers.py b/tests/test_nested_in_containers.py index 819998176..973c28583 100644 --- a/tests/test_nested_in_containers.py +++ b/tests/test_nested_in_containers.py @@ -324,7 +324,9 @@ def main( assert tyro.cli( main, args="--x.float.g 0.1".split(" "), - )["float"] == GenericColor(0.5, 0.1, 0.3) + )[ + "float" + ] == GenericColor(0.5, 0.1, 0.3) assert tyro.cli( main, args="--x.int.g 0".split(" "), @@ -353,7 +355,9 @@ def main( assert tyro.cli( main, args="--x.hello.float.g 0.1".split(" "), - )["hello"]["float"] == GenericColor(0.5, 0.1, 0.3) + )["hello"][ + "float" + ] == GenericColor(0.5, 0.1, 0.3) assert tyro.cli( main, args="--x.hello.int.g 0".split(" "), @@ -376,4 +380,6 @@ def main( assert tyro.cli( main, args="--x.hello.a.g 1".split(" "), - )["hello"]["a"] == Color(5, 1, 3) + )["hello"][ + "a" + ] == Color(5, 1, 3) diff --git a/tests/test_py311_generated/test_dict_namedtuple_generated.py b/tests/test_py311_generated/test_dict_namedtuple_generated.py index 44e560990..638a2a12c 100644 --- a/tests/test_py311_generated/test_dict_namedtuple_generated.py +++ b/tests/test_py311_generated/test_dict_namedtuple_generated.py @@ -545,7 +545,7 @@ def test_nested_dict_annotations() -> None: def test_functional_typeddict(): """Source: https://github.com/brentyi/tyro/issues/87""" NerfMLPHiddenLayers_0 = TypedDict( - "NerfMLPHiddenLayers0", + "NerfMLPHiddenLayers_0", { "hidden_layers.0": int, "hidden_layers.1": int, @@ -558,7 +558,7 @@ def test_functional_typeddict(): }, ) NerfMLPHiddenLayers_1 = TypedDict( - "NerfMLPHiddenLayers1", + "NerfMLPHiddenLayers_1", { "hidden_layers.0": NotRequired[int], "hidden_layers.1": NotRequired[int], @@ -571,7 +571,7 @@ def test_functional_typeddict(): }, ) NerfMLPHiddenLayers_2 = TypedDict( - "NerfMLPHiddenLayers2", + "NerfMLPHiddenLayers_2", { "hidden_layers.0": int, "hidden_layers.1": int, @@ -597,7 +597,7 @@ def test_functional_typeddict(): def test_functional_typeddict_with_default(): """Source: https://github.com/brentyi/tyro/issues/87""" NerfMLPHiddenLayers_0 = TypedDict( - "NerfMLPHiddenLayers0", + "NerfMLPHiddenLayers_0", { "hidden_layers.0": int, "hidden_layers.1": int, @@ -610,7 +610,7 @@ def test_functional_typeddict_with_default(): }, ) NerfMLPHiddenLayers_1 = TypedDict( - "NerfMLPHiddenLayers1", + "NerfMLPHiddenLayers_1", { "hidden_layers.0": NotRequired[int], "hidden_layers.1": NotRequired[int], @@ -623,7 +623,7 @@ def test_functional_typeddict_with_default(): }, ) NerfMLPHiddenLayers_2 = TypedDict( - "NerfMLPHiddenLayers2", + "NerfMLPHiddenLayers_2", { "hidden_layers.0": int, "hidden_layers.1": int, diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py index aded14844..f1cb30441 100644 --- a/tests/test_py311_generated/test_helptext_generated.py +++ b/tests/test_py311_generated/test_helptext_generated.py @@ -1,5 +1,6 @@ import dataclasses import enum +import json import os import pathlib from collections.abc import Callable @@ -19,6 +20,8 @@ from helptext_utils import get_helptext from torch import nn +import tyro + def test_helptext() -> None: @dataclasses.dataclass @@ -752,3 +755,101 @@ class CmdCheckout012: help = get_helptext(A | CmdCheckout012) # type: ignore assert help.count("cmd-checkout012") == 3 + + +def test_tuple_default() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: Tuple[str, str] = ("hello", "world") + + help = get_helptext(A) + assert "STR STR" in help + assert "hello world" in help + assert "('hello', 'world')" not in help + + +def test_argconf_constructor() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: Annotated[ + Tuple[str, str], tyro.conf.arg(constructor=lambda x: ("a", "b")) + ] = ("hello", "world") + + help = get_helptext(A) + assert "STR STR" not in help + # Unlike case above, should not be converted to 'hello world'. + assert "('hello', 'world')" in help # JSON special case. + + +def test_argconf_constructor_json_special_case() -> None: + @dataclasses.dataclass + class A: + """Help message.""" + + x: Annotated[ + Tuple[str, str], tyro.conf.arg(constructor=json.loads, metavar="JSON") + ] = ("hello", "world") + y: Annotated[ + Tuple[int, int], tyro.conf.arg(constructor=json.loads, metavar="JSON") + ] = (3, 5) + + help = get_helptext(A) + assert "STR STR" not in help + assert "JSON" in help + assert '["hello", "world"]' in help # JSON special case. + assert "[3, 5]" in help, help # JSON special case. + + +def test_optional_group() -> None: + def f( + x: Annotated[ + Tuple[str, str], tyro.conf.arg(constructor=json.loads, metavar="JSON") + ] = ("hello", "world"), + y: int = 3, + ) -> int: + del x, y + return 5 + + help = get_helptext(f, default=3) + assert 'default if used: ["hello", "world"]' in help + assert "default if used: 3" in help + + +def test_append_fixed() -> None: + def f( + x: tyro.conf.UseAppendAction[Tuple[str, str]], + y: int = 3, + ) -> int: + del x, y + return 5 + + help = get_helptext(f) + assert "repeatable" not in help, help + + +def test_append_good() -> None: + def f( + x: tyro.conf.UseAppendAction[Tuple[str, ...]], + y: int = 3, + ) -> int: + del x, y + return 5 + + help = get_helptext(f) + assert "repeatable" in help, help + + +def test_append_with_default() -> None: + def f( + x: tyro.conf.UseAppendAction[Tuple[str, ...]] = ("hello world", "hello"), + y: int = 3, + ) -> int: + del x, y + return 5 + + help = get_helptext(f) + assert "repeatable, appends to: 'hello world' hello" in help, help From a32b88f643afe6b2bbe7278eab212c028ad8ec7d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 26 Dec 2023 16:07:00 -0800 Subject: [PATCH 362/491] Fix helptext + tyro.conf.configure() edge cases - NotRequired[] helptext for TypedDict was broken by the previous commit - Previously, wrapping a @tyro.conf.configure()-decorated class with Annotated[] would hide the @tyro.conf.configure() flags --- pyproject.toml | 2 +- src/tyro/_argparse_formatter.py | 4 +--- src/tyro/_arguments.py | 3 ++- src/tyro/_cli.py | 6 +++++- src/tyro/_fields.py | 3 +-- src/tyro/_parsers.py | 18 ++++++++---------- src/tyro/_resolver.py | 10 +++++++++- tests/test_helptext.py | 10 +++++++++- tests/test_nested.py | 4 +++- tests/test_nested_in_containers.py | 12 +++--------- 10 files changed, 42 insertions(+), 30 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3fdeb881f..40ef20cbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.6.1" +version = "0.6.2" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index d7dc169b7..3ad28ff88 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -66,9 +66,7 @@ def set_accent_color(accent_color: Optional[str]) -> None: THEME.helptext = Style(dim=True) THEME.helptext_required = Style(color="bright_red", bold=True) THEME.helptext_default = Style( - color="cyan" - if accent_color != "cyan" - else "magenta" + color="cyan" if accent_color != "cyan" else "magenta" # Another option: make default color match accent color. This is maybe more # visually consistent, but harder to read. # color=accent_color if accent_color is not None else "cyan", diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index db74fac90..4c216e494 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -473,7 +473,8 @@ def _rule_generate_helptext( elif lowered.action == "append" and len(cast(tuple, default)) > 0: assert default is not None # Just for type checker. default_text = f"(repeatable, appends to: {default_label})" - elif default is _fields.EXCLUDE_FROM_CALL: + elif arg.field.default is _fields.EXCLUDE_FROM_CALL: + # ^important to use arg.field.default and not the stringified default variable. default_text = "(unset by default)" elif ( _markers._OPTIONAL_GROUP in arg.field.markers diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index 7c364db54..bfa5dc75f 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -270,7 +270,11 @@ def _cli_impl( return_parser: bool, return_unknown_args: bool, **deprecated_kwargs, -) -> Union[OutT, argparse.ArgumentParser, Tuple[OutT, List[str]],]: +) -> Union[ + OutT, + argparse.ArgumentParser, + Tuple[OutT, List[str]], +]: """Helper for stitching the `tyro` pipeline together.""" if "default_instance" in deprecated_kwargs: warnings.warn( diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 268c4fffc..8f088c7e8 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -1051,8 +1051,7 @@ def _get_dataclass_field_default( # The only time this matters is when we our dataclass has a `__post_init__` # function that mutates the dataclass. We choose here to use the default values # before this method is called. - dataclasses.is_dataclass(field.type) - and field.default_factory is field.type + dataclasses.is_dataclass(field.type) and field.default_factory is field.type ): return field.default_factory() diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 620caaa2c..fc97627d6 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -73,10 +73,8 @@ def from_callable_or_type( """Create a parser definition from a callable or type.""" # Consolidate subcommand types. - consolidate_subcommand_args = ( - _markers.ConsolidateSubcommandArgs - in _resolver.unwrap_annotated(f, _markers._Marker)[1] - ) + markers = _resolver.unwrap_annotated(f, _markers._Marker)[1] + consolidate_subcommand_args = _markers.ConsolidateSubcommandArgs in markers # Resolve the type of `f`, generate a field list. f, type_from_typevar, field_list = _fields.field_list_from_callable( @@ -84,6 +82,10 @@ def from_callable_or_type( default_instance=default_instance, support_single_arg_types=support_single_arg_types, ) + for i in range(len(field_list)): + field_list[i] = dataclasses.replace( + field_list[i], markers=field_list[i].markers | set(markers) + ) # Cycle detection. # @@ -393,9 +395,7 @@ def from_field( options = [ ( # Cast seems unnecessary but needed in mypy... (1.4.1) - cast(Callable, none_proxy) - if o is type(None) - else o + cast(Callable, none_proxy) if o is type(None) else o ) for o in options ] @@ -551,9 +551,7 @@ def from_field( description = ( # We use `None` instead of an empty string to prevent a line break from # being created where the description would be. - " ".join(description_parts) - if len(description_parts) > 0 - else None + " ".join(description_parts) if len(description_parts) > 0 else None ) return SubparsersSpecification( diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index a8d1ca0a0..5c587c65f 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -237,6 +237,7 @@ def unwrap_annotated( - Annotated[int, 1], int => (int, (1,)) - Annotated[int, "1"], int => (int, ()) """ + # Check for __tyro_markers__ from @configure. targets = tuple( x for x in getattr(typ, "__tyro_markers__", tuple()) @@ -250,11 +251,18 @@ def unwrap_annotated( assert len(args) >= 2 # Look through metadata for desired metadata type. - targets = tuple( + targets += tuple( x for x in targets + args[1:] if search_type is Any or isinstance(x, search_type) ) + + # Check for __tyro_markers__ in unwrapped type. + targets += tuple( + x + for x in getattr(args[0], "__tyro_markers__", tuple()) + if search_type is Any or isinstance(x, search_type) + ) return args[0], targets # type: ignore diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 07500f400..f8a03dba6 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -8,7 +8,7 @@ from helptext_utils import get_helptext from torch import nn -from typing_extensions import Annotated, Literal +from typing_extensions import Annotated, Literal, NotRequired, TypedDict import tyro @@ -854,3 +854,11 @@ def f( help = get_helptext(f) assert "repeatable, appends to: 'hello world' hello" in help, help + + +def test_typeddict_exclude() -> None: + class Special(TypedDict): + x: NotRequired[int] + + help = get_helptext(Special) + assert "unset by default" in help, help diff --git a/tests/test_nested.py b/tests/test_nested.py index aa78b554b..7132b5e64 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -517,7 +517,9 @@ class C: c: int with pytest.warns(UserWarning): - assert tyro.cli(Union[A, B], default=C(3), args=["c", "--c", "2"]) == C(2) # type: ignore + assert tyro.cli( + Union[A, Annotated[B, None]], default=C(3), args=["c", "--c", "2"] + ) == C(2) # type: ignore def test_optional_subparser() -> None: diff --git a/tests/test_nested_in_containers.py b/tests/test_nested_in_containers.py index 973c28583..819998176 100644 --- a/tests/test_nested_in_containers.py +++ b/tests/test_nested_in_containers.py @@ -324,9 +324,7 @@ def main( assert tyro.cli( main, args="--x.float.g 0.1".split(" "), - )[ - "float" - ] == GenericColor(0.5, 0.1, 0.3) + )["float"] == GenericColor(0.5, 0.1, 0.3) assert tyro.cli( main, args="--x.int.g 0".split(" "), @@ -355,9 +353,7 @@ def main( assert tyro.cli( main, args="--x.hello.float.g 0.1".split(" "), - )["hello"][ - "float" - ] == GenericColor(0.5, 0.1, 0.3) + )["hello"]["float"] == GenericColor(0.5, 0.1, 0.3) assert tyro.cli( main, args="--x.hello.int.g 0".split(" "), @@ -380,6 +376,4 @@ def main( assert tyro.cli( main, args="--x.hello.a.g 1".split(" "), - )["hello"][ - "a" - ] == Color(5, 1, 3) + )["hello"]["a"] == Color(5, 1, 3) From 34d74a1eb73adb2842ca2fddd541ca28225c526f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 31 Dec 2023 17:46:07 -0800 Subject: [PATCH 363/491] More succinct usage when a lot of options are present (#114) * More succinct usage when a lot of options are present * Fix tests * Bump version * Fix error message mistake flagged by codecov + add test * no cover for column count --- examples/02_nesting/01_nesting.py | 1 + pyproject.toml | 2 +- src/tyro/_argparse_formatter.py | 40 +++++++-- src/tyro/_parsers.py | 6 +- tests/test_errors.py | 17 ++-- tests/test_helptext.py | 80 +++++++++++++++++ tests/test_missing.py | 12 ++- tests/test_nested.py | 4 +- .../test_errors_generated.py | 17 ++-- .../test_helptext_generated.py | 90 +++++++++++++++++++ .../test_missing_generated.py | 12 ++- .../test_nested_generated.py | 8 +- .../test_union_from_mapping_generated.py | 2 +- tests/test_union_from_mapping.py | 2 +- 14 files changed, 254 insertions(+), 39 deletions(-) diff --git a/examples/02_nesting/01_nesting.py b/examples/02_nesting/01_nesting.py index b6e673056..154a3bdab 100644 --- a/examples/02_nesting/01_nesting.py +++ b/examples/02_nesting/01_nesting.py @@ -52,6 +52,7 @@ class ExperimentConfig: def train( out_dir: pathlib.Path, config: ExperimentConfig, + /, restore_checkpoint: bool = False, checkpoint_interval: int = 1000, ) -> None: diff --git a/pyproject.toml b/pyproject.toml index 40ef20cbd..80b611a64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.6.2" +version = "0.6.3" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index 3ad28ff88..0528864ad 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -20,10 +20,11 @@ import difflib import itertools import re as _re +import shlex import shutil import sys from gettext import gettext as _ -from typing import Any, Dict, Generator, List, NoReturn, Optional, Set, Tuple +from typing import Any, Dict, Generator, Iterable, List, NoReturn, Optional, Set, Tuple from rich.columns import Columns from rich.console import Console, Group, RenderableType @@ -564,15 +565,15 @@ def error(self, message: str) -> NoReturn: extra_info: List[RenderableType] = [] global global_unrecognized_args if len(global_unrecognized_args) == 0 and message.startswith( - "unrecognized arguments: " + "unrecognized options: " ): global_unrecognized_args = message.partition(":")[2].strip().split(" ") message_title = "Parsing error" if len(global_unrecognized_args) > 0: - message_title = "Unrecognized arguments" - message = f"Unrecognized arguments: {' '.join(global_unrecognized_args)}" + message_title = "Unrecognized options" + message = f"Unrecognized options: {' '.join(global_unrecognized_args)}" unrecognized_arguments = set( arg for arg in global_unrecognized_args @@ -588,7 +589,7 @@ def error(self, message: str) -> NoReturn: ) if has_subcommands and same_exists: - message = f"Unrecognized or misplaced arguments: {' '.join(global_unrecognized_args)}" + message = f"Unrecognized or misplaced options: {' '.join(global_unrecognized_args)}" # Show similar arguments for keyword options. for unrecognized_argument in unrecognized_arguments: @@ -739,7 +740,7 @@ def get_score(option_string: str) -> float: prev_argument_help = arg_info.help elif message.startswith("the following arguments are required:"): - message_title = "Required arguments" + message_title = "Required options" info_from_required_arg: Dict[str, Optional[_ArgumentInfo]] = {} for arg in message.partition(":")[2].strip().split(", "): @@ -970,7 +971,7 @@ def _tyro_format_root(self): len(column_parts), ), ) - if column_count > 1: + if column_count > 1: # pragma: no cover column_width = self.formatter._width // column_count - 1 # Correct the line count for each panel using the known column # width. This will account for word wrap. @@ -1289,7 +1290,30 @@ def _format_actions_usage(self, actions, groups): # pragma: no cover return text @override - def _format_usage(self, usage, actions, groups, prefix) -> str: + def _format_usage( + self, usage, actions: Iterable[argparse.Action], groups, prefix + ) -> str: + assert isinstance(actions, list) + if len(actions) > 4: + new_actions = [] + prog_parts = shlex.split(self._prog) + optional_args_action = argparse.Action( + [ + "OPTIONS" + if len(prog_parts) == 1 + else prog_parts[-1].upper() + " OPTIONS" + ], + dest="", + ) + for action in actions: + if action.dest == "help" or len(action.option_strings) == 0: + new_actions.append(action) + elif ( + len(new_actions) == 0 or new_actions[-1] is not optional_args_action + ): + new_actions.append(optional_args_action) + actions = new_actions + # Format the usage label. if prefix is None: prefix = str_from_rich("[bold]usage[/bold]: ") diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index fc97627d6..624fe75f0 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -221,14 +221,14 @@ def apply( if subparser_group is not None: parser._action_groups.append(subparser_group) - # Break some API boundaries to rename the "optional arguments" => "arguments". + # Break some API boundaries to rename the "optional arguments" => "options". assert parser._action_groups[1].title in ( # python <= 3.9 "optional arguments", # python >= 3.10 "options", ) - parser._action_groups[1].title = "arguments" + parser._action_groups[1].title = "options" return leaves @@ -240,7 +240,7 @@ def apply_args(self, parser: argparse.ArgumentParser) -> None: # Make argument groups. def format_group_name(prefix: str) -> str: - return (prefix + " arguments").strip() + return (prefix + " options").strip() group_from_prefix: Dict[str, argparse._ArgumentGroup] = { "": parser._action_groups[1], diff --git a/tests/test_errors.py b/tests/test_errors.py index e9cb504be..30b6900ae 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -171,10 +171,9 @@ class Class: tyro.cli(Class, args="--reward.trac".split(" ")) error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Perhaps you meant:" in error - # --reward.track should appear in both the usage string and as a similar argument. assert error.count("--reward.track") == 1 assert error.count("--help") == 1 @@ -197,7 +196,7 @@ class ClassB: tyro.cli(Union[ClassA, ClassB], args="--reward.trac".split(" ")) # type: ignore error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Perhaps you meant:" in error assert error.count("--reward.track") == 1 assert error.count("--help") == 3 @@ -222,7 +221,7 @@ class ClassB: tyro.cli(Union[ClassA, ClassB], args="--fjdkslaj --reward.trac".split(" ")) # type: ignore error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Arguments similar to --reward.trac" in error assert error.count("--reward.track {True,False}") == 1 assert error.count("--reward.trace INT") == 1 @@ -248,7 +247,7 @@ class ClassB: tyro.cli(Union[ClassA, ClassB], args="--rd.trac".split(" ")) # type: ignore error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Perhaps you meant:" in error assert error.count("--reward.track {True,False}") == 1 assert error.count("--reward.trace INT") == 1 @@ -274,7 +273,7 @@ class ClassB: tyro.cli(Union[ClassA, ClassB], args="--track".split(" ")) # type: ignore error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Perhaps you meant:" in error assert error.count("--reward.track {True,False}") == 1 assert ( @@ -316,7 +315,7 @@ class ClassB: tyro.cli(Union[ClassA, ClassB], args="--track".split(" ")) # type: ignore error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Perhaps you meant:" in error assert error.count("--reward.track") == 10 assert "[...]" not in error @@ -384,7 +383,7 @@ class ClassI: ) error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Perhaps you meant:" in error assert error.count("--reward.track") == 1 assert "[...]" in error @@ -443,7 +442,7 @@ class ClassI: ) error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Arguments similar to --track" in error assert error.count("--rewar") == 1 assert "rewarde" not in error diff --git a/tests/test_helptext.py b/tests/test_helptext.py index f8a03dba6..5fd3ea676 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -413,6 +413,79 @@ class MultipleSubparsers: default_factory=Subcommand3 ) + d: bool = False + + helptext = get_helptext(MultipleSubparsers) + + assert "2% milk." in helptext + assert "Field a description." in helptext + assert "Field b description." not in helptext + assert "Field c description." not in helptext + + # Not enough args for usage shortening to kick in. + assert "[OPTIONS]" not in helptext + assert "[B:SUBCOMMAND2 OPTIONS]" not in helptext + + helptext = get_helptext( + MultipleSubparsers, args=["a:subcommand1", "b:subcommand1", "--help"] + ) + + assert "2% milk." in helptext + assert "Field a description." not in helptext + assert "Field b description." not in helptext + assert "Field c description." in helptext + assert "(default: c:subcommand3)" in helptext + + # Not enough args for usage shortening to kick in. + assert "[OPTIONS]" not in helptext + assert "[B:SUBCOMMAND1 OPTIONS]" not in helptext + assert "[B:SUBCOMMAND2 OPTIONS]" not in helptext + + +def test_multiple_subparsers_helptext_shortened_usage() -> None: + @dataclasses.dataclass + class Subcommand1: + """2% milk.""" # % symbol is prone to bugs in argparse. + + a: int = 0 + b: int = 0 + c: int = 0 + d: int = 0 + e: int = 0 + + @dataclasses.dataclass + class Subcommand2: + a: int = 0 + b: int = 0 + c: int = 0 + d: int = 0 + e: int = 0 + + @dataclasses.dataclass + class Subcommand3: + a: int = 0 + b: int = 0 + c: int = 0 + d: int = 0 + e: int = 0 + + @dataclasses.dataclass + class MultipleSubparsers: + # Field a description. + a: Union[Subcommand1, Subcommand2, Subcommand3] + # Field b description. + b: Union[Subcommand1, Subcommand2, Subcommand3] + # Field c description. + c: Union[Subcommand1, Subcommand2, Subcommand3] = dataclasses.field( + default_factory=Subcommand3 + ) + + d: bool = False + f: bool = False + g: bool = False + h: bool = False + i: bool = False + helptext = get_helptext(MultipleSubparsers) assert "2% milk." in helptext @@ -420,6 +493,9 @@ class MultipleSubparsers: assert "Field b description." not in helptext assert "Field c description." not in helptext + assert "[OPTIONS]" in helptext + assert "[B:SUBCOMMAND2 OPTIONS]" not in helptext + helptext = get_helptext( MultipleSubparsers, args=["a:subcommand1", "b:subcommand1", "--help"] ) @@ -430,6 +506,10 @@ class MultipleSubparsers: assert "Field c description." in helptext assert "(default: c:subcommand3)" in helptext + assert "[OPTIONS]" not in helptext + assert "[B:SUBCOMMAND1 OPTIONS]" in helptext + assert "[B:SUBCOMMAND2 OPTIONS]" not in helptext + def test_optional_helptext() -> None: @dataclasses.dataclass diff --git a/tests/test_missing.py b/tests/test_missing.py index 3b8b8a47c..1554bc043 100644 --- a/tests/test_missing.py +++ b/tests/test_missing.py @@ -1,6 +1,7 @@ """Tests for tyro.MISSING.""" - +import contextlib import dataclasses +import io from typing import Tuple import pytest @@ -12,8 +13,15 @@ def test_missing() -> None: def main(a: int = 5, b: int = tyro.MISSING, c: int = 3) -> Tuple[int, int, int]: return a, b, c - with pytest.raises(SystemExit): + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): tyro.cli(main, args=[]) + message = target.getvalue() + assert "Required options" in message + assert "Argument helptext" in message + assert "--a INT" not in message + assert "--b INT" in message + assert "(required)" in message assert tyro.cli(main, args=["--b", "7"]) == (5, 7, 3) diff --git a/tests/test_nested.py b/tests/test_nested.py index 7132b5e64..530ceb006 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -378,7 +378,9 @@ class DefaultHTTPServer_: y: int DefaultHTTPServer__ = NewType("DefaultHTTPServer__", DefaultHTTPServer_) - DefaultHTTPServer = NewType("DefaultHTTPServer", DefaultHTTPServer__) + DefaultHTTPServer = NewType("DefaultHTTPServer", DefaultHTTPServer__) # type: ignore + # ^nesting NewType is not technically allowed and pyright will complain, + # but we should try to be robust to it anyways. def make_http_server(y: int) -> DefaultHTTPServer: return DefaultHTTPServer(DefaultHTTPServer__(DefaultHTTPServer_(y))) diff --git a/tests/test_py311_generated/test_errors_generated.py b/tests/test_py311_generated/test_errors_generated.py index dbf343697..e48201d8f 100644 --- a/tests/test_py311_generated/test_errors_generated.py +++ b/tests/test_py311_generated/test_errors_generated.py @@ -170,10 +170,9 @@ class Class: tyro.cli(Class, args="--reward.trac".split(" ")) error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Perhaps you meant:" in error - # --reward.track should appear in both the usage string and as a similar argument. assert error.count("--reward.track") == 1 assert error.count("--help") == 1 @@ -196,7 +195,7 @@ class ClassB: tyro.cli(ClassA | ClassB, args="--reward.trac".split(" ")) # type: ignore error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Perhaps you meant:" in error assert error.count("--reward.track") == 1 assert error.count("--help") == 3 @@ -221,7 +220,7 @@ class ClassB: tyro.cli(ClassA | ClassB, args="--fjdkslaj --reward.trac".split(" ")) # type: ignore error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Arguments similar to --reward.trac" in error assert error.count("--reward.track {True,False}") == 1 assert error.count("--reward.trace INT") == 1 @@ -247,7 +246,7 @@ class ClassB: tyro.cli(ClassA | ClassB, args="--rd.trac".split(" ")) # type: ignore error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Perhaps you meant:" in error assert error.count("--reward.track {True,False}") == 1 assert error.count("--reward.trace INT") == 1 @@ -273,7 +272,7 @@ class ClassB: tyro.cli(ClassA | ClassB, args="--track".split(" ")) # type: ignore error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Perhaps you meant:" in error assert error.count("--reward.track {True,False}") == 1 assert ( @@ -315,7 +314,7 @@ class ClassB: tyro.cli(ClassA | ClassB, args="--track".split(" ")) # type: ignore error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Perhaps you meant:" in error assert error.count("--reward.track") == 10 assert "[...]" not in error @@ -389,7 +388,7 @@ class ClassI: ) error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Perhaps you meant:" in error assert error.count("--reward.track") == 1 assert "[...]" in error @@ -454,7 +453,7 @@ class ClassI: ) error = target.getvalue() - assert "Unrecognized argument" in error + assert "Unrecognized option" in error assert "Arguments similar to --track" in error assert error.count("--rewar") == 1 assert "rewarde" not in error diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py index f1cb30441..ef47b2807 100644 --- a/tests/test_py311_generated/test_helptext_generated.py +++ b/tests/test_py311_generated/test_helptext_generated.py @@ -11,8 +11,10 @@ Generic, List, Literal, + NotRequired, Optional, Tuple, + TypedDict, TypeVar, cast, ) @@ -423,6 +425,8 @@ class MultipleSubparsers: default_factory=Subcommand3 ) + d: bool = False + helptext = get_helptext(MultipleSubparsers) assert "2% milk." in helptext @@ -430,6 +434,10 @@ class MultipleSubparsers: assert "Field b description." not in helptext assert "Field c description." not in helptext + # Not enough args for usage shortening to kick in. + assert "[OPTIONS]" not in helptext + assert "[B:SUBCOMMAND2 OPTIONS]" not in helptext + helptext = get_helptext( MultipleSubparsers, args=["a:subcommand1", "b:subcommand1", "--help"] ) @@ -440,6 +448,80 @@ class MultipleSubparsers: assert "Field c description." in helptext assert "(default: c:subcommand3)" in helptext + # Not enough args for usage shortening to kick in. + assert "[OPTIONS]" not in helptext + assert "[B:SUBCOMMAND1 OPTIONS]" not in helptext + assert "[B:SUBCOMMAND2 OPTIONS]" not in helptext + + +def test_multiple_subparsers_helptext_shortened_usage() -> None: + @dataclasses.dataclass + class Subcommand1: + """2% milk.""" # % symbol is prone to bugs in argparse. + + a: int = 0 + b: int = 0 + c: int = 0 + d: int = 0 + e: int = 0 + + @dataclasses.dataclass + class Subcommand2: + a: int = 0 + b: int = 0 + c: int = 0 + d: int = 0 + e: int = 0 + + @dataclasses.dataclass + class Subcommand3: + a: int = 0 + b: int = 0 + c: int = 0 + d: int = 0 + e: int = 0 + + @dataclasses.dataclass + class MultipleSubparsers: + # Field a description. + a: Subcommand1 | Subcommand2 | Subcommand3 + # Field b description. + b: Subcommand1 | Subcommand2 | Subcommand3 + # Field c description. + c: Subcommand1 | Subcommand2 | Subcommand3 = dataclasses.field( + default_factory=Subcommand3 + ) + + d: bool = False + f: bool = False + g: bool = False + h: bool = False + i: bool = False + + helptext = get_helptext(MultipleSubparsers) + + assert "2% milk." in helptext + assert "Field a description." in helptext + assert "Field b description." not in helptext + assert "Field c description." not in helptext + + assert "[OPTIONS]" in helptext + assert "[B:SUBCOMMAND2 OPTIONS]" not in helptext + + helptext = get_helptext( + MultipleSubparsers, args=["a:subcommand1", "b:subcommand1", "--help"] + ) + + assert "2% milk." in helptext + assert "Field a description." not in helptext + assert "Field b description." not in helptext + assert "Field c description." in helptext + assert "(default: c:subcommand3)" in helptext + + assert "[OPTIONS]" not in helptext + assert "[B:SUBCOMMAND1 OPTIONS]" in helptext + assert "[B:SUBCOMMAND2 OPTIONS]" not in helptext + def test_optional_helptext() -> None: @dataclasses.dataclass @@ -853,3 +935,11 @@ def f( help = get_helptext(f) assert "repeatable, appends to: 'hello world' hello" in help, help + + +def test_typeddict_exclude() -> None: + class Special(TypedDict): + x: NotRequired[int] + + help = get_helptext(Special) + assert "unset by default" in help, help diff --git a/tests/test_py311_generated/test_missing_generated.py b/tests/test_py311_generated/test_missing_generated.py index 3b8b8a47c..1554bc043 100644 --- a/tests/test_py311_generated/test_missing_generated.py +++ b/tests/test_py311_generated/test_missing_generated.py @@ -1,6 +1,7 @@ """Tests for tyro.MISSING.""" - +import contextlib import dataclasses +import io from typing import Tuple import pytest @@ -12,8 +13,15 @@ def test_missing() -> None: def main(a: int = 5, b: int = tyro.MISSING, c: int = 3) -> Tuple[int, int, int]: return a, b, c - with pytest.raises(SystemExit): + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): tyro.cli(main, args=[]) + message = target.getvalue() + assert "Required options" in message + assert "Argument helptext" in message + assert "--a INT" not in message + assert "--b INT" in message + assert "(required)" in message assert tyro.cli(main, args=["--b", "7"]) == (5, 7, 3) diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index c72222f22..34a7b077a 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -387,7 +387,9 @@ class DefaultHTTPServer_: y: int DefaultHTTPServer__ = NewType("DefaultHTTPServer__", DefaultHTTPServer_) - DefaultHTTPServer = NewType("DefaultHTTPServer", DefaultHTTPServer__) + DefaultHTTPServer = NewType("DefaultHTTPServer", DefaultHTTPServer__) # type: ignore + # ^nesting NewType is not technically allowed and pyright will complain, + # but we should try to be robust to it anyways. def make_http_server(y: int) -> DefaultHTTPServer: return DefaultHTTPServer(DefaultHTTPServer__(DefaultHTTPServer_(y))) @@ -526,7 +528,9 @@ class C: c: int with pytest.warns(UserWarning): - assert tyro.cli(A | B, default=C(3), args=["c", "--c", "2"]) == C(2) # type: ignore + assert tyro.cli( + A | Annotated[B, None], default=C(3), args=["c", "--c", "2"] + ) == C(2) # type: ignore def test_optional_subparser() -> None: diff --git a/tests/test_py311_generated/test_union_from_mapping_generated.py b/tests/test_py311_generated/test_union_from_mapping_generated.py index 0b8b6b3c9..2eb8ff42b 100644 --- a/tests/test_py311_generated/test_union_from_mapping_generated.py +++ b/tests/test_py311_generated/test_union_from_mapping_generated.py @@ -35,7 +35,7 @@ def test_union_from_mapping_in_function(): else: ConfigUnion = tyro.extras.subcommand_type_from_defaults(base_configs) - def main(config: ConfigUnion, flag: bool = False) -> Optional[A]: + def main(config: ConfigUnion, flag: bool = False) -> Optional[A]: # type: ignore if flag: return config return None diff --git a/tests/test_union_from_mapping.py b/tests/test_union_from_mapping.py index 0b8b6b3c9..2eb8ff42b 100644 --- a/tests/test_union_from_mapping.py +++ b/tests/test_union_from_mapping.py @@ -35,7 +35,7 @@ def test_union_from_mapping_in_function(): else: ConfigUnion = tyro.extras.subcommand_type_from_defaults(base_configs) - def main(config: ConfigUnion, flag: bool = False) -> Optional[A]: + def main(config: ConfigUnion, flag: bool = False) -> Optional[A]: # type: ignore if flag: return config return None From 3962ed87171b473e7a47ad8c77c508378d02f45b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 31 Dec 2023 17:50:15 -0800 Subject: [PATCH 364/491] Superficial cleanup --- examples/02_nesting/01_nesting.py | 1 - src/tyro/_argparse_formatter.py | 25 +++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/examples/02_nesting/01_nesting.py b/examples/02_nesting/01_nesting.py index 154a3bdab..b6e673056 100644 --- a/examples/02_nesting/01_nesting.py +++ b/examples/02_nesting/01_nesting.py @@ -52,7 +52,6 @@ class ExperimentConfig: def train( out_dir: pathlib.Path, config: ExperimentConfig, - /, restore_checkpoint: bool = False, checkpoint_interval: int = 1000, ) -> None: diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index 0528864ad..44fff8d88 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -1297,21 +1297,22 @@ def _format_usage( if len(actions) > 4: new_actions = [] prog_parts = shlex.split(self._prog) - optional_args_action = argparse.Action( - [ - "OPTIONS" - if len(prog_parts) == 1 - else prog_parts[-1].upper() + " OPTIONS" - ], - dest="", - ) + added_options = False for action in actions: if action.dest == "help" or len(action.option_strings) == 0: new_actions.append(action) - elif ( - len(new_actions) == 0 or new_actions[-1] is not optional_args_action - ): - new_actions.append(optional_args_action) + elif not added_options: + added_options = True + new_actions.append( + argparse.Action( + [ + "OPTIONS" + if len(prog_parts) == 1 + else prog_parts[-1].upper() + " OPTIONS" + ], + dest="", + ) + ) actions = new_actions # Format the usage label. From caf7bdbfb87b7165752d4b2ab015e4033ba65f14 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 13 Jan 2024 23:07:20 -0800 Subject: [PATCH 365/491] Fix tyro.conf.Suppress for subcommands / nested types --- pyproject.toml | 2 +- src/tyro/_arguments.py | 2 +- src/tyro/_parsers.py | 2 +- tests/test_conf.py | 21 +++++++++++++++++++++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 80b611a64..e759e3eea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.6.3" +version = "0.6.4" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index 4c216e494..975bde52d 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -119,7 +119,7 @@ def add_argument( """Add a defined argument to a parser.""" # Get keyword arguments, with None values removed. - kwargs = dataclasses.asdict(self.lowered) # type: ignore + kwargs = dict(self.lowered.__dict__) # type: ignore kwargs.pop("instantiator") kwargs = {k: v for k, v in kwargs.items() if v is not None} name_or_flag = kwargs.pop("name_or_flag") diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 624fe75f0..98563c437 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -301,7 +301,7 @@ def handle_field( f"Field {field.intern_name} has an unbound TypeVar: {field.type_or_callable}." ) - if _markers.Fixed not in field.markers: + if _markers.Fixed not in field.markers and _markers.Suppress not in field.markers: # (1) Handle Unions over callables; these result in subparsers. subparsers_attempt = SubparsersSpecification.from_field( field, diff --git a/tests/test_conf.py b/tests/test_conf.py index 929fcfc80..25a38a022 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -13,6 +13,27 @@ import tyro +def test_suppress_subcommand() -> None: + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + flag: bool = True + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + bc: tyro.conf.Suppress[ + Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] + ] = DefaultInstanceHTTPServer() + + assert "bc" not in get_helptext(DefaultInstanceSubparser) + + def test_omit_subcommand_prefix() -> None: @dataclasses.dataclass class DefaultInstanceHTTPServer: From 9553886ad20e239b2daf862d16f6c767aedde6f8 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 13 Jan 2024 23:14:59 -0800 Subject: [PATCH 366/491] Python 3.11 test + coverage fixes --- tests/test_conf.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/test_conf.py b/tests/test_conf.py index 25a38a022..d426fd17b 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -29,7 +29,7 @@ class DefaultInstanceSubparser: # bc: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] bc: tyro.conf.Suppress[ Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] - ] = DefaultInstanceHTTPServer() + ] = dataclasses.field(default_factory=DefaultInstanceHTTPServer) assert "bc" not in get_helptext(DefaultInstanceSubparser) @@ -517,6 +517,19 @@ def main(x: Any = Struct()): assert "--x.b" not in helptext +def test_suppress_manual_fixed_one_arg_only() -> None: + @dataclasses.dataclass + class Struct: + b: tyro.conf.SuppressFixed[tyro.conf.Fixed[str]] = "7" + + def main(x: Any = Struct()): + pass + + helptext = get_helptext(main) + assert "--x.a" not in helptext + assert "--x.b" not in helptext + + def test_suppress_auto_fixed() -> None: @dataclasses.dataclass class Struct: From 6bef529e04091c388f6cafe33e935386c02759ac Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 17 Jan 2024 01:26:56 -0800 Subject: [PATCH 367/491] Use external field name in group helptext, add test for ~reproducing HfArgumentParser --- pyproject.toml | 15 ++-- src/tyro/_arguments.py | 2 - src/tyro/_fields.py | 2 +- src/tyro/_parsers.py | 12 +-- src/tyro/_resolver.py | 11 +-- tests/test_conf.py | 52 ++++++++++- .../test_conf_generated.py | 86 ++++++++++++++++++- 7 files changed, 155 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e759e3eea..e53b4ed72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,12 +41,12 @@ dev = [ "attrs>=21.4.0", "torch>=1.10.0", "pyright>=1.1.264", - "ruff>=0.1.7", + "ruff>=0.1.13", "mypy>=1.4.1", "numpy>=1.20.0", # As of 7/27/2023, flax install fails for Python 3.7 without pinning to an # old version. But doing so breaks other Python versions. - "flax>=0.6.9;python_version>='3.8'", + "flax>=0.6.9;python_version>='3.8'", "pydantic>=2.5.2", "coverage[toml]>=6.5.0" ] @@ -96,10 +96,11 @@ exclude_lines = [ "except ImportError:", # or anything that's deprecated - "deprecated", + "deprecated" ] [tool.ruff] +src = ["src"] # Needed to recognize first-party import location in GitHub action. select = [ "E", # pycodestyle errors. "F", # Pyflakes rules. @@ -107,6 +108,7 @@ select = [ "PLE", # Pylint errors. "PLR", # Pylint refactor recommendations. "PLW", # Pylint warnings. + "I" # Import sorting. ] ignore = [ "E741", # Ambiguous variable name. (l, O, or I) @@ -120,10 +122,5 @@ ignore = [ "PLR0911", # Too many return statements. "PLR0912", # Too many branches. "PLW0603", # Global statement updates are discouraged. - "PLW2901", # For loop variable overwritten. -] - -[tool.pytest.ini_options] -pythonpath = [ - "." + "PLW2901" # For loop variable overwritten. ] diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index 975bde52d..ffd36add7 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -341,8 +341,6 @@ def append_instantiator(x: Any) -> Any: return type(out)(arg.field.default) + out - return out - return dataclasses.replace( lowered, instantiator=append_instantiator, diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 8f088c7e8..4f7ebdf06 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -41,7 +41,6 @@ is_typeddict, ) -from . import conf # Avoid circular import. from . import ( _docstrings, _instantiators, @@ -49,6 +48,7 @@ _singleton, _strings, _unsafe_cache, + conf, # Avoid circular import. ) from ._typing import TypeForm from .conf import _confstruct, _markers diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 98563c437..1669aeed6 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -257,13 +257,13 @@ def format_group_name(prefix: str) -> str: for arg in self.args: if ( arg.lowered.help is not argparse.SUPPRESS - and arg.intern_prefix not in group_from_prefix + and arg.extern_prefix not in group_from_prefix ): description = self.helptext_from_nested_class_field_name.get( - arg.intern_prefix + arg.extern_prefix ) - group_from_prefix[arg.intern_prefix] = parser.add_argument_group( - format_group_name(arg.intern_prefix), + group_from_prefix[arg.extern_prefix] = parser.add_argument_group( + format_group_name(arg.extern_prefix), description=description, ) @@ -273,8 +273,8 @@ def format_group_name(prefix: str) -> str: arg.add_argument(positional_group) continue - if arg.intern_prefix in group_from_prefix: - arg.add_argument(group_from_prefix[arg.intern_prefix]) + if arg.extern_prefix in group_from_prefix: + arg.add_argument(group_from_prefix[arg.extern_prefix]) else: # Suppressed argument: still need to add them, but they won't show up in # the helptext so it doesn't matter which group. diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 5c587c65f..b28a94540 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -272,11 +272,12 @@ def apply_type_from_typevar( if typ in type_from_typevar: return type_from_typevar[typ] # type: ignore - if len(get_args(typ)) > 0: - args = get_args(typ) - if get_origin(typ) is Annotated: + origin = get_origin(typ) + args = get_args(typ) + if len(args) > 0: + if origin is Annotated: args = args[:1] - if get_origin(typ) is collections.abc.Callable: + if origin is collections.abc.Callable: assert isinstance(args[0], list) args = tuple(args[0]) + args[1:] @@ -296,7 +297,7 @@ def apply_type_from_typevar( shim_table[types.UnionType] = Union # type: ignore for new, old in shim_table.items(): - if isinstance(typ, new) or get_origin(typ) is new: # type: ignore + if isinstance(typ, new) or origin is new: # type: ignore typ = old.__getitem__(args) # type: ignore return typ.copy_with( # type: ignore diff --git a/tests/test_conf.py b/tests/test_conf.py index d426fd17b..3fcce313e 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -4,7 +4,7 @@ import io import json as json_ import shlex -from typing import Any, Dict, Generic, List, Tuple, TypeVar, Union +from typing import Any, Dict, Generic, List, Tuple, Type, TypeVar, Union import pytest from helptext_utils import get_helptext @@ -1258,3 +1258,53 @@ class Arg: assert tyro.cli(t, args=["arg"]) == Arg() assert tyro.cli(t, args=["checkout-renamed", "--branch", "main"]) == "main" assert tyro.cli(t, args=["commit", "--message", "hi", "--all", "True"]) == "hi True" + + +def test_merge() -> None: + """Test effect of tyro.conf.arg() on nested structures by approximating an + HfArgumentParser-style API.""" + + T = TypeVar("T") + + # We could add a lot overloads here if we were doing this for real. :) + def instantiate_dataclasses( + classes: Tuple[Type[T], ...], args: List[str] + ) -> Tuple[T, ...]: + return tyro.cli( + tyro.conf.OmitArgPrefixes[ # type: ignore + # Convert (type1, type2) into Tuple[type1, type2] + Tuple.__getitem__( # type: ignore + tuple(Annotated[c, tyro.conf.arg(name=c.__name__)] for c in classes) + ) + ], + args=args, + ) + + @dataclasses.dataclass(frozen=True) + class OptimizerConfig: + lr: float = 1e-4 + weight: int = 10 + + @dataclasses.dataclass(frozen=True) + class DatasetConfig: + batch_size: int = 1 + shuffle: bool = False + + assert instantiate_dataclasses( + (OptimizerConfig, DatasetConfig), args=["--lr", "1e-3"] + ) == ( + OptimizerConfig(1e-3), + DatasetConfig(), + ) + assert instantiate_dataclasses( + (OptimizerConfig, DatasetConfig), args=["--lr", "1e-3", "--shuffle"] + ) == ( + OptimizerConfig(1e-3), + DatasetConfig(shuffle=True), + ) + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + instantiate_dataclasses((OptimizerConfig, DatasetConfig), args=["--help"]) + helptext = target.getvalue() + assert "OptimizerConfig options" in helptext + assert "DatasetConfig options" in helptext diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index d851fa5eb..75dcc1e08 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -4,7 +4,7 @@ import io import json as json_ import shlex -from typing import Annotated, Any, Dict, Generic, List, Tuple, TypeVar +from typing import Annotated, Any, Dict, Generic, List, Tuple, Type, TypeVar import pytest from helptext_utils import get_helptext @@ -12,6 +12,27 @@ import tyro +def test_suppress_subcommand() -> None: + @dataclasses.dataclass + class DefaultInstanceHTTPServer: + y: int = 0 + flag: bool = True + + @dataclasses.dataclass + class DefaultInstanceSMTPServer: + z: int = 0 + + @dataclasses.dataclass + class DefaultInstanceSubparser: + x: int + # bc: DefaultInstanceHTTPServer| DefaultInstanceSMTPServer + bc: tyro.conf.Suppress[ + DefaultInstanceHTTPServer | DefaultInstanceSMTPServer + ] = dataclasses.field(default_factory=DefaultInstanceHTTPServer) + + assert "bc" not in get_helptext(DefaultInstanceSubparser) + + def test_omit_subcommand_prefix() -> None: @dataclasses.dataclass class DefaultInstanceHTTPServer: @@ -491,6 +512,19 @@ def main(x: Any = Struct()): assert "--x.b" not in helptext +def test_suppress_manual_fixed_one_arg_only() -> None: + @dataclasses.dataclass + class Struct: + b: tyro.conf.SuppressFixed[tyro.conf.Fixed[str]] = "7" + + def main(x: Any = Struct()): + pass + + helptext = get_helptext(main) + assert "--x.a" not in helptext + assert "--x.b" not in helptext + + def test_suppress_auto_fixed() -> None: @dataclasses.dataclass class Struct: @@ -1217,3 +1251,53 @@ class Arg: assert tyro.cli(t, args=["arg"]) == Arg() assert tyro.cli(t, args=["checkout-renamed", "--branch", "main"]) == "main" assert tyro.cli(t, args=["commit", "--message", "hi", "--all", "True"]) == "hi True" + + +def test_merge() -> None: + """Test effect of tyro.conf.arg() on nested structures by approximating an + HfArgumentParser-style API.""" + + T = TypeVar("T") + + # We could add a lot overloads here if we were doing this for real. :) + def instantiate_dataclasses( + classes: Tuple[Type[T], ...], args: List[str] + ) -> Tuple[T, ...]: + return tyro.cli( + tyro.conf.OmitArgPrefixes[ + # Convert (type1, type2) into Tuple[type1, type2] + Tuple.__getitem__( # type: ignore + tuple(Annotated[c, tyro.conf.arg(name=c.__name__)] for c in classes) + ) + ], + args=args, + ) + + @dataclasses.dataclass(frozen=True) + class OptimizerConfig: + lr: float = 1e-4 + weight: int = 10 + + @dataclasses.dataclass(frozen=True) + class DatasetConfig: + batch_size: int = 1 + shuffle: bool = False + + assert instantiate_dataclasses( + (OptimizerConfig, DatasetConfig), args=["--lr", "1e-3"] + ) == ( + OptimizerConfig(1e-3), + DatasetConfig(), + ) + assert instantiate_dataclasses( + (OptimizerConfig, DatasetConfig), args=["--lr", "1e-3", "--shuffle"] + ) == ( + OptimizerConfig(1e-3), + DatasetConfig(shuffle=True), + ) + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + instantiate_dataclasses((OptimizerConfig, DatasetConfig), args=["--help"]) + helptext = target.getvalue() + assert "OptimizerConfig options" in helptext + assert "DatasetConfig options" in helptext From 00e16952286c865846d239aedc161f81dd4c7f61 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 18 Jan 2024 01:40:09 -0800 Subject: [PATCH 368/491] Sphinx autodoc comment support, adapted from @kschwab / #115 --- src/tyro/_docstrings.py | 31 +++++++++++++++++++++++++++---- tests/test_helptext.py | 25 +++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/src/tyro/_docstrings.py b/src/tyro/_docstrings.py index f69c01078..88a2262c2 100644 --- a/src/tyro/_docstrings.py +++ b/src/tyro/_docstrings.py @@ -200,7 +200,11 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: if final_token_on_line.token_type == tokenize.COMMENT: comment: str = final_token_on_line.content assert comment.startswith("#") - return _strings.remove_single_line_breaks(comment[1:].strip()) + if comment.startswith("#:"): # Sphinx autodoc-style comment. + # https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#directive-autoattribute + return _strings.remove_single_line_breaks(comment[2:].strip()) + else: + return _strings.remove_single_line_breaks(comment[1:].strip()) # Check for comments that come before the field. This is intentionally written to # support comments covering multiple (grouped) fields, for example: @@ -211,7 +215,16 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: # beta2: float # # In this case, 'Optimizer hyperparameters' will be treated as the docstring for all - # 3 fields. + # 3 fields. There are tradeoffs we are making here. + # + # The exception this is Sphinx-style comments: + # + # #: The learning rate. + # learning_rate: float + # beta1: float + # beta2: float + # + # Where, by convention the comment only applies to the field that directly follows it. # Get first line of the class definition, excluding decorators. This logic is only # needed for Python >= 3.9; in 3.8, we can simply use @@ -225,6 +238,8 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: comments: List[str] = [] current_actual_line = field_data.actual_line - 1 + directly_above_field = True + is_sphinx_doc_comment = False while current_actual_line in tokenization.tokens_from_actual_line: actual_line_tokens = tokenization.tokens_from_actual_line[current_actual_line] @@ -245,14 +260,22 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: ): (comment_token,) = actual_line_tokens assert comment_token.content.startswith("#") - comments.append(comment_token.content[1:].strip()) + if comment_token.content.startswith("#:"): # Sphinx autodoc-style comment. + # https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#directive-autoattribute + comments.append(comment_token.content[2:].strip()) + is_sphinx_doc_comment = True + else: + comments.append(comment_token.content[1:].strip()) elif len(comments) > 0: # Comments should be contiguous. break + else: + # This comment is not directly above the current field. + directly_above_field = False current_actual_line -= 1 - if len(comments) > 0: + if len(comments) > 0 and not (is_sphinx_doc_comment and not directly_above_field): return _strings.remove_single_line_breaks("\n".join(reversed(comments))) return None diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 5fd3ea676..11b267658 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -36,6 +36,31 @@ class Helptext: assert "Documentation 3 (default: 3)" in helptext +def test_helptext_sphinx_autodoc_style() -> None: + @dataclasses.dataclass + class Helptext: + """This docstring should be printed as a description.""" + + x: int #: Documentation 1 + + #:Documentation 2 + y: Annotated[int, "ignored"] + z: int = 3 + + helptext = get_helptext(Helptext) + assert cast(str, helptext) in helptext + assert "x INT" in helptext + assert "y INT" in helptext + assert "z INT" in helptext + assert "Documentation 1 (required)" in helptext + assert ": Documentation 1" not in helptext + assert "Documentation 2 (required)" in helptext + assert ":Documentation 2" not in helptext + + # :Documentation 2 should not be applied to `z`. + assert helptext.count("Documentation 2") == 1 + + def test_helptext_from_class_docstring() -> None: @dataclasses.dataclass class Helptext2: From bc55c77da353144697f1fbb0d3205f0ac9860818 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 18 Jan 2024 01:42:27 -0800 Subject: [PATCH 369/491] Bump version Signed-off-by: Brent Yi --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e53b4ed72..d19615b57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.6.4" +version = "0.6.5" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } From 171b636a16f695c667e52eecc0f4e2822e8176b5 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 18 Jan 2024 19:17:39 -0800 Subject: [PATCH 370/491] Fix subcommand override edge case (#117) * Fix subcommand override edge case * Fix typo * ruff * Fix tests * More tweaks, leave refactor TODO note --- pyproject.toml | 2 +- src/tyro/_fields.py | 62 ++++++++++++++++++++++--------- src/tyro/_parsers.py | 20 +++++++++- tests/test_base_configs_nested.py | 32 ++++++++++++++++ 4 files changed, 97 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d19615b57..73d98ad7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.6.5" +version = "0.6.6" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 4f7ebdf06..c0d814a8f 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -62,6 +62,10 @@ class FieldDefinition: """Type or callable for this field. This should have all Annotated[] annotations stripped.""" default: Any + # We need to record whether defaults are from default instances to + # determine if they should override the default in + # tyro.conf.subcommand(default=...). + is_default_from_default_instance: bool helptext: Optional[str] markers: FrozenSet[Any] custom_constructor: bool @@ -85,6 +89,7 @@ def make( name: str, type_or_callable: Union[TypeForm[Any], Callable], default: Any, + is_default_from_default_instance: bool, helptext: Optional[str], call_argname_override: Optional[Any] = None, *, @@ -125,6 +130,7 @@ def make( if argconf.constructor_factory is None else argconf.constructor_factory(), default=default, + is_default_from_default_instance=is_default_from_default_instance, helptext=helptext, markers=frozenset(inferred_markers).union(markers), custom_constructor=argconf.constructor_factory is not None, @@ -285,6 +291,7 @@ def field_list_from_callable( extern_name="value", # Doesn't matter. type_or_callable=f, default=default_instance, + is_default_from_default_instance=True, helptext="", custom_constructor=False, markers=frozenset( @@ -370,6 +377,9 @@ def _try_field_list_from_callable( f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: + # TODO: this is needed when field_list_from_callable() is called in _calling.py. + # It's basically duplicated from (completely separate!) logic in + # _parsers.py, which is risky and might caused edge cases. f, found_subcommand_configs = _resolver.unwrap_annotated( f, conf._confstruct._SubcommandConfiguration ) @@ -463,8 +473,10 @@ def _field_list_from_typeddict( assert not valid_default_instance or isinstance(default_instance, dict) for name, typ in _resolver.get_type_hints(cls, include_extras=True).items(): typ_origin = get_origin(typ) + is_default_from_default_instance = False if valid_default_instance and name in cast(dict, default_instance): default = cast(dict, default_instance)[name] + is_default_from_default_instance = True elif typ_origin is Required and total is False: # Support total=False. default = MISSING_PROP @@ -500,6 +512,7 @@ def _field_list_from_typeddict( name=name, type_or_callable=typ, default=default, + is_default_from_default_instance=is_default_from_default_instance, helptext=_docstrings.get_field_docstring(cls, name), ) ) @@ -531,6 +544,7 @@ def _field_list_from_namedtuple( name=name, type_or_callable=typ, default=default, + is_default_from_default_instance=True, helptext=_docstrings.get_field_docstring(cls, name), ) ) @@ -562,7 +576,9 @@ def _field_list_from_dataclass( if is_flax_module and dc_field.name in ("name", "parent"): continue - default = _get_dataclass_field_default(dc_field, default_instance) + default, is_default_from_default_instance = _get_dataclass_field_default( + dc_field, default_instance + ) # Try to get helptext from field metadata. This is also intended to be # compatible with HuggingFace-style config objects. @@ -579,6 +595,7 @@ def _field_list_from_dataclass( name=dc_field.name, type_or_callable=dc_field.type, default=default, + is_default_from_default_instance=is_default_from_default_instance, helptext=helptext, ) ) @@ -630,7 +647,7 @@ def _field_list_from_pydantic( if helptext is None: helptext = _docstrings.get_field_docstring(cls, pd1_field.name) - default = _get_pydantic_v1_field_default( + default, is_default_from_default_instance = _get_pydantic_v1_field_default( pd1_field.name, pd1_field, default_instance ) @@ -639,6 +656,7 @@ def _field_list_from_pydantic( name=pd1_field.name, type_or_callable=pd1_field.outer_type_, default=default, + is_default_from_default_instance=is_default_from_default_instance, helptext=helptext, ) ) @@ -649,7 +667,9 @@ def _field_list_from_pydantic( if helptext is None: helptext = _docstrings.get_field_docstring(cls, name) - default = _get_pydantic_v2_field_default(name, pd2_field, default_instance) + default, is_default_from_default_instance = _get_pydantic_v2_field_default( + name, pd2_field, default_instance + ) field_list.append( FieldDefinition.make( @@ -660,6 +680,7 @@ def _field_list_from_pydantic( if len(pd2_field.metadata) > 0 else pd2_field.annotation, default=default, + is_default_from_default_instance=is_default_from_default_instance, helptext=helptext, ) ) @@ -687,9 +708,11 @@ def _field_list_from_attrs( # Default handling. name = attr_field.name default = attr_field.default + is_default_from_default_instance = False if default_instance not in MISSING_SINGLETONS: if hasattr(default_instance, name): default = getattr(default_instance, name) + is_default_from_default_instance = True else: warnings.warn( f"Could not find field {name} in default instance" @@ -708,6 +731,7 @@ def _field_list_from_attrs( name=name, type_or_callable=attr_field.type, default=default, + is_default_from_default_instance=is_default_from_default_instance, helptext=_docstrings.get_field_docstring(cls, name), ) ) @@ -754,6 +778,7 @@ def _field_list_from_tuple( name=str(i), type_or_callable=child, default=default_i, + is_default_from_default_instance=True, helptext="", # This should really set the positional marker, but the CLI is more # intuitive for mixed nested/non-nested types in tuples when we stick @@ -839,6 +864,7 @@ def _try_field_list_from_sequence_inner( name=str(i), type_or_callable=contained_type, default=default_i, + is_default_from_default_instance=True, helptext="", ) ) @@ -860,6 +886,7 @@ def _field_list_from_dict( name=str(k) if not isinstance(k, enum.Enum) else k.name, type_or_callable=type(v), default=v, + is_default_from_default_instance=True, helptext=None, # Dictionary specific key: call_argname_override=k, @@ -984,6 +1011,7 @@ def _field_list_from_params( # Note that param.annotation doesn't resolve forward references. type_or_callable=typ, default=default, + is_default_from_default_instance=False, helptext=helptext, markers=markers, ) @@ -1009,12 +1037,12 @@ def _ensure_dataclass_instance_used_as_default_is_frozen( def _get_dataclass_field_default( field: dataclasses.Field, parent_default_instance: Any -) -> Any: +) -> Tuple[Any, bool]: """Helper for getting the default instance for a dataclass field.""" # If the dataclass's parent is explicitly marked MISSING, mark this field as missing # as well. if parent_default_instance is MISSING_PROP: - return MISSING_PROP + return MISSING_PROP, False # Try grabbing default from parent instance. if ( @@ -1023,7 +1051,7 @@ def _get_dataclass_field_default( ): # Populate default from some parent, eg `default=` in `tyro.cli()`. if hasattr(parent_default_instance, field.name): - return getattr(parent_default_instance, field.name) + return getattr(parent_default_instance, field.name), True else: warnings.warn( f"Could not find field {field.name} in default instance" @@ -1039,7 +1067,7 @@ def _get_dataclass_field_default( # _types_, not just instances. if type(default) is not type and dataclasses.is_dataclass(default): _ensure_dataclass_instance_used_as_default_is_frozen(field, default) - return default + return default, False # Populate default from `dataclasses.field(default_factory=...)`. if field.default_factory is not dataclasses.MISSING and not ( @@ -1053,18 +1081,18 @@ def _get_dataclass_field_default( # before this method is called. dataclasses.is_dataclass(field.type) and field.default_factory is field.type ): - return field.default_factory() + return field.default_factory(), False # Otherwise, no default. This is different from MISSING, because MISSING propagates # to children. We could revisit this design to make it clearer. - return MISSING_NONPROP + return MISSING_NONPROP, False def _get_pydantic_v1_field_default( name: str, field: pydantic_v1.fields.ModelField, parent_default_instance: DefaultInstance, -) -> Any: +) -> Tuple[Any, bool]: """Helper for getting the default instance for a Pydantic field.""" # Try grabbing default from parent instance. @@ -1074,7 +1102,7 @@ def _get_pydantic_v1_field_default( ): # Populate default from some parent, eg `default=` in `tyro.cli()`. if hasattr(parent_default_instance, name): - return getattr(parent_default_instance, name) + return getattr(parent_default_instance, name), True else: warnings.warn( f"Could not find field {name} in default instance" @@ -1084,17 +1112,17 @@ def _get_pydantic_v1_field_default( ) if not field.required: - return field.get_default() + return field.get_default(), False # Otherwise, no default. - return MISSING_NONPROP + return MISSING_NONPROP, False def _get_pydantic_v2_field_default( name: str, field: pydantic.fields.FieldInfo, parent_default_instance: DefaultInstance, -) -> Any: +) -> Tuple[Any, bool]: """Helper for getting the default instance for a Pydantic field.""" # Try grabbing default from parent instance. @@ -1104,7 +1132,7 @@ def _get_pydantic_v2_field_default( ): # Populate default from some parent, eg `default=` in `tyro.cli()`. if hasattr(parent_default_instance, name): - return getattr(parent_default_instance, name) + return getattr(parent_default_instance, name), True else: warnings.warn( f"Could not find field {name} in default instance" @@ -1114,7 +1142,7 @@ def _get_pydantic_v2_field_default( ) if not field.is_required(): - return field.get_default(call_default_factory=True) + return field.get_default(call_default_factory=True), False # Otherwise, no default. - return MISSING_NONPROP + return MISSING_NONPROP, False diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 1669aeed6..98da4fb2e 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -493,10 +493,28 @@ def from_field( ) # If names match, borrow subcommand default from field default. - if default_name == subcommand_name: + if default_name == subcommand_name and ( + field.is_default_from_default_instance + or subcommand_config.default in _fields.MISSING_SINGLETONS + ): subcommand_config = dataclasses.replace( subcommand_config, default=field.default ) + + # Strip the subcommand config from the option type. + option_origin, annotations = _resolver.unwrap_annotated(option) + annotations = tuple( + a + for a in annotations + if not isinstance(a, _confstruct._SubcommandConfiguration) + ) + if len(annotations) == 0: + option = option_origin + else: + option = Annotated.__class_getitem__( # type: ignore + (option_origin,) + annotations + ) + subparser = ParserSpecification.from_callable_or_type( ( # Recursively apply markers. diff --git a/tests/test_base_configs_nested.py b/tests/test_base_configs_nested.py index 1933617ad..7500df324 100644 --- a/tests/test_base_configs_nested.py +++ b/tests/test_base_configs_nested.py @@ -178,3 +178,35 @@ def main(cfg: BaseConfig) -> BaseConfig: ), DataConfig(2), ) + + +def test_pernicious_override(): + """From: https://github.com/nerfstudio-project/nerfstudio/issues/2789 + + Situation where we: + - have a default value in the config class + - override that default value with a subcommand annotation + - override it again with a default instance + """ + assert ( + tyro.cli( + BaseConfig, + default=BaseConfig( + "test", + "test", + ExperimentConfig( + dataset="mnist", + optimizer=AdamOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + seed=0, + activation=nn.ReLU, + ), + DataConfig(0), + ), + args="small small-data".split(" "), + ).data_config.test + == 0 + ) From b42b4856979f4bcb40735809c5a71ddd72c857cb Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 19 Jan 2024 00:31:24 -0800 Subject: [PATCH 371/491] Suppress mypy error --- src/tyro/_parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 98da4fb2e..1a8e0473a 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -502,7 +502,7 @@ def from_field( ) # Strip the subcommand config from the option type. - option_origin, annotations = _resolver.unwrap_annotated(option) + option_origin, annotations = _resolver.unwrap_annotated(option) # type: ignore annotations = tuple( a for a in annotations From d242ddacb31e9a35e8b1cad24aa85e93ba9e476b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 19 Jan 2024 04:05:39 -0800 Subject: [PATCH 372/491] Consolidate field list generation (#118) * Start refactor/simplify * Bump version * Remove unnecessary prefix logic * ruff * String cleanup * Remove unnecessary annotation manipulation --- pyproject.toml | 2 +- src/tyro/_argparse_formatter.py | 3 ++ src/tyro/_calling.py | 14 ++------ src/tyro/_fields.py | 10 +----- src/tyro/_parsers.py | 57 +++++++++++++++------------------ 5 files changed, 34 insertions(+), 52 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 73d98ad7d..2b58863f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.6.6" +version = "0.7.0" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index 44fff8d88..eec5c2158 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -174,6 +174,9 @@ def _recursive_arg_search( + (1 if subparser_name in args else -0.001), ) + for child in parser_spec.child_from_prefix.values(): + _recursive_arg_search(child, prog, subcommand_match_score) + _recursive_arg_search(parser_spec, prog, 0) return arguments, has_subcommands, same_exists diff --git a/src/tyro/_calling.py b/src/tyro/_calling.py index d1cfd6ee7..ffadc5592 100644 --- a/src/tyro/_calling.py +++ b/src/tyro/_calling.py @@ -36,11 +36,6 @@ def call_from_args( Returns the output of `f` and a set of used arguments.""" - # Resolve the type of `f`, generate a field list. - f, type_from_typevar, field_list = _fields.field_list_from_callable( - f=f, default_instance=default_instance, support_single_arg_types=True - ) - positional_args: List[Any] = [] kwargs: Dict[str, Any] = {} consumed_keywords: Set[str] = set() @@ -61,7 +56,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: any_arguments_provided = False - for field in field_list: + for field in parser_definition.field_list: value: Any prefixed_field_name = _strings.make_field_name( [field_name_prefix, field.intern_name] @@ -120,16 +115,13 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: " is a fixed argument that cannot be parsed", arg, ) - elif ( - prefixed_field_name - in parser_definition.helptext_from_nested_class_field_name - ): + elif prefixed_field_name in parser_definition.child_from_prefix: # Nested callable. if _resolver.unwrap_origin_strip_extras(field_type) is Union: field_type = type(field.default) value, consumed_keywords_child = call_from_args( field_type, - parser_definition, + parser_definition.child_from_prefix[prefixed_field_name], field.default, value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index c0d814a8f..dd2c4b97d 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -377,16 +377,8 @@ def _try_field_list_from_callable( f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - # TODO: this is needed when field_list_from_callable() is called in _calling.py. - # It's basically duplicated from (completely separate!) logic in - # _parsers.py, which is risky and might caused edge cases. - f, found_subcommand_configs = _resolver.unwrap_annotated( - f, conf._confstruct._SubcommandConfiguration - ) - if len(found_subcommand_configs) > 0: - default_instance = found_subcommand_configs[0].default - # Unwrap generics. + f = _resolver.unwrap_annotated(f)[0] f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.apply_type_from_typevar(f, type_from_typevar) f = _resolver.unwrap_newtype_and_narrow_subtypes(f, default_instance) diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 1a8e0473a..bf9241043 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -43,7 +43,9 @@ class ParserSpecification: f: Callable description: str args: List[_arguments.ArgumentDefinition] - helptext_from_nested_class_field_name: Dict[str, Optional[str]] + field_list: List[_fields.FieldDefinition] + child_from_prefix: Dict[str, ParserSpecification] + helptext_from_intern_prefixed_field_name: Dict[str, Optional[str]] # We have two mechanics for tracking subparser groups: # - A single subparser group, which is what gets added in the tree structure built @@ -105,6 +107,8 @@ def from_callable_or_type( args = [] helptext_from_intern_prefixed_field_name: Dict[str, Optional[str]] = {} + child_from_prefix: Dict[str, ParserSpecification] = {} + subparsers = None subparsers_from_prefix = {} @@ -129,10 +133,10 @@ def from_callable_or_type( elif isinstance(field_out, ParserSpecification): # Handle nested parsers. nested_parser = field_out + child_from_prefix[field_out.intern_prefix] = nested_parser if nested_parser.has_required_args: has_required_args = True - args.extend(nested_parser.args) # Include nested subparsers. if nested_parser.subparsers is not None: @@ -143,15 +147,7 @@ def from_callable_or_type( subparsers, nested_parser.subparsers ) - # Include nested strings. - for ( - k, - v, - ) in nested_parser.helptext_from_nested_class_field_name.items(): - helptext_from_intern_prefixed_field_name[ - _strings.make_field_name([field.intern_name, k]) - ] = v - + # Helptext for this field; used as description for grouping arguments. class_field_name = _strings.make_field_name([field.intern_name]) if field.helptext is not None: helptext_from_intern_prefixed_field_name[ @@ -185,7 +181,9 @@ def from_callable_or_type( else _docstrings.get_callable_description(f) ), args=args, - helptext_from_nested_class_field_name=helptext_from_intern_prefixed_field_name, + field_list=field_list, + child_from_prefix=child_from_prefix, + helptext_from_intern_prefixed_field_name=helptext_from_intern_prefixed_field_name, subparsers=subparsers, subparsers_from_intern_prefix=subparsers_from_prefix, intern_prefix=intern_prefix, @@ -232,7 +230,11 @@ def apply( return leaves - def apply_args(self, parser: argparse.ArgumentParser) -> None: + def apply_args( + self, + parser: argparse.ArgumentParser, + parent: Optional[ParserSpecification] = None, + ) -> None: """Create defined arguments and subparsers.""" # Generate helptext. @@ -259,8 +261,12 @@ def format_group_name(prefix: str) -> str: arg.lowered.help is not argparse.SUPPRESS and arg.extern_prefix not in group_from_prefix ): - description = self.helptext_from_nested_class_field_name.get( - arg.extern_prefix + description = ( + parent.helptext_from_intern_prefixed_field_name.get( + arg.intern_prefix + ) + if parent is not None + else None ) group_from_prefix[arg.extern_prefix] = parser.add_argument_group( format_group_name(arg.extern_prefix), @@ -281,6 +287,9 @@ def format_group_name(prefix: str) -> str: assert arg.lowered.help is argparse.SUPPRESS arg.add_argument(group_from_prefix[""]) + for child in self.child_from_prefix.values(): + child.apply_args(parser, parent=self) + def handle_field( field: _fields.FieldDefinition, @@ -501,20 +510,6 @@ def from_field( subcommand_config, default=field.default ) - # Strip the subcommand config from the option type. - option_origin, annotations = _resolver.unwrap_annotated(option) # type: ignore - annotations = tuple( - a - for a in annotations - if not isinstance(a, _confstruct._SubcommandConfiguration) - ) - if len(annotations) == 0: - option = option_origin - else: - option = Annotated.__class_getitem__( # type: ignore - (option_origin,) + annotations - ) - subparser = ParserSpecification.from_callable_or_type( ( # Recursively apply markers. @@ -534,9 +529,9 @@ def from_field( # Apply prefix to helptext in nested classes in subparsers. subparser = dataclasses.replace( subparser, - helptext_from_nested_class_field_name={ + helptext_from_intern_prefixed_field_name={ _strings.make_field_name([intern_prefix, k]): v - for k, v in subparser.helptext_from_nested_class_field_name.items() + for k, v in subparser.helptext_from_intern_prefixed_field_name.items() }, ) parser_from_name[subcommand_name] = subparser From 7049ef0f2e5f5ae433c47b75178ad7a7fb05961c Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 19 Jan 2024 17:45:16 -0800 Subject: [PATCH 373/491] Fix edge case when default must be specified, and is specified in subcommand config --- src/tyro/_fields.py | 10 ++++++++- src/tyro/_parsers.py | 15 +++++++++++++ tests/test_conf.py | 52 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index dd2c4b97d..9c72f10d2 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -377,8 +377,16 @@ def _try_field_list_from_callable( f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: + # Check for default instances in subcommand configs. This is needed for + # is_nested_type() when arguments are not valid without a default, and this + # default is specified in the subcommand config. + f, found_subcommand_configs = _resolver.unwrap_annotated( + f, conf._confstruct._SubcommandConfiguration + ) + if len(found_subcommand_configs) > 0: + default_instance = found_subcommand_configs[0].default + # Unwrap generics. - f = _resolver.unwrap_annotated(f)[0] f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.apply_type_from_typevar(f, type_from_typevar) f = _resolver.unwrap_newtype_and_narrow_subtypes(f, default_instance) diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index bf9241043..3c8eb09d0 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -510,6 +510,21 @@ def from_field( subcommand_config, default=field.default ) + # Strip the subcommand config from the option type. + # Relevant: https://github.com/brentyi/tyro/pull/117 + option_origin, annotations = _resolver.unwrap_annotated(option) # type: ignore + annotations = tuple( + a + for a in annotations + if not isinstance(a, _confstruct._SubcommandConfiguration) + ) + if len(annotations) == 0: + option = option_origin + else: + option = Annotated.__class_getitem__( # type: ignore + (option_origin,) + annotations + ) + subparser = ParserSpecification.from_callable_or_type( ( # Recursively apply markers. diff --git a/tests/test_conf.py b/tests/test_conf.py index 3fcce313e..5321b0551 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -226,6 +226,58 @@ class Parent: ) == Parent(Nested1(Nested2(B(7)))) +def test_subparser_in_nested_with_metadata_suppressed() -> None: + @dataclasses.dataclass(frozen=True) + class A: + a: tyro.conf.Suppress[int] + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + @dataclasses.dataclass + class Nested2: + subcommand: Union[ + Annotated[A, tyro.conf.subcommand("command-a", default=A(7))], + Annotated[B, tyro.conf.subcommand("command-b", default=B(9))], + ] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2 + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert tyro.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert tyro.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert tyro.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert tyro.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + def test_subparser_in_nested_with_metadata_generic() -> None: @dataclasses.dataclass(frozen=True) class A: From d660da24da127282762c378c6a7b0c2a101b8674 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 20 Jan 2024 18:21:56 -0800 Subject: [PATCH 374/491] Support `typing.Self` (#120) --- src/tyro/_parsers.py | 2 +- src/tyro/_resolver.py | 14 ++- .../test_base_configs_nested_generated.py | 32 +++++++ .../test_conf_generated.py | 53 ++++++++++- .../test_helptext_generated.py | 25 +++++ .../test_self_type_generated.py | 92 +++++++++++++++++++ tests/test_self_type.py | 91 ++++++++++++++++++ 7 files changed, 305 insertions(+), 4 deletions(-) create mode 100644 tests/test_py311_generated/test_self_type_generated.py create mode 100644 tests/test_self_type.py diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 3c8eb09d0..736b13e9d 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -95,7 +95,7 @@ def from_callable_or_type( # superclass. if f in parent_classes and f is not dict: raise _instantiators.UnsupportedTypeAnnotationError( - f"Found a cyclic dataclass dependency with type {f}." + f"Found a cyclic dependency with type {f}." ) # TODO: we are abusing the (minor) distinctions between types, classes, and diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index b28a94540..5e5e5726f 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -2,6 +2,7 @@ import collections.abc import copy import dataclasses +import inspect import sys import types import warnings @@ -20,7 +21,7 @@ cast, ) -from typing_extensions import Annotated, get_args, get_origin, get_type_hints +from typing_extensions import Annotated, Self, get_args, get_origin, get_type_hints from . import _fields, _unsafe_cache from ._typing import TypeForm @@ -61,8 +62,17 @@ def resolve_generic_types( # We'll ignore NewType when getting the origin + args for generics. origin_cls = get_origin(unwrap_newtype(cls)[0]) + type_from_typevar: Dict[TypeVar, TypeForm[Any]] = {} + + # Support typing.Self. + # We'll do this by pretending that `Self` is a TypeVar... + if hasattr(cls, "__self__"): + self_type = getattr(cls, "__self__") + if inspect.isclass(self_type): + type_from_typevar[cast(TypeVar, Self)] = self_type + else: + type_from_typevar[cast(TypeVar, Self)] = self_type.__class__ - type_from_typevar = {} if ( # Apply some heuristics for generic types. Should revisit this. origin_cls is not None diff --git a/tests/test_py311_generated/test_base_configs_nested_generated.py b/tests/test_py311_generated/test_base_configs_nested_generated.py index ba83318e2..8134dbe09 100644 --- a/tests/test_py311_generated/test_base_configs_nested_generated.py +++ b/tests/test_py311_generated/test_base_configs_nested_generated.py @@ -177,3 +177,35 @@ def main(cfg: BaseConfig) -> BaseConfig: ), DataConfig(2), ) + + +def test_pernicious_override(): + """From: https://github.com/nerfstudio-project/nerfstudio/issues/2789 + + Situation where we: + - have a default value in the config class + - override that default value with a subcommand annotation + - override it again with a default instance + """ + assert ( + tyro.cli( + BaseConfig, + default=BaseConfig( + "test", + "test", + ExperimentConfig( + dataset="mnist", + optimizer=AdamOptimizer(), + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + seed=0, + activation=nn.ReLU, + ), + DataConfig(0), + ), + args="small small-data".split(" "), + ).data_config.test + == 0 + ) diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index 75dcc1e08..5945752ec 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -224,6 +224,57 @@ class Parent: ) == Parent(Nested1(Nested2(B(7)))) +def test_subparser_in_nested_with_metadata_suppressed() -> None: + @dataclasses.dataclass(frozen=True) + class A: + a: tyro.conf.Suppress[int] + + @dataclasses.dataclass + class B: + b: int + a: A = A(5) + + @dataclasses.dataclass + class Nested2: + subcommand: Annotated[ + A, tyro.conf.subcommand("command-a", default=A(7)) + ] | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] + + @dataclasses.dataclass + class Nested1: + nested2: Nested2 + + @dataclasses.dataclass + class Parent: + nested1: Nested1 + + assert tyro.cli( + Parent, + args="nested1.nested2.subcommand:command-a".split(" "), + ) == Parent(Nested1(Nested2(A(7)))) + assert tyro.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-a --nested1.nested2.subcommand.a 3".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(A(3)))) + + assert tyro.cli( + Parent, + args="nested1.nested2.subcommand:command-b".split(" "), + ) == Parent(Nested1(Nested2(B(9)))) + assert tyro.cli( + Parent, + args=( + "nested1.nested2.subcommand:command-b --nested1.nested2.subcommand.b 7".split( + " " + ) + ), + ) == Parent(Nested1(Nested2(B(7)))) + + def test_subparser_in_nested_with_metadata_generic() -> None: @dataclasses.dataclass(frozen=True) class A: @@ -1264,7 +1315,7 @@ def instantiate_dataclasses( classes: Tuple[Type[T], ...], args: List[str] ) -> Tuple[T, ...]: return tyro.cli( - tyro.conf.OmitArgPrefixes[ + tyro.conf.OmitArgPrefixes[ # type: ignore # Convert (type1, type2) into Tuple[type1, type2] Tuple.__getitem__( # type: ignore tuple(Annotated[c, tyro.conf.arg(name=c.__name__)] for c in classes) diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py index ef47b2807..48952a959 100644 --- a/tests/test_py311_generated/test_helptext_generated.py +++ b/tests/test_py311_generated/test_helptext_generated.py @@ -48,6 +48,31 @@ class Helptext: assert "Documentation 3 (default: 3)" in helptext +def test_helptext_sphinx_autodoc_style() -> None: + @dataclasses.dataclass + class Helptext: + """This docstring should be printed as a description.""" + + x: int #: Documentation 1 + + #:Documentation 2 + y: Annotated[int, "ignored"] + z: int = 3 + + helptext = get_helptext(Helptext) + assert cast(str, helptext) in helptext + assert "x INT" in helptext + assert "y INT" in helptext + assert "z INT" in helptext + assert "Documentation 1 (required)" in helptext + assert ": Documentation 1" not in helptext + assert "Documentation 2 (required)" in helptext + assert ":Documentation 2" not in helptext + + # :Documentation 2 should not be applied to `z`. + assert helptext.count("Documentation 2") == 1 + + def test_helptext_from_class_docstring() -> None: @dataclasses.dataclass class Helptext2: diff --git a/tests/test_py311_generated/test_self_type_generated.py b/tests/test_py311_generated/test_self_type_generated.py new file mode 100644 index 000000000..45058b305 --- /dev/null +++ b/tests/test_py311_generated/test_self_type_generated.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from typing import Self + +import pytest + +import tyro + + +class TestClass: + def __init__(self, a: int, b: int) -> None: + self.a = a + self.b = b + + def method1(self, x: Self) -> None: + self.effect = x + + @classmethod + def method2(cls, x: Self) -> TestClass: + return x + + # Self is not valid in static methods. + # https://peps.python.org/pep-0673/#valid-locations-for-self + # + # @staticmethod + # def method3(x: Self) -> TestClass: + # return x + + +class TestSubclass(TestClass): + ... + + +def test_method() -> None: + x = TestClass(0, 0) + with pytest.raises(SystemExit): + tyro.cli(x.method1, args=[]) + + assert tyro.cli(x.method1, args="--x.a 3 --x.b 3".split(" ")) is None + assert x.effect.a == 3 and x.effect.b == 3 + assert isinstance(x, TestClass) + + +def test_classmethod() -> None: + x = TestClass(0, 0) + with pytest.raises(SystemExit): + tyro.cli(x.method2, args=[]) + with pytest.raises(SystemExit): + tyro.cli(TestClass.method2, args=[]) + + y = tyro.cli(x.method2, args="--x.a 3 --x.b 3".split(" ")) + assert y.a == 3 + assert y.b == 3 + assert isinstance(y, TestClass) + + y = tyro.cli(TestClass.method2, args="--x.a 3 --x.b 3".split(" ")) + assert y.a == 3 + assert y.b == 3 + assert isinstance(y, TestClass) + + +def test_subclass_method() -> None: + x = TestSubclass(0, 0) + with pytest.raises(SystemExit): + tyro.cli(x.method1, args=[]) + + assert tyro.cli(x.method1, args="--x.a 3 --x.b 3".split(" ")) is None + assert x.effect.a == 3 and x.effect.b == 3 + assert isinstance(x, TestSubclass) + + y = tyro.cli(x.method2, args="--x.a 3 --x.b 3".split(" ")) + assert y.a == 3 + assert y.b == 3 + assert isinstance(y, TestClass) + + +def test_subclass_classmethod() -> None: + x = TestSubclass(0, 0) + with pytest.raises(SystemExit): + tyro.cli(x.method2, args=[]) + with pytest.raises(SystemExit): + tyro.cli(TestSubclass.method2, args=[]) + + y = tyro.cli(x.method2, args="--x.a 3 --x.b 3".split(" ")) + assert y.a == 3 + assert y.b == 3 + assert isinstance(y, TestClass) + + y = tyro.cli(TestSubclass.method2, args="--x.a 3 --x.b 3".split(" ")) + assert y.a == 3 + assert y.b == 3 + assert isinstance(y, TestClass) diff --git a/tests/test_self_type.py b/tests/test_self_type.py new file mode 100644 index 000000000..0b9ffbfc7 --- /dev/null +++ b/tests/test_self_type.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import pytest +from typing_extensions import Self + +import tyro + + +class TestClass: + def __init__(self, a: int, b: int) -> None: + self.a = a + self.b = b + + def method1(self, x: Self) -> None: + self.effect = x + + @classmethod + def method2(cls, x: Self) -> TestClass: + return x + + # Self is not valid in static methods. + # https://peps.python.org/pep-0673/#valid-locations-for-self + # + # @staticmethod + # def method3(x: Self) -> TestClass: + # return x + + +class TestSubclass(TestClass): + ... + + +def test_method() -> None: + x = TestClass(0, 0) + with pytest.raises(SystemExit): + tyro.cli(x.method1, args=[]) + + assert tyro.cli(x.method1, args="--x.a 3 --x.b 3".split(" ")) is None + assert x.effect.a == 3 and x.effect.b == 3 + assert isinstance(x, TestClass) + + +def test_classmethod() -> None: + x = TestClass(0, 0) + with pytest.raises(SystemExit): + tyro.cli(x.method2, args=[]) + with pytest.raises(SystemExit): + tyro.cli(TestClass.method2, args=[]) + + y = tyro.cli(x.method2, args="--x.a 3 --x.b 3".split(" ")) + assert y.a == 3 + assert y.b == 3 + assert isinstance(y, TestClass) + + y = tyro.cli(TestClass.method2, args="--x.a 3 --x.b 3".split(" ")) + assert y.a == 3 + assert y.b == 3 + assert isinstance(y, TestClass) + + +def test_subclass_method() -> None: + x = TestSubclass(0, 0) + with pytest.raises(SystemExit): + tyro.cli(x.method1, args=[]) + + assert tyro.cli(x.method1, args="--x.a 3 --x.b 3".split(" ")) is None + assert x.effect.a == 3 and x.effect.b == 3 + assert isinstance(x, TestSubclass) + + y = tyro.cli(x.method2, args="--x.a 3 --x.b 3".split(" ")) + assert y.a == 3 + assert y.b == 3 + assert isinstance(y, TestClass) + + +def test_subclass_classmethod() -> None: + x = TestSubclass(0, 0) + with pytest.raises(SystemExit): + tyro.cli(x.method2, args=[]) + with pytest.raises(SystemExit): + tyro.cli(TestSubclass.method2, args=[]) + + y = tyro.cli(x.method2, args="--x.a 3 --x.b 3".split(" ")) + assert y.a == 3 + assert y.b == 3 + assert isinstance(y, TestClass) + + y = tyro.cli(TestSubclass.method2, args="--x.a 3 --x.b 3".split(" ")) + assert y.a == 3 + assert y.b == 3 + assert isinstance(y, TestClass) From 402f7a20c7ce4c6e03cbd6b3f022c998d1238c00 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Feb 2024 13:04:51 -0800 Subject: [PATCH 375/491] Fix type narrowing edge case, bump to 0.7.1 (#122) * Fix type narrowing edge case * Fix * Fix * ruff --- pyproject.toml | 2 +- src/tyro/__init__.py | 4 ++++ src/tyro/_calling.py | 2 -- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2b58863f0..ac685f3a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.7.0" +version = "0.7.1" description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index 947254dc3..df35b703f 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -11,3 +11,7 @@ # Deprecated interface. if not TYPE_CHECKING: from ._deprecated import * # noqa + + +# TODO: this should be synchronized automatically with the pyproject.toml. +__version__ = "0.7.1" diff --git a/src/tyro/_calling.py b/src/tyro/_calling.py index ffadc5592..d4fc46d8e 100644 --- a/src/tyro/_calling.py +++ b/src/tyro/_calling.py @@ -138,8 +138,6 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: subparser_name = get_value_from_arg(subparser_dest) else: assert subparser_def.default_instance not in _fields.MISSING_SINGLETONS - default_instance = subparser_def.default_instance - # assert default_instance is not None subparser_name = None if subparser_name is None: From 1791cdd61dbc3c45c40400f42c4ed6ab8788659e Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Feb 2024 13:19:51 -0800 Subject: [PATCH 376/491] Pin pyright Signed-off-by: Brent Yi --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ac685f3a6..f4171b247 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dev = [ "omegaconf>=2.2.2", "attrs>=21.4.0", "torch>=1.10.0", - "pyright>=1.1.264", + "pyright==1.1.348", "ruff>=0.1.13", "mypy>=1.4.1", "numpy>=1.20.0", From 165b83dc99747866c8e2c171f6553da3c3b538fd Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 1 Feb 2024 13:26:20 -0800 Subject: [PATCH 377/491] Pin pyright Signed-off-by: Brent Yi --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f4171b247..f8e4514ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dev = [ "omegaconf>=2.2.2", "attrs>=21.4.0", "torch>=1.10.0", - "pyright==1.1.348", + "pyright==1.1.347", "ruff>=0.1.13", "mypy>=1.4.1", "numpy>=1.20.0", From 30803e8a4ee871693f7b021be84265780ddc0146 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 2 Feb 2024 01:23:29 -0800 Subject: [PATCH 378/491] Resolve pyright errors --- pyproject.toml | 2 +- src/tyro/_resolver.py | 2 +- tests/test_nested.py | 6 ++++-- tests/test_py311_generated/test_nested_generated.py | 6 ++++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f8e4514ca..4cd3f272c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dev = [ "omegaconf>=2.2.2", "attrs>=21.4.0", "torch>=1.10.0", - "pyright==1.1.347", + "pyright>=1.1.349", "ruff>=0.1.13", "mypy>=1.4.1", "numpy>=1.20.0", diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 5e5e5726f..959dd70cb 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -238,7 +238,7 @@ def narrow_collection_types( def unwrap_annotated( typ: TypeOrCallable, - search_type: TypeForm[MetadataType] = Any, # type: ignore + search_type: TypeForm[MetadataType] = cast(TypeForm[Any], Any), ) -> Tuple[TypeOrCallable, Tuple[MetadataType, ...]]: """Helper for parsing typing.Annotated types. diff --git a/tests/test_nested.py b/tests/test_nested.py index 530ceb006..21fa874c1 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -520,8 +520,10 @@ class C: with pytest.warns(UserWarning): assert tyro.cli( - Union[A, Annotated[B, None]], default=C(3), args=["c", "--c", "2"] - ) == C(2) # type: ignore + Union[A, Annotated[B, None]], # type: ignore + default=C(3), + args=["c", "--c", "2"], + ) == C(2) def test_optional_subparser() -> None: diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index 34a7b077a..389e35b2b 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -529,8 +529,10 @@ class C: with pytest.warns(UserWarning): assert tyro.cli( - A | Annotated[B, None], default=C(3), args=["c", "--c", "2"] - ) == C(2) # type: ignore + A | Annotated[B, None], # type: ignore + default=C(3), + args=["c", "--c", "2"], + ) == C(2) def test_optional_subparser() -> None: From 05039507a8a4e058b38879d2a9facbbbca65dbf4 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 6 Feb 2024 22:25:02 -0800 Subject: [PATCH 379/491] (Should break tests) Add failing docstring tests cc #123 --- tests/test_helptext.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 11b267658..8aebd0880 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -164,13 +164,15 @@ class ChildClass(ParentClass): """This docstring should be printed as a description.""" def main(x: ParentClass = ChildClass(x=5, y=5)) -> Any: + """Main function.""" return x helptext = get_helptext(main) + assert "Main function." in helptext assert "Documentation 1" in helptext assert "Documentation 2" in helptext assert "__not__" not in helptext - assert "should be printed" in helptext + assert helptext.count("should be printed") == 1 def test_helptext_nested() -> None: @@ -197,11 +199,14 @@ def main_no_docstring(a: Inner) -> None: helptext = get_helptext(main_with_docstring) assert "Documented in function" in helptext and str(Inner.__doc__) not in helptext + assert "main_with_docstring." in helptext + assert "Args:" not in helptext assert "Args:" not in helptext assert "Hello world!" in helptext helptext = get_helptext(main_no_docstring) assert "Something" in helptext + assert "main_no_docstring." in helptext assert "Args:" not in helptext assert "Hello world!" in helptext From ea045c1887fb0e0e2c8bd1512b050931dca862ee Mon Sep 17 00:00:00 2001 From: Kevin Chen <79341521+kevinddchen@users.noreply.github.com> Date: Tue, 6 Feb 2024 22:31:05 -0800 Subject: [PATCH 380/491] remove extraneous description setting (#124) --- src/tyro/_parsers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 736b13e9d..af929259c 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -237,9 +237,6 @@ def apply_args( ) -> None: """Create defined arguments and subparsers.""" - # Generate helptext. - parser.description = self.description - # Make argument groups. def format_group_name(prefix: str) -> str: return (prefix + " options").strip() From ffb4b6e0a1b76e08b1b7b41c48a0b09732cc7015 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 6 Feb 2024 22:35:03 -0800 Subject: [PATCH 381/491] Bump version to 0.7.2 --- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4cd3f272c..488c28452 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.7.1" +version = "0.7.2" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index df35b703f..c25c01828 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.7.1" +__version__ = "0.7.2" From d560e5f6878bfe4d5193eccbcc93b695fbe6fc13 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 14 Feb 2024 18:40:39 -0800 Subject: [PATCH 382/491] Minor docs fix for tab completions Signed-off-by: Brent Yi --- docs/source/tab_completion.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/tab_completion.md b/docs/source/tab_completion.md index 7984d9a06..a7ac735f0 100644 --- a/docs/source/tab_completion.md +++ b/docs/source/tab_completion.md @@ -6,8 +6,8 @@ shells without any source code modification. Completion scripts can be generated by passing the `--tyro-write-completion {bash/zsh/tcsh} PATH` flag to a tyro CLI. This generates a completion script via [shtab](https://docs.iterative.ai/shtab/) and -prints it to stdout. To set up tab completion, the printed script simply needs -to be written somewhere where your shell will find it. +writes it to a specified file. To set up tab completion, the printed script simply +needs to be written somewhere where your shell will find it. ## Zsh From f674d1d7e535b1abd023efde71dd2f659f38aac1 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 14 Feb 2024 23:46:00 -0800 Subject: [PATCH 383/491] Fix dictionary example commands in docs --- .../examples/04_additional/02_dictionaries.rst | 16 ++++++++++++---- examples/04_additional/02_dictionaries.py | 5 +++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/source/examples/04_additional/02_dictionaries.rst b/docs/source/examples/04_additional/02_dictionaries.rst index d8f4dd51c..8002b2504 100644 --- a/docs/source/examples/04_additional/02_dictionaries.rst +++ b/docs/source/examples/04_additional/02_dictionaries.rst @@ -68,14 +68,22 @@ Dictionary inputs can be specified using either a standard ``Dict[K, V]`` annota .. raw:: html - python 04_additional/02_dictionaries.py --typed-dict.learning-rate 3e-4 + python 04_additional/02_dictionaries.py --typed-dict-a.learning-rate 3e-4 -.. program-output:: python ../../examples/04_additional/02_dictionaries.py --typed-dict.learning-rate 3e-4 +.. program-output:: python ../../examples/04_additional/02_dictionaries.py --typed-dict-a.learning-rate 3e-4 ------------ .. raw:: html - python 04_additional/02_dictionaries.py --typed-dict.betas 0.9 0.999 + python 04_additional/02_dictionaries.py --typed-dict-a.learning-rate 3e-4 --typed-dict-b.betas 0.9 0.999 -.. program-output:: python ../../examples/04_additional/02_dictionaries.py --typed-dict.betas 0.9 0.999 +.. program-output:: python ../../examples/04_additional/02_dictionaries.py --typed-dict-a.learning-rate 3e-4 --typed-dict-b.betas 0.9 0.999 + +------------ + +.. raw:: html + + python 04_additional/02_dictionaries.py --typed-dict-b.betas 0.9 0.999 + +.. program-output:: python ../../examples/04_additional/02_dictionaries.py --typed-dict-b.betas 0.9 0.999 diff --git a/examples/04_additional/02_dictionaries.py b/examples/04_additional/02_dictionaries.py index 15d4bbfd0..2dbfce9f0 100644 --- a/examples/04_additional/02_dictionaries.py +++ b/examples/04_additional/02_dictionaries.py @@ -5,8 +5,9 @@ Usage: `python ./02_dictionaries.py --help` -`python ./02_dictionaries.py --typed-dict.learning-rate 3e-4` -`python ./02_dictionaries.py --typed-dict.betas 0.9 0.999` +`python ./02_dictionaries.py --typed-dict-a.learning-rate 3e-4` +`python ./02_dictionaries.py --typed-dict-a.learning-rate 3e-4 --typed-dict-b.betas 0.9 0.999` +`python ./02_dictionaries.py --typed-dict-b.betas 0.9 0.999` """ from typing import Dict, Tuple, TypedDict From a26cf34232651b35635959eb8cf2310433bab34b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 16 Feb 2024 02:35:48 -0800 Subject: [PATCH 384/491] Fix `init=False` and forward annotations for attrs (#126) * Fix `init=False` and forward annotations for attrs * Test forward references for Pydantic, add init=False test for attrs * init_var=False test for pydantic * ruff --- src/tyro/_fields.py | 7 ++++++ tests/test_attrs.py | 4 ++++ .../test_attrs_generated.py | 4 ++++ .../test_helptext_generated.py | 7 +++++- .../test_pydantic_generated.py | 22 +++++++++++++------ tests/test_pydantic.py | 22 +++++++++++++------ 6 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 9c72f10d2..cb300fe01 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -702,9 +702,16 @@ def _field_list_from_attrs( ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: assert attr is not None + # Resolve forward references in-place, if any exist. + attr.resolve_types(cls) + # Handle attr classes. field_list = [] for attr_field in attr.fields(cls): + # Skip fields with init=False. + if not attr_field.init: + continue + # Default handling. name = attr_field.name default = attr_field.default diff --git a/tests/test_attrs.py b/tests/test_attrs.py index a38bb95fd..6f44068d3 100644 --- a/tests/test_attrs.py +++ b/tests/test_attrs.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import contextlib import io import pathlib @@ -18,6 +20,7 @@ class ManyTypesA: s: str = attr.ib() f: float = attr.ib() p: pathlib.Path = attr.ib() + ignored: int = attr.ib(default=3, init=False) # We can directly pass a dataclass to `tyro.cli()`: assert tyro.cli( @@ -109,6 +112,7 @@ class ManyTypesB: i: int = attr.ib() s: str = attr.ib() f: float = attr.ib(default=1.0) + k: float = attr.ib(default=1.0) assert tyro.cli( ManyTypesB, diff --git a/tests/test_py311_generated/test_attrs_generated.py b/tests/test_py311_generated/test_attrs_generated.py index a38bb95fd..6f44068d3 100644 --- a/tests/test_py311_generated/test_attrs_generated.py +++ b/tests/test_py311_generated/test_attrs_generated.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import contextlib import io import pathlib @@ -18,6 +20,7 @@ class ManyTypesA: s: str = attr.ib() f: float = attr.ib() p: pathlib.Path = attr.ib() + ignored: int = attr.ib(default=3, init=False) # We can directly pass a dataclass to `tyro.cli()`: assert tyro.cli( @@ -109,6 +112,7 @@ class ManyTypesB: i: int = attr.ib() s: str = attr.ib() f: float = attr.ib(default=1.0) + k: float = attr.ib(default=1.0) assert tyro.cli( ManyTypesB, diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py index 48952a959..d945804b3 100644 --- a/tests/test_py311_generated/test_helptext_generated.py +++ b/tests/test_py311_generated/test_helptext_generated.py @@ -176,13 +176,15 @@ class ChildClass(ParentClass): """This docstring should be printed as a description.""" def main(x: ParentClass = ChildClass(x=5, y=5)) -> Any: + """Main function.""" return x helptext = get_helptext(main) + assert "Main function." in helptext assert "Documentation 1" in helptext assert "Documentation 2" in helptext assert "__not__" not in helptext - assert "should be printed" in helptext + assert helptext.count("should be printed") == 1 def test_helptext_nested() -> None: @@ -209,11 +211,14 @@ def main_no_docstring(a: Inner) -> None: helptext = get_helptext(main_with_docstring) assert "Documented in function" in helptext and str(Inner.__doc__) not in helptext + assert "main_with_docstring." in helptext + assert "Args:" not in helptext assert "Args:" not in helptext assert "Hello world!" in helptext helptext = get_helptext(main_no_docstring) assert "Something" in helptext + assert "main_no_docstring." in helptext assert "Args:" not in helptext assert "Hello world!" in helptext diff --git a/tests/test_py311_generated/test_pydantic_generated.py b/tests/test_py311_generated/test_pydantic_generated.py index e60b8d3fa..c54ef87df 100644 --- a/tests/test_py311_generated/test_pydantic_generated.py +++ b/tests/test_py311_generated/test_pydantic_generated.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import contextlib import io import pathlib @@ -16,6 +18,7 @@ class ManyTypesA(BaseModel): s: str = "hello" f: float = Field(default_factory=lambda: 3.0) p: pathlib.Path + ignored: float = Field(default_factory=lambda: 3.0, init_var=False) # We can directly pass a dataclass to `tyro.cli()`: assert tyro.cli( @@ -141,7 +144,7 @@ class Inside(BaseModel): x: int = 1 class Outside(BaseModel): - i: Inside = Inside(x=2) + i: "Inside" = Inside(x=2) assert tyro.cli(Outside, args=[]).i.x == 2, ( "Expected x value from the default instance", @@ -165,15 +168,20 @@ class Outside(BaseModel): assert tyro.cli(Outside, args=["--m.i.x", "3"]).m.i.x == 3 -def test_pydantic_v1_nested_default_instance() -> None: - class Inside(v1.BaseModel): - x: int = 1 +# Updating forward references in Pydantic v1 requires that these classes are global. - class Middle(v1.BaseModel): - i: Inside +class InsideV1(v1.BaseModel): + x: int = 1 + + +class MiddleV1(v1.BaseModel): + i: InsideV1 + + +def test_pydantic_v1_nested_default_instance() -> None: class Outside(v1.BaseModel): - m: Middle = Middle(i=Inside(x=2)) + m: MiddleV1 = MiddleV1(i=InsideV1(x=2)) assert tyro.cli(Outside, args=[]).m.i.x == 2, ( "Expected x value from the default instance", diff --git a/tests/test_pydantic.py b/tests/test_pydantic.py index 8882404ae..5a588e71c 100644 --- a/tests/test_pydantic.py +++ b/tests/test_pydantic.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import contextlib import io import pathlib @@ -17,6 +19,7 @@ class ManyTypesA(BaseModel): s: str = "hello" f: float = Field(default_factory=lambda: 3.0) p: pathlib.Path + ignored: float = Field(default_factory=lambda: 3.0, init_var=False) # We can directly pass a dataclass to `tyro.cli()`: assert tyro.cli( @@ -142,7 +145,7 @@ class Inside(BaseModel): x: int = 1 class Outside(BaseModel): - i: Inside = Inside(x=2) + i: "Inside" = Inside(x=2) assert tyro.cli(Outside, args=[]).i.x == 2, ( "Expected x value from the default instance", @@ -166,15 +169,20 @@ class Outside(BaseModel): assert tyro.cli(Outside, args=["--m.i.x", "3"]).m.i.x == 3 -def test_pydantic_v1_nested_default_instance() -> None: - class Inside(v1.BaseModel): - x: int = 1 +# Updating forward references in Pydantic v1 requires that these classes are global. - class Middle(v1.BaseModel): - i: Inside +class InsideV1(v1.BaseModel): + x: int = 1 + + +class MiddleV1(v1.BaseModel): + i: InsideV1 + + +def test_pydantic_v1_nested_default_instance() -> None: class Outside(v1.BaseModel): - m: Middle = Middle(i=Inside(x=2)) + m: MiddleV1 = MiddleV1(i=InsideV1(x=2)) assert tyro.cli(Outside, args=[]).m.i.x == 2, ( "Expected x value from the default instance", From 28082f8e04846cfd36de11674e9292e6a60c6a33 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 16 Feb 2024 11:31:18 -0800 Subject: [PATCH 385/491] Bump version --- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 488c28452..73b90fbe7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.7.2" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.7.3" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index c25c01828..e9ff70179 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.7.2" +__version__ = "0.7.3" From f70d6fc8aeb728e690a19831ae8766a947f71d9f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 3 Apr 2024 17:12:30 -0700 Subject: [PATCH 386/491] Support `X | Y` and `list[T]` for older versions of Python (#134) * Support `X | Y` and `list[T]` for older versions of Python * Fix type checking errors * Use new syntax in examples (likely breaks CI) * Add hasattr for __annotations__ * Coverage suppress --- .../examples/01_basics/03_collections.rst | 15 +++-- docs/source/examples/01_basics/05_flags.rst | 3 +- docs/source/examples/01_basics/07_unions.rst | 14 ++--- .../examples/02_nesting/02_subcommands.rst | 5 +- .../02_nesting/03_multiple_subcommands.rst | 8 +-- .../02_nesting/04_nesting_in_containers.rst | 7 +-- .../03_config_systems/01_base_configs.rst | 4 +- .../04_additional/02_dictionaries.rst | 8 +-- .../examples/04_additional/03_tuples.rst | 6 +- examples/01_basics/03_collections.py | 15 +++-- examples/01_basics/05_flags.py | 3 +- examples/01_basics/07_unions.py | 14 ++--- examples/02_nesting/02_subcommands.py | 5 +- .../02_nesting/03_multiple_subcommands.py | 9 +-- .../02_nesting/04_nesting_in_containers.py | 8 +-- examples/03_config_systems/01_base_configs.py | 4 +- examples/04_additional/02_dictionaries.py | 8 +-- examples/04_additional/03_tuples.py | 6 +- examples/04_additional/08_pydantic.py | 1 + pyproject.toml | 10 ++-- src/tyro/_argparse_formatter.py | 2 +- src/tyro/_arguments.py | 1 + src/tyro/_cli.py | 19 +++--- src/tyro/_fields.py | 15 ++++- src/tyro/_instantiators.py | 12 ++-- src/tyro/_parsers.py | 12 ++-- src/tyro/_resolver.py | 59 +++++++++++-------- src/tyro/conf/_confstruct.py | 12 ++-- src/tyro/extras/__init__.py | 2 +- src/tyro/extras/_serialization.py | 1 + src/tyro/extras/_subcommand_cli_from_dict.py | 6 +- tests/test_errors_new_annotations.py | 35 ----------- tests/test_flax_min_py38.py | 1 + tests/test_missing.py | 1 + tests/test_mixed_unions.py | 1 - .../test_conf_generated.py | 31 +++++----- .../test_errors_new_annotations_generated.py | 6 +- .../test_flax_min_py38_generated.py | 1 + .../test_missing_generated.py | 1 + .../test_mixed_unions_generated.py | 1 - .../test_pydantic_with_newtype_generated.py | 6 +- .../test_self_type_generated.py | 3 +- ...t_unsupported_but_should_work_generated.py | 1 + tests/test_pydantic_with_newtype.py | 6 +- tests/test_self_type.py | 37 ++++++------ tests/test_unsupported_but_should_work.py | 1 + 46 files changed, 199 insertions(+), 227 deletions(-) delete mode 100644 tests/test_errors_new_annotations.py diff --git a/docs/source/examples/01_basics/03_collections.rst b/docs/source/examples/01_basics/03_collections.rst index d939cd48c..f2ea82004 100644 --- a/docs/source/examples/01_basics/03_collections.rst +++ b/docs/source/examples/01_basics/03_collections.rst @@ -5,9 +5,9 @@ Multi-value Arguments ========================================== -Arguments of both fixed and variable lengths can be annotated with standard Python -collection types: ``typing.List[T]``\ , ``typing.Tuple[T1, T2]``\ , etc. In Python >=3.9, -``list[T]`` and ``tuple[T]`` are also supported. +Arguments of both fixed and variable lengths can be annotated with standard +Python collection types. For Python 3.7 and 3.8, we can use either +``from __future__ import annotations`` or\ ``typing.List[T]``\ , ``typing.Tuple[T1, T2]``\ , etc. @@ -17,20 +17,19 @@ collection types: ``typing.List[T]``\ , ``typing.Tuple[T1, T2]``\ , etc. In Pyth import dataclasses import pathlib - from typing import Tuple import tyro @dataclasses.dataclass(frozen=True) class TrainConfig: - # Example of a variable-length tuple. `typing.List`, `typing.Sequence`, - # `typing.Set`, `typing.Dict`, etc are all supported as well. - dataset_sources: Tuple[pathlib.Path, ...] + # Example of a variable-length tuple. `list[T]`, `set[T]`, + # `dict[K, V]`, etc are supported as well. + dataset_sources: tuple[pathlib.Path, ...] """Paths to load training data from. This can be multiple!""" # Fixed-length tuples are also okay. - image_dimensions: Tuple[int, int] = (32, 32) + image_dimensions: tuple[int, int] = (32, 32) """Height and width of some image data.""" diff --git a/docs/source/examples/01_basics/05_flags.rst b/docs/source/examples/01_basics/05_flags.rst index 66c807af9..47ada767d 100644 --- a/docs/source/examples/01_basics/05_flags.rst +++ b/docs/source/examples/01_basics/05_flags.rst @@ -17,7 +17,6 @@ To turn off conversion, see :class:`tyro.conf.FlagConversionOff`. import dataclasses - from typing import Optional import tyro @@ -28,7 +27,7 @@ To turn off conversion, see :class:`tyro.conf.FlagConversionOff`. boolean: bool # Optional boolean. Same as above, but can be omitted. - optional_boolean: Optional[bool] = None + optional_boolean: bool | None = None # Pass --flag-a in to set this value to True. flag_a: bool = False diff --git a/docs/source/examples/01_basics/07_unions.rst b/docs/source/examples/01_basics/07_unions.rst index 67438832e..c188f15a8 100644 --- a/docs/source/examples/01_basics/07_unions.rst +++ b/docs/source/examples/01_basics/07_unions.rst @@ -5,7 +5,7 @@ Unions ========================================== -``typing.Union[]`` can be used to expand inputs to multiple types. +``X | Y`` or ``typing.Union[X, Y]`` can be used to expand inputs to multiple types. @@ -15,7 +15,7 @@ Unions import dataclasses import enum - from typing import Literal, Optional, Tuple, Union + from typing import Literal, Optional import tyro @@ -29,19 +29,19 @@ Unions @dataclasses.dataclass(frozen=True) class Args: # Unions can be used to specify multiple allowable types. - union_over_types: Union[int, str] = 0 - string_or_enum: Union[Literal["red", "green"], Color] = "red" + union_over_types: int | str = 0 + string_or_enum: Literal["red", "green"] | Color = "red" # Unions also work over more complex nested types. - union_over_tuples: Union[Tuple[int, int], Tuple[str]] = ("1",) + union_over_tuples: tuple[int, int] | tuple[str] = ("1",) # And can be nested in other types. - tuple_of_string_or_enum: Tuple[Union[Literal["red", "green"], Color], ...] = ( + tuple_of_string_or_enum: tuple[Literal["red", "green"] | Color, ...] = ( "red", Color.RED, ) - # Optional[T] is equivalent to Union[T, None]. + # Optional[T] is equivalent to `T | None`. integer: Optional[Literal[0, 1, 2, 3]] = None diff --git a/docs/source/examples/02_nesting/02_subcommands.rst b/docs/source/examples/02_nesting/02_subcommands.rst index 38b434363..09a03f0ed 100644 --- a/docs/source/examples/02_nesting/02_subcommands.rst +++ b/docs/source/examples/02_nesting/02_subcommands.rst @@ -19,7 +19,6 @@ For configuring subcommands beyond what can be expressed with type annotations, from __future__ import annotations import dataclasses - from typing import Union import tyro @@ -39,12 +38,12 @@ For configuring subcommands beyond what can be expressed with type annotations, all: bool = False - def main(cmd: Union[Checkout, Commit]) -> None: + def main(cmd: Checkout | Commit) -> None: print(cmd) if __name__ == "__main__": - # Note that we can also pass `Union[Checkout, Command]` directly into + # Note that we can also pass `Checkout | Command` directly into # `tyro.cli()`; this is understood by tyro and pyright, but unfortunately not by # mypy. tyro.cli(main) diff --git a/docs/source/examples/02_nesting/03_multiple_subcommands.rst b/docs/source/examples/02_nesting/03_multiple_subcommands.rst index 480ddbee4..9a0a7406a 100644 --- a/docs/source/examples/02_nesting/03_multiple_subcommands.rst +++ b/docs/source/examples/02_nesting/03_multiple_subcommands.rst @@ -16,7 +16,7 @@ Multiple unions over nested types are populated using a series of subcommands. from __future__ import annotations import dataclasses - from typing import Literal, Tuple, Union + from typing import Literal import tyro @@ -41,7 +41,7 @@ Multiple unions over nested types are populated using a series of subcommands. @dataclasses.dataclass class Adam: learning_rate: float = 1e-3 - betas: Tuple[float, float] = (0.9, 0.999) + betas: tuple[float, float] = (0.9, 0.999) @dataclasses.dataclass @@ -54,8 +54,8 @@ Multiple unions over nested types are populated using a series of subcommands. @tyro.conf.configure(tyro.conf.ConsolidateSubcommandArgs) def train( - dataset: Union[Mnist, ImageNet] = Mnist(), - optimizer: Union[Adam, Sgd] = Adam(), + dataset: Mnist | ImageNet = Mnist(), + optimizer: Adam | Sgd = Adam(), ) -> None: """Example training script. diff --git a/docs/source/examples/02_nesting/04_nesting_in_containers.rst b/docs/source/examples/02_nesting/04_nesting_in_containers.rst index 8a8089376..bb9690ba1 100644 --- a/docs/source/examples/02_nesting/04_nesting_in_containers.rst +++ b/docs/source/examples/02_nesting/04_nesting_in_containers.rst @@ -17,7 +17,6 @@ parsing default values. import dataclasses - from typing import Dict, Tuple import tyro @@ -43,16 +42,16 @@ parsing default values. @dataclasses.dataclass class Args: # Example of specifying nested structures via a fixed-length tuple. - color_tuple: Tuple[RGB, HSL] + color_tuple: tuple[RGB, HSL] # Examples of nested structures in variable-length containers. These need a default # provided for length inference; we don't currently support specifying dynamic # container lengths directly from the commandline. - color_tuple_alt: Tuple[Color, ...] = ( + color_tuple_alt: tuple[Color, ...] = ( RGB(255, 0, 0), HSL(0, 255, 0), ) - color_map: Dict[str, RGB] = dataclasses.field( + color_map: dict[str, RGB] = dataclasses.field( # We can't use mutable values as defaults directly. default_factory={ "red": RGB(255, 0, 0), diff --git a/docs/source/examples/03_config_systems/01_base_configs.rst b/docs/source/examples/03_config_systems/01_base_configs.rst index 30f5f4b3f..3cb08479a 100644 --- a/docs/source/examples/03_config_systems/01_base_configs.rst +++ b/docs/source/examples/03_config_systems/01_base_configs.rst @@ -16,7 +16,7 @@ use the CLI to either override (existing) or fill in (missing) values. from dataclasses import dataclass - from typing import Callable, Literal, Tuple + from typing import Callable, Literal from torch import nn @@ -26,7 +26,7 @@ use the CLI to either override (existing) or fill in (missing) values. @dataclass(frozen=True) class AdamOptimizer: learning_rate: float = 1e-3 - betas: Tuple[float, float] = (0.9, 0.999) + betas: tuple[float, float] = (0.9, 0.999) @dataclass(frozen=True) diff --git a/docs/source/examples/04_additional/02_dictionaries.rst b/docs/source/examples/04_additional/02_dictionaries.rst index 8002b2504..7f15d7613 100644 --- a/docs/source/examples/04_additional/02_dictionaries.rst +++ b/docs/source/examples/04_additional/02_dictionaries.rst @@ -14,7 +14,7 @@ Dictionary inputs can be specified using either a standard ``Dict[K, V]`` annota :linenos: - from typing import Dict, Tuple, TypedDict + from typing import TypedDict from typing_extensions import NotRequired @@ -27,19 +27,19 @@ Dictionary inputs can be specified using either a standard ``Dict[K, V]`` annota total=False, ): learning_rate: float - betas: Tuple[float, float] + betas: tuple[float, float] class DictionarySchemaB(TypedDict): learning_rate: NotRequired[float] """NotRequired[] specifies that a particular key doesn't need to exist.""" - betas: Tuple[float, float] + betas: tuple[float, float] def main( typed_dict_a: DictionarySchemaA, typed_dict_b: DictionarySchemaB, - standard_dict: Dict[str, float] = { + standard_dict: dict[str, float] = { "learning_rate": 3e-4, "beta1": 0.9, "beta2": 0.999, diff --git a/docs/source/examples/04_additional/03_tuples.rst b/docs/source/examples/04_additional/03_tuples.rst index 2cfc03f8a..c355783a8 100644 --- a/docs/source/examples/04_additional/03_tuples.rst +++ b/docs/source/examples/04_additional/03_tuples.rst @@ -13,7 +13,7 @@ Example using ``tyro.cli()`` to instantiate tuple types. :linenos: - from typing import NamedTuple, Tuple + from typing import NamedTuple import tyro @@ -30,10 +30,10 @@ Example using ``tyro.cli()`` to instantiate tuple types. This should show up in the helptext!""" # Tuple types can contain raw values. - color: Tuple[int, int, int] = (255, 0, 0) + color: tuple[int, int, int] = (255, 0, 0) # Tuple types can contain nested structures. - two_colors: Tuple[Color, Color] = (Color(255, 0, 0), Color(0, 255, 0)) + two_colors: tuple[Color, Color] = (Color(255, 0, 0), Color(0, 255, 0)) if __name__ == "__main__": diff --git a/examples/01_basics/03_collections.py b/examples/01_basics/03_collections.py index ea4c0d108..c82dd7dd7 100644 --- a/examples/01_basics/03_collections.py +++ b/examples/01_basics/03_collections.py @@ -1,8 +1,8 @@ """Multi-value Arguments -Arguments of both fixed and variable lengths can be annotated with standard Python -collection types: `typing.List[T]`, `typing.Tuple[T1, T2]`, etc. In Python >=3.9, -`list[T]` and `tuple[T]` are also supported. +Arguments of both fixed and variable lengths can be annotated with standard +Python collection types. For Python 3.7 and 3.8, we can use either +`from __future__ import annotations` or`typing.List[T]`, `typing.Tuple[T1, T2]`, etc. Usage: `python ./03_collections.py --help` @@ -12,20 +12,19 @@ import dataclasses import pathlib -from typing import Tuple import tyro @dataclasses.dataclass(frozen=True) class TrainConfig: - # Example of a variable-length tuple. `typing.List`, `typing.Sequence`, - # `typing.Set`, `typing.Dict`, etc are all supported as well. - dataset_sources: Tuple[pathlib.Path, ...] + # Example of a variable-length tuple. `list[T]`, `set[T]`, + # `dict[K, V]`, etc are supported as well. + dataset_sources: tuple[pathlib.Path, ...] """Paths to load training data from. This can be multiple!""" # Fixed-length tuples are also okay. - image_dimensions: Tuple[int, int] = (32, 32) + image_dimensions: tuple[int, int] = (32, 32) """Height and width of some image data.""" diff --git a/examples/01_basics/05_flags.py b/examples/01_basics/05_flags.py index 5059ed8b5..11c07b8bf 100644 --- a/examples/01_basics/05_flags.py +++ b/examples/01_basics/05_flags.py @@ -13,7 +13,6 @@ """ import dataclasses -from typing import Optional import tyro @@ -24,7 +23,7 @@ class Args: boolean: bool # Optional boolean. Same as above, but can be omitted. - optional_boolean: Optional[bool] = None + optional_boolean: bool | None = None # Pass --flag-a in to set this value to True. flag_a: bool = False diff --git a/examples/01_basics/07_unions.py b/examples/01_basics/07_unions.py index a94c472b8..ca024cbbc 100644 --- a/examples/01_basics/07_unions.py +++ b/examples/01_basics/07_unions.py @@ -1,6 +1,6 @@ """Unions -`typing.Union[]` can be used to expand inputs to multiple types. +`X | Y` or `typing.Union[X, Y]` can be used to expand inputs to multiple types. Usage: `python ./07_unions.py --help` @@ -8,7 +8,7 @@ import dataclasses import enum -from typing import Literal, Optional, Tuple, Union +from typing import Literal, Optional import tyro @@ -22,19 +22,19 @@ class Color(enum.Enum): @dataclasses.dataclass(frozen=True) class Args: # Unions can be used to specify multiple allowable types. - union_over_types: Union[int, str] = 0 - string_or_enum: Union[Literal["red", "green"], Color] = "red" + union_over_types: int | str = 0 + string_or_enum: Literal["red", "green"] | Color = "red" # Unions also work over more complex nested types. - union_over_tuples: Union[Tuple[int, int], Tuple[str]] = ("1",) + union_over_tuples: tuple[int, int] | tuple[str] = ("1",) # And can be nested in other types. - tuple_of_string_or_enum: Tuple[Union[Literal["red", "green"], Color], ...] = ( + tuple_of_string_or_enum: tuple[Literal["red", "green"] | Color, ...] = ( "red", Color.RED, ) - # Optional[T] is equivalent to Union[T, None]. + # Optional[T] is equivalent to `T | None`. integer: Optional[Literal[0, 1, 2, 3]] = None diff --git a/examples/02_nesting/02_subcommands.py b/examples/02_nesting/02_subcommands.py index 7948b2ab1..5ccc26610 100644 --- a/examples/02_nesting/02_subcommands.py +++ b/examples/02_nesting/02_subcommands.py @@ -16,7 +16,6 @@ from __future__ import annotations import dataclasses -from typing import Union import tyro @@ -36,12 +35,12 @@ class Commit: all: bool = False -def main(cmd: Union[Checkout, Commit]) -> None: +def main(cmd: Checkout | Commit) -> None: print(cmd) if __name__ == "__main__": - # Note that we can also pass `Union[Checkout, Command]` directly into + # Note that we can also pass `Checkout | Command` directly into # `tyro.cli()`; this is understood by tyro and pyright, but unfortunately not by # mypy. tyro.cli(main) diff --git a/examples/02_nesting/03_multiple_subcommands.py b/examples/02_nesting/03_multiple_subcommands.py index 08536e014..4c2561c11 100644 --- a/examples/02_nesting/03_multiple_subcommands.py +++ b/examples/02_nesting/03_multiple_subcommands.py @@ -8,10 +8,11 @@ `python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --help` `python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 --dataset.binary` """ + from __future__ import annotations import dataclasses -from typing import Literal, Tuple, Union +from typing import Literal import tyro @@ -36,7 +37,7 @@ class ImageNet: @dataclasses.dataclass class Adam: learning_rate: float = 1e-3 - betas: Tuple[float, float] = (0.9, 0.999) + betas: tuple[float, float] = (0.9, 0.999) @dataclasses.dataclass @@ -49,8 +50,8 @@ class Sgd: @tyro.conf.configure(tyro.conf.ConsolidateSubcommandArgs) def train( - dataset: Union[Mnist, ImageNet] = Mnist(), - optimizer: Union[Adam, Sgd] = Adam(), + dataset: Mnist | ImageNet = Mnist(), + optimizer: Adam | Sgd = Adam(), ) -> None: """Example training script. diff --git a/examples/02_nesting/04_nesting_in_containers.py b/examples/02_nesting/04_nesting_in_containers.py index bd8e02694..18892696d 100644 --- a/examples/02_nesting/04_nesting_in_containers.py +++ b/examples/02_nesting/04_nesting_in_containers.py @@ -9,8 +9,8 @@ Usage: `python ./04_nesting_in_containers.py.py --help` """ + import dataclasses -from typing import Dict, Tuple import tyro @@ -36,16 +36,16 @@ class HSL(Color): @dataclasses.dataclass class Args: # Example of specifying nested structures via a fixed-length tuple. - color_tuple: Tuple[RGB, HSL] + color_tuple: tuple[RGB, HSL] # Examples of nested structures in variable-length containers. These need a default # provided for length inference; we don't currently support specifying dynamic # container lengths directly from the commandline. - color_tuple_alt: Tuple[Color, ...] = ( + color_tuple_alt: tuple[Color, ...] = ( RGB(255, 0, 0), HSL(0, 255, 0), ) - color_map: Dict[str, RGB] = dataclasses.field( + color_map: dict[str, RGB] = dataclasses.field( # We can't use mutable values as defaults directly. default_factory={ "red": RGB(255, 0, 0), diff --git a/examples/03_config_systems/01_base_configs.py b/examples/03_config_systems/01_base_configs.py index f62559304..0d2a59e20 100644 --- a/examples/03_config_systems/01_base_configs.py +++ b/examples/03_config_systems/01_base_configs.py @@ -14,7 +14,7 @@ """ from dataclasses import dataclass -from typing import Callable, Literal, Tuple +from typing import Callable, Literal from torch import nn @@ -24,7 +24,7 @@ @dataclass(frozen=True) class AdamOptimizer: learning_rate: float = 1e-3 - betas: Tuple[float, float] = (0.9, 0.999) + betas: tuple[float, float] = (0.9, 0.999) @dataclass(frozen=True) diff --git a/examples/04_additional/02_dictionaries.py b/examples/04_additional/02_dictionaries.py index 2dbfce9f0..1504c3b42 100644 --- a/examples/04_additional/02_dictionaries.py +++ b/examples/04_additional/02_dictionaries.py @@ -10,7 +10,7 @@ `python ./02_dictionaries.py --typed-dict-b.betas 0.9 0.999` """ -from typing import Dict, Tuple, TypedDict +from typing import TypedDict from typing_extensions import NotRequired @@ -23,19 +23,19 @@ class DictionarySchemaA( total=False, ): learning_rate: float - betas: Tuple[float, float] + betas: tuple[float, float] class DictionarySchemaB(TypedDict): learning_rate: NotRequired[float] """NotRequired[] specifies that a particular key doesn't need to exist.""" - betas: Tuple[float, float] + betas: tuple[float, float] def main( typed_dict_a: DictionarySchemaA, typed_dict_b: DictionarySchemaB, - standard_dict: Dict[str, float] = { + standard_dict: dict[str, float] = { "learning_rate": 3e-4, "beta1": 0.9, "beta2": 0.999, diff --git a/examples/04_additional/03_tuples.py b/examples/04_additional/03_tuples.py index 0d48453b1..5c0865785 100644 --- a/examples/04_additional/03_tuples.py +++ b/examples/04_additional/03_tuples.py @@ -8,7 +8,7 @@ `python ./03_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0` """ -from typing import NamedTuple, Tuple +from typing import NamedTuple import tyro @@ -25,10 +25,10 @@ class TupleType(NamedTuple): This should show up in the helptext!""" # Tuple types can contain raw values. - color: Tuple[int, int, int] = (255, 0, 0) + color: tuple[int, int, int] = (255, 0, 0) # Tuple types can contain nested structures. - two_colors: Tuple[Color, Color] = (Color(255, 0, 0), Color(0, 255, 0)) + two_colors: tuple[Color, Color] = (Color(255, 0, 0), Color(0, 255, 0)) if __name__ == "__main__": diff --git a/examples/04_additional/08_pydantic.py b/examples/04_additional/08_pydantic.py index 5af836d90..92407a7a4 100644 --- a/examples/04_additional/08_pydantic.py +++ b/examples/04_additional/08_pydantic.py @@ -8,6 +8,7 @@ `python ./08_pydantic.py --field1 hello` `python ./08_pydantic.py --field1 hello --field2 5` """ + from pydantic import BaseModel, Field import tyro diff --git a/pyproject.toml b/pyproject.toml index 73b90fbe7..1c9449584 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,8 @@ dependencies = [ "backports.cached-property>=1.0.2; python_version<'3.8'", "colorama>=0.4.0; platform_system=='Windows'", "rich>=11.1.0", - "shtab>=1.5.6" + "shtab>=1.5.6", + "eval_type_backport>=0.1.3; python_version<'3.10'", ] [project.optional-dependencies] @@ -48,7 +49,8 @@ dev = [ # old version. But doing so breaks other Python versions. "flax>=0.6.9;python_version>='3.8'", "pydantic>=2.5.2", - "coverage[toml]>=6.5.0" + "coverage[toml]>=6.5.0", + "eval_type_backport>=0.1.3", ] [project.urls] @@ -101,7 +103,7 @@ exclude_lines = [ [tool.ruff] src = ["src"] # Needed to recognize first-party import location in GitHub action. -select = [ +lint.select = [ "E", # pycodestyle errors. "F", # Pyflakes rules. "PLC", # Pylint convention warnings. @@ -110,7 +112,7 @@ select = [ "PLW", # Pylint warnings. "I" # Import sorting. ] -ignore = [ +lint.ignore = [ "E741", # Ambiguous variable name. (l, O, or I) "E501", # Line too long. "PLR2004", # Magic value used in comparison. diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index eec5c2158..7bf2bfa60 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -881,7 +881,7 @@ def add_argument(self, action): # pragma: no cover if self._action_max_length > self._max_help_position + 2: self._action_max_length = prev_max_length - def _split_lines(self, text, width): + def _split_lines(self, text, width): # pragma: no cover text = self._whitespace_matcher.sub(" ", text).strip() # The textwrap module is used only for formatting help. # Delay its import for speeding up the common usage of argparse. diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index ffd36add7..de7c94b5c 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -1,5 +1,6 @@ """Rules for taking high-level field definitions and lowering them into inputs for argparse's `add_argument()`.""" + from __future__ import annotations import argparse diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index bfa5dc75f..a3be76a7d 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -1,4 +1,5 @@ """Core public API.""" + import argparse import dataclasses import pathlib @@ -51,8 +52,7 @@ def cli( default: Optional[OutT] = None, return_unknown_args: Literal[False] = False, use_underscores: bool = False, -) -> OutT: - ... +) -> OutT: ... @overload @@ -65,8 +65,7 @@ def cli( default: Optional[OutT] = None, return_unknown_args: Literal[True], use_underscores: bool = False, -) -> Tuple[OutT, List[str]]: - ... +) -> Tuple[OutT, List[str]]: ... @overload @@ -82,8 +81,7 @@ def cli( default: None = None, return_unknown_args: Literal[False] = False, use_underscores: bool = False, -) -> OutT: - ... +) -> OutT: ... @overload @@ -99,8 +97,7 @@ def cli( default: None = None, return_unknown_args: Literal[True], use_underscores: bool = False, -) -> Tuple[OutT, List[str]]: - ... +) -> Tuple[OutT, List[str]]: ... def cli( @@ -213,8 +210,7 @@ def get_parser( description: Optional[str] = None, default: Optional[OutT] = None, use_underscores: bool = False, -) -> argparse.ArgumentParser: - ... +) -> argparse.ArgumentParser: ... @overload @@ -225,8 +221,7 @@ def get_parser( description: Optional[str] = None, default: Optional[OutT] = None, use_underscores: bool = False, -) -> argparse.ArgumentParser: - ... +) -> argparse.ArgumentParser: ... def get_parser( diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index cb300fe01..7ff01bb9b 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -1,5 +1,6 @@ """Abstractions for pulling out 'field' definitions, which specify inputs, types, and # type: ignore defaults, from general callables.""" + from __future__ import annotations import collections @@ -471,7 +472,9 @@ def _field_list_from_typeddict( total = getattr(cls, "__total__", True) assert isinstance(total, bool) assert not valid_default_instance or isinstance(default_instance, dict) - for name, typ in _resolver.get_type_hints(cls, include_extras=True).items(): + for name, typ in _resolver.get_type_hints_with_backported_syntax( + cls, include_extras=True + ).items(): typ_origin = get_origin(typ) is_default_from_default_instance = False if valid_default_instance and name in cast(dict, default_instance): @@ -531,7 +534,9 @@ def _field_list_from_namedtuple( field_defaults = getattr(cls, "_field_defaults") # Note that _field_types is removed in Python 3.9. - for name, typ in _resolver.get_type_hints(cls, include_extras=True).items(): + for name, typ in _resolver.get_type_hints_with_backported_syntax( + cls, include_extras=True + ).items(): # Get default, with priority for `default_instance`. default = field_defaults.get(name, MISSING_NONPROP) if hasattr(default_instance, name): @@ -608,12 +613,16 @@ def _field_list_from_dataclass( except ImportError: if not TYPE_CHECKING: pydantic = None # type: ignore + else: + import pydantic try: from pydantic import v1 as pydantic_v1 except ImportError: if not TYPE_CHECKING: pydantic_v1 = None # type: ignore + else: + from pydantic import v1 as pydantic_v1 def _is_pydantic(cls: TypeForm[Any]) -> bool: @@ -961,7 +970,7 @@ def _field_list_from_params( # This will throw a type error for torch.device, typing.Dict, etc. try: - hints = _resolver.get_type_hints(f, include_extras=True) + hints = _resolver.get_type_hints_with_backported_syntax(f, include_extras=True) except TypeError: return UnsupportedNestedTypeMessage(f"Could not get hints for {f}!") diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index 0a7322f3e..7eb9513aa 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -30,6 +30,7 @@ ) ``` """ + import collections.abc import dataclasses import enum @@ -121,7 +122,7 @@ def is_type_string_converter(typ: Union[Callable, TypeForm[Any]]) -> bool: # One day this should be fixed with `__text_signature__`. return True - type_annotations = _resolver.get_type_hints_with_nicer_errors(typ) + type_annotations = _resolver.get_type_hints_with_backported_syntax(typ) # Some checks we can do if the signature is available! for i, param in enumerate(signature.parameters.values()): annotation = type_annotations.get(param.name, param.annotation) @@ -269,8 +270,7 @@ def _instantiator_from_type_inner( type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal["fixed_length"], markers: FrozenSet[_markers.Marker], -) -> Tuple[Instantiator, InstantiatorMetadata]: - ... +) -> Tuple[Instantiator, InstantiatorMetadata]: ... @overload @@ -279,8 +279,7 @@ def _instantiator_from_type_inner( type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal[False], markers: FrozenSet[_markers.Marker], -) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: - ... +) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: ... @overload @@ -289,8 +288,7 @@ def _instantiator_from_type_inner( type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal[True], markers: FrozenSet[_markers.Marker], -) -> Tuple[Instantiator, InstantiatorMetadata]: - ... +) -> Tuple[Instantiator, InstantiatorMetadata]: ... def _instantiator_from_type_inner( diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index af929259c..523c9cf28 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -150,13 +150,13 @@ def from_callable_or_type( # Helptext for this field; used as description for grouping arguments. class_field_name = _strings.make_field_name([field.intern_name]) if field.helptext is not None: - helptext_from_intern_prefixed_field_name[ - class_field_name - ] = field.helptext + helptext_from_intern_prefixed_field_name[class_field_name] = ( + field.helptext + ) else: - helptext_from_intern_prefixed_field_name[ - class_field_name - ] = _docstrings.get_callable_description(nested_parser.f) + helptext_from_intern_prefixed_field_name[class_field_name] = ( + _docstrings.get_callable_description(nested_parser.f) + ) # If arguments are in an optional group, it indicates that the default_instance # will be used if none of the arguments are passed in. diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 959dd70cb..ae8759f7b 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -1,4 +1,5 @@ """Utilities for resolving types and forward references.""" + import collections.abc import copy import dataclasses @@ -21,7 +22,14 @@ cast, ) -from typing_extensions import Annotated, Self, get_args, get_origin, get_type_hints +from typing_extensions import ( + Annotated, + ForwardRef, + Self, + get_args, + get_origin, + get_type_hints, +) from . import _fields, _unsafe_cache from ._typing import TypeForm @@ -111,7 +119,9 @@ def resolved_fields(cls: TypeForm) -> List[dataclasses.Field]: assert dataclasses.is_dataclass(cls) fields = [] - annotations = get_type_hints(cast(Callable, cls), include_extras=True) + annotations = get_type_hints_with_backported_syntax( + cast(Callable, cls), include_extras=True + ) for field in getattr(cls, "__dataclass_fields__").values(): # Avoid mutating original field. field = copy.copy(field) @@ -349,32 +359,31 @@ def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCalla return typ -def get_type_hints_with_nicer_errors( +def get_type_hints_with_backported_syntax( obj: Callable[..., Any], include_extras: bool = False ) -> Dict[str, Any]: + """Same as `typing.get_type_hints()`, but supports new union syntax (X | Y) + and generics (list[str]) in older versions of Python.""" try: return get_type_hints(obj, include_extras=include_extras) except TypeError as e: # pragma: no cover - common_message = f"\n-----\n\nError calling typing.get_type_hints() on {obj}! " - message = e.args[0] - if message.startswith( - "unsupported operand type(s) for |" - ) and sys.version_info < (3, 10): - # PEP 604. Requires Python 3.10. - raise TypeError( - e.args[0] - + common_message - + "You may be using a Union in the form of `X | Y`; to support Python versions that lower than 3.10, you need to use `typing.Union[X, Y]` instead of `X | Y` and `typing.Optional[X]` instead of `X | None`.", - *e.args[1:], - ) - elif message == "'type' object is not subscriptable" and ( - sys.version_info < (3, 9) - ): - # PEP 585. Requires Python 3.9. - raise TypeError( - e.args[0] - + common_message - + "You may be using a standard collection as a generic, like `list[T]` or `tuple[T1, T2]`. To support Python versions lower than 3.9, you need to using `typing.List[T]` and `typing.Tuple[T1, T2]`.", - *e.args[1:], - ) + # Resolve new type syntax using eval_type_backport. + if hasattr(obj, "__annotations__"): + try: + from eval_type_backport import eval_type_backport + + # Get global namespace for functions. + globalns = getattr(obj, "__globals__", None) + + # Get global namespace for classes. + if globalns is None and hasattr(globalns, "__init__"): + globalns = getattr(getattr(obj, "__init__"), "__globals__", None) + + out = { + k: eval_type_backport(ForwardRef(v), globalns=globalns, localns={}) + for k, v in getattr(obj, "__annotations__").items() + } + return out + except ImportError: + pass raise e diff --git a/src/tyro/conf/_confstruct.py b/src/tyro/conf/_confstruct.py index 9e5d85b67..c7c27c295 100644 --- a/src/tyro/conf/_confstruct.py +++ b/src/tyro/conf/_confstruct.py @@ -25,8 +25,7 @@ def subcommand( prefix_name: bool = True, constructor: None = None, constructor_factory: Optional[Callable[[], Union[Type, Callable]]] = None, -) -> Any: - ... +) -> Any: ... @overload @@ -38,8 +37,7 @@ def subcommand( prefix_name: bool = True, constructor: Optional[Union[Type, Callable]] = None, constructor_factory: None = None, -) -> Any: - ... +) -> Any: ... def subcommand( @@ -132,8 +130,7 @@ def arg( prefix_name: Optional[bool] = None, constructor: None = None, constructor_factory: Optional[Callable[[], Union[Type, Callable]]] = None, -) -> Any: - ... +) -> Any: ... @overload @@ -146,8 +143,7 @@ def arg( prefix_name: Optional[bool] = None, constructor: Optional[Union[Type, Callable]] = None, constructor_factory: None = None, -) -> Any: - ... +) -> Any: ... def arg( diff --git a/src/tyro/extras/__init__.py b/src/tyro/extras/__init__.py index b4dda741b..567ddaf31 100644 --- a/src/tyro/extras/__init__.py +++ b/src/tyro/extras/__init__.py @@ -1,6 +1,6 @@ """The :mod:`tyro.extras` submodule contains helpers that complement :func:`tyro.cli()`. -Compared to the core interface, APIs here are more likely to be changed or deprecated. """ +Compared to the core interface, APIs here are more likely to be changed or deprecated.""" from .._argparse_formatter import set_accent_color as set_accent_color from .._cli import get_parser as get_parser diff --git a/src/tyro/extras/_serialization.py b/src/tyro/extras/_serialization.py index 2e2adc9c6..6ae4771e1 100644 --- a/src/tyro/extras/_serialization.py +++ b/src/tyro/extras/_serialization.py @@ -1,4 +1,5 @@ """Type-safe, human-readable serialization helpers for dataclasses.""" + from __future__ import annotations import dataclasses diff --git a/src/tyro/extras/_subcommand_cli_from_dict.py b/src/tyro/extras/_subcommand_cli_from_dict.py index 0e1cd56cd..b2f769630 100644 --- a/src/tyro/extras/_subcommand_cli_from_dict.py +++ b/src/tyro/extras/_subcommand_cli_from_dict.py @@ -16,8 +16,7 @@ def subcommand_cli_from_dict( description: Optional[str] = None, args: Optional[Sequence[str]] = None, use_underscores: bool = False, -) -> T: - ... +) -> T: ... # TODO: hack. We prefer the above signature, which Pyright understands, but as of 1.6.1 @@ -30,8 +29,7 @@ def subcommand_cli_from_dict( description: Optional[str] = None, args: Optional[Sequence[str]] = None, use_underscores: bool = False, -) -> Any: - ... +) -> Any: ... def subcommand_cli_from_dict( diff --git a/tests/test_errors_new_annotations.py b/tests/test_errors_new_annotations.py deleted file mode 100644 index 75b69f58f..000000000 --- a/tests/test_errors_new_annotations.py +++ /dev/null @@ -1,35 +0,0 @@ -from __future__ import annotations - -import sys - -import pytest - -import tyro - - -@pytest.mark.skipif( - sys.version_info >= (3, 10), reason="No error for newer versions of Python." -) -def test_new_union_error() -> None: - """PEP 604 allows `|` to be used as a type annotation in Python >=3.10.""" - - def main(x: int | str) -> None: - ... - - with pytest.raises(TypeError) as e: - tyro.cli(main) - assert "You may be using a Union in the form of `X | Y`" in e.value.args[0] - - -@pytest.mark.skipif( - sys.version_info >= (3, 9), reason="No error for newer versions of Python." -) -def test_new_collection_error() -> None: - """PEP 585 allows standard collections to be used as generics in Python >=3.9.""" - - def main(x: list[int]) -> None: - ... - - with pytest.raises(TypeError) as e: - tyro.cli(main) - assert "You may be using a standard collection as a generic" in e.value.args[0] diff --git a/tests/test_flax_min_py38.py b/tests/test_flax_min_py38.py index 6f944693a..056f70949 100644 --- a/tests/test_flax_min_py38.py +++ b/tests/test_flax_min_py38.py @@ -1,4 +1,5 @@ """Tests initializing flax modules directly via tyro.""" + from typing import cast import jax diff --git a/tests/test_missing.py b/tests/test_missing.py index 1554bc043..223eb2560 100644 --- a/tests/test_missing.py +++ b/tests/test_missing.py @@ -1,4 +1,5 @@ """Tests for tyro.MISSING.""" + import contextlib import dataclasses import io diff --git a/tests/test_mixed_unions.py b/tests/test_mixed_unions.py index 2da3329f7..d6444f9bd 100644 --- a/tests/test_mixed_unions.py +++ b/tests/test_mixed_unions.py @@ -5,7 +5,6 @@ handling gets more complicated but should still be supported! """ - import dataclasses from typing import Any, Dict, List, Tuple, Union diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index 5945752ec..cfd7211a2 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -185,9 +185,10 @@ class B: @dataclasses.dataclass class Nested2: - subcommand: Annotated[ - A, tyro.conf.subcommand("command-a", default=A(7)) - ] | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] + subcommand: ( + Annotated[A, tyro.conf.subcommand("command-a", default=A(7))] + | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] + ) @dataclasses.dataclass class Nested1: @@ -236,9 +237,10 @@ class B: @dataclasses.dataclass class Nested2: - subcommand: Annotated[ - A, tyro.conf.subcommand("command-a", default=A(7)) - ] | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] + subcommand: ( + Annotated[A, tyro.conf.subcommand("command-a", default=A(7))] + | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] + ) @dataclasses.dataclass class Nested1: @@ -343,9 +345,10 @@ class B: @dataclasses.dataclass class Nested2(Generic[T]): - subcommand: Annotated[ - T, tyro.conf.subcommand("command-a", default=A(7)) - ] | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] + subcommand: ( + Annotated[T, tyro.conf.subcommand("command-a", default=A(7))] + | Annotated[B, tyro.conf.subcommand("command-b", default=B(9))] + ) @dataclasses.dataclass class Nested1: @@ -397,11 +400,11 @@ class B: @dataclasses.dataclass class Nested: - subcommand: Annotated[ - B, tyro.conf.subcommand("one", default=default_one) - ] | Annotated[B, tyro.conf.subcommand("two")] | Annotated[ - B, tyro.conf.subcommand("three", default=default_three) - ] + subcommand: ( + Annotated[B, tyro.conf.subcommand("one", default=default_one)] + | Annotated[B, tyro.conf.subcommand("two")] + | Annotated[B, tyro.conf.subcommand("three", default=default_three)] + ) # Match by hash. def main_one(x: Nested = Nested(default_one)) -> None: diff --git a/tests/test_py311_generated/test_errors_new_annotations_generated.py b/tests/test_py311_generated/test_errors_new_annotations_generated.py index 75b69f58f..da4aee733 100644 --- a/tests/test_py311_generated/test_errors_new_annotations_generated.py +++ b/tests/test_py311_generated/test_errors_new_annotations_generated.py @@ -13,8 +13,7 @@ def test_new_union_error() -> None: """PEP 604 allows `|` to be used as a type annotation in Python >=3.10.""" - def main(x: int | str) -> None: - ... + def main(x: int | str) -> None: ... with pytest.raises(TypeError) as e: tyro.cli(main) @@ -27,8 +26,7 @@ def main(x: int | str) -> None: def test_new_collection_error() -> None: """PEP 585 allows standard collections to be used as generics in Python >=3.9.""" - def main(x: list[int]) -> None: - ... + def main(x: list[int]) -> None: ... with pytest.raises(TypeError) as e: tyro.cli(main) diff --git a/tests/test_py311_generated/test_flax_min_py38_generated.py b/tests/test_py311_generated/test_flax_min_py38_generated.py index 6f944693a..056f70949 100644 --- a/tests/test_py311_generated/test_flax_min_py38_generated.py +++ b/tests/test_py311_generated/test_flax_min_py38_generated.py @@ -1,4 +1,5 @@ """Tests initializing flax modules directly via tyro.""" + from typing import cast import jax diff --git a/tests/test_py311_generated/test_missing_generated.py b/tests/test_py311_generated/test_missing_generated.py index 1554bc043..223eb2560 100644 --- a/tests/test_py311_generated/test_missing_generated.py +++ b/tests/test_py311_generated/test_missing_generated.py @@ -1,4 +1,5 @@ """Tests for tyro.MISSING.""" + import contextlib import dataclasses import io diff --git a/tests/test_py311_generated/test_mixed_unions_generated.py b/tests/test_py311_generated/test_mixed_unions_generated.py index 1b10b2945..c361de491 100644 --- a/tests/test_py311_generated/test_mixed_unions_generated.py +++ b/tests/test_py311_generated/test_mixed_unions_generated.py @@ -5,7 +5,6 @@ handling gets more complicated but should still be supported! """ - import dataclasses from typing import Any, Dict, List, Tuple diff --git a/tests/test_py311_generated/test_pydantic_with_newtype_generated.py b/tests/test_py311_generated/test_pydantic_with_newtype_generated.py index 1ac21745c..44ecb7300 100644 --- a/tests/test_py311_generated/test_pydantic_with_newtype_generated.py +++ b/tests/test_py311_generated/test_pydantic_with_newtype_generated.py @@ -10,9 +10,9 @@ class Measurements(BaseModel): single: Microliter = pydantic.Field(10) - renamed_single: Annotated[ - Microliter, tyro.conf.arg(name="other_single") - ] = pydantic.Field(10) + renamed_single: Annotated[Microliter, tyro.conf.arg(name="other_single")] = ( + pydantic.Field(10) + ) pair: Tuple[Microliter, Microliter] = pydantic.Field((20, 30)) diff --git a/tests/test_py311_generated/test_self_type_generated.py b/tests/test_py311_generated/test_self_type_generated.py index 45058b305..eaffa0314 100644 --- a/tests/test_py311_generated/test_self_type_generated.py +++ b/tests/test_py311_generated/test_self_type_generated.py @@ -27,8 +27,7 @@ def method2(cls, x: Self) -> TestClass: # return x -class TestSubclass(TestClass): - ... +class TestSubclass(TestClass): ... def test_method() -> None: diff --git a/tests/test_py311_generated/test_unsupported_but_should_work_generated.py b/tests/test_py311_generated/test_unsupported_but_should_work_generated.py index 7341d3251..74327ac80 100644 --- a/tests/test_py311_generated/test_unsupported_but_should_work_generated.py +++ b/tests/test_py311_generated/test_unsupported_but_should_work_generated.py @@ -3,6 +3,7 @@ Includes things like omegaconf.MISSING, attrs, etc, which mostly work but either likely have corner cases or just seem sketchy. """ + from typing import Tuple import omegaconf diff --git a/tests/test_pydantic_with_newtype.py b/tests/test_pydantic_with_newtype.py index 9ec0252fe..2b4d71d30 100644 --- a/tests/test_pydantic_with_newtype.py +++ b/tests/test_pydantic_with_newtype.py @@ -11,9 +11,9 @@ class Measurements(BaseModel): single: Microliter = pydantic.Field(10) - renamed_single: Annotated[ - Microliter, tyro.conf.arg(name="other_single") - ] = pydantic.Field(10) + renamed_single: Annotated[Microliter, tyro.conf.arg(name="other_single")] = ( + pydantic.Field(10) + ) pair: Tuple[Microliter, Microliter] = pydantic.Field((20, 30)) diff --git a/tests/test_self_type.py b/tests/test_self_type.py index 0b9ffbfc7..e1fd2270c 100644 --- a/tests/test_self_type.py +++ b/tests/test_self_type.py @@ -6,7 +6,7 @@ import tyro -class TestClass: +class SomeClass: def __init__(self, a: int, b: int) -> None: self.a = a self.b = b @@ -15,7 +15,7 @@ def method1(self, x: Self) -> None: self.effect = x @classmethod - def method2(cls, x: Self) -> TestClass: + def method2(cls, x: Self) -> SomeClass: return x # Self is not valid in static methods. @@ -26,66 +26,65 @@ def method2(cls, x: Self) -> TestClass: # return x -class TestSubclass(TestClass): - ... +class SomeSubclass(SomeClass): ... def test_method() -> None: - x = TestClass(0, 0) + x = SomeClass(0, 0) with pytest.raises(SystemExit): tyro.cli(x.method1, args=[]) assert tyro.cli(x.method1, args="--x.a 3 --x.b 3".split(" ")) is None assert x.effect.a == 3 and x.effect.b == 3 - assert isinstance(x, TestClass) + assert isinstance(x, SomeClass) def test_classmethod() -> None: - x = TestClass(0, 0) + x = SomeClass(0, 0) with pytest.raises(SystemExit): tyro.cli(x.method2, args=[]) with pytest.raises(SystemExit): - tyro.cli(TestClass.method2, args=[]) + tyro.cli(SomeClass.method2, args=[]) y = tyro.cli(x.method2, args="--x.a 3 --x.b 3".split(" ")) assert y.a == 3 assert y.b == 3 - assert isinstance(y, TestClass) + assert isinstance(y, SomeClass) - y = tyro.cli(TestClass.method2, args="--x.a 3 --x.b 3".split(" ")) + y = tyro.cli(SomeClass.method2, args="--x.a 3 --x.b 3".split(" ")) assert y.a == 3 assert y.b == 3 - assert isinstance(y, TestClass) + assert isinstance(y, SomeClass) def test_subclass_method() -> None: - x = TestSubclass(0, 0) + x = SomeSubclass(0, 0) with pytest.raises(SystemExit): tyro.cli(x.method1, args=[]) assert tyro.cli(x.method1, args="--x.a 3 --x.b 3".split(" ")) is None assert x.effect.a == 3 and x.effect.b == 3 - assert isinstance(x, TestSubclass) + assert isinstance(x, SomeSubclass) y = tyro.cli(x.method2, args="--x.a 3 --x.b 3".split(" ")) assert y.a == 3 assert y.b == 3 - assert isinstance(y, TestClass) + assert isinstance(y, SomeClass) def test_subclass_classmethod() -> None: - x = TestSubclass(0, 0) + x = SomeSubclass(0, 0) with pytest.raises(SystemExit): tyro.cli(x.method2, args=[]) with pytest.raises(SystemExit): - tyro.cli(TestSubclass.method2, args=[]) + tyro.cli(SomeSubclass.method2, args=[]) y = tyro.cli(x.method2, args="--x.a 3 --x.b 3".split(" ")) assert y.a == 3 assert y.b == 3 - assert isinstance(y, TestClass) + assert isinstance(y, SomeClass) - y = tyro.cli(TestSubclass.method2, args="--x.a 3 --x.b 3".split(" ")) + y = tyro.cli(SomeSubclass.method2, args="--x.a 3 --x.b 3".split(" ")) assert y.a == 3 assert y.b == 3 - assert isinstance(y, TestClass) + assert isinstance(y, SomeClass) diff --git a/tests/test_unsupported_but_should_work.py b/tests/test_unsupported_but_should_work.py index 7341d3251..74327ac80 100644 --- a/tests/test_unsupported_but_should_work.py +++ b/tests/test_unsupported_but_should_work.py @@ -3,6 +3,7 @@ Includes things like omegaconf.MISSING, attrs, etc, which mostly work but either likely have corner cases or just seem sketchy. """ + from typing import Tuple import omegaconf From 42d694478fd59b7e32add7f23b6b34b2d8ae5dd7 Mon Sep 17 00:00:00 2001 From: Paul Wais Date: Wed, 3 Apr 2024 17:13:15 -0700 Subject: [PATCH 387/491] Make _fields.py assertion much more helpful (#131) Signed-off-by: Paul Wais Co-authored-by: Brent Yi --- src/tyro/_fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 7ff01bb9b..a9d3d9fc5 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -741,7 +741,7 @@ def _field_list_from_attrs( elif isinstance(default, attr.Factory): # type: ignore default = default.factory() # type: ignore - assert attr_field.type is not None + assert attr_field.type is not None, attr_field field_list.append( FieldDefinition.make( name=name, From b43c668c5b811baf73c03a53c41140bf18dde97f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 4 Apr 2024 01:08:39 -0700 Subject: [PATCH 388/491] Implement counter actions (#130) * Implement counter actions * ruff * Add counter test, sync docs * Special-case for Python 3.7 * Formatting --- .../examples/04_additional/12_counters.rst | 69 +++++++++++++++++++ examples/04_additional/12_counters.py | 33 +++++++++ src/tyro/_arguments.py | 30 +++++++- src/tyro/conf/__init__.py | 1 + src/tyro/conf/_markers.py | 4 ++ tests/test_conf.py | 24 +++++++ 6 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 docs/source/examples/04_additional/12_counters.rst create mode 100644 examples/04_additional/12_counters.py diff --git a/docs/source/examples/04_additional/12_counters.rst b/docs/source/examples/04_additional/12_counters.rst new file mode 100644 index 000000000..d0bedf571 --- /dev/null +++ b/docs/source/examples/04_additional/12_counters.rst @@ -0,0 +1,69 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Counters +========================================== + + +Repeatable 'counter' arguments can be specified via :data:`tyro.conf.UseCounterAction`. + + + +.. code-block:: python + :linenos: + + + from typing_extensions import Annotated + + import tyro + from tyro.conf import UseCounterAction + + + def main( + verbosity: UseCounterAction[int], + aliased_verbosity: Annotated[UseCounterAction[int], tyro.conf.arg(aliases=["-v"])], + ) -> None: + """Example showing how to use counter actions. + + Args: + verbosity: Verbosity level. + aliased_verbosity: Same as above, but can also be specified with -v, -vv, -vvv, etc. + """ + print("Verbosity level:", verbosity) + print("Verbosity level (aliased):", aliased_verbosity) + + + if __name__ == "__main__": + tyro.cli(main) + +------------ + +.. raw:: html + + python 04_additional/12_counters.py --help + +.. program-output:: python ../../examples/04_additional/12_counters.py --help + +------------ + +.. raw:: html + + python 04_additional/12_counters.py --verbosity + +.. program-output:: python ../../examples/04_additional/12_counters.py --verbosity + +------------ + +.. raw:: html + + python 04_additional/12_counters.py --verbosity --verbosity + +.. program-output:: python ../../examples/04_additional/12_counters.py --verbosity --verbosity + +------------ + +.. raw:: html + + python 04_additional/12_counters.py -vvv + +.. program-output:: python ../../examples/04_additional/12_counters.py -vvv diff --git a/examples/04_additional/12_counters.py b/examples/04_additional/12_counters.py new file mode 100644 index 000000000..2a6dd2bbf --- /dev/null +++ b/examples/04_additional/12_counters.py @@ -0,0 +1,33 @@ +"""Counters + +Repeatable 'counter' arguments can be specified via :data:`tyro.conf.UseCounterAction`. + +Usage: +`python ./12_counters.py --help` +`python ./12_counters.py --verbosity` +`python ./12_counters.py --verbosity --verbosity` +`python ./12_counters.py -vvv` +""" + +from typing_extensions import Annotated + +import tyro +from tyro.conf import UseCounterAction + + +def main( + verbosity: UseCounterAction[int], + aliased_verbosity: Annotated[UseCounterAction[int], tyro.conf.arg(aliases=["-v"])], +) -> None: + """Example showing how to use counter actions. + + Args: + verbosity: Verbosity level. + aliased_verbosity: Same as above, but can also be specified with -v, -vv, -vvv, etc. + """ + print("Verbosity level:", verbosity) + print("Verbosity level (aliased):", aliased_verbosity) + + +if __name__ == "__main__": + tyro.cli(main) diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index de7c94b5c..5c90a0ac9 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -132,9 +132,9 @@ def add_argument( # directly be used. This helps reduce the likelihood of issues with converting # the field default to a string format, then back to the desired type. action = kwargs.get("action", None) - if action != "append": + if action not in {"append", "count"}: kwargs["default"] = _fields.MISSING_NONPROP - elif action == BooleanOptionalAction: + elif action in {BooleanOptionalAction, "count"}: pass else: kwargs["default"] = [] @@ -193,6 +193,7 @@ def lowered(self) -> LoweredArgumentDefinition: _rule_handle_boolean_flags, _rule_recursive_instantiator_from_type, _rule_convert_defaults_to_strings, + _rule_counters, _rule_generate_helptext, _rule_set_name_or_flag_and_dest, _rule_positional_special_handling, @@ -405,6 +406,28 @@ def _rich_tag_if_enabled(x: str, tag: str) -> str: return x if not USE_RICH else f"[{tag}]{x}[/{tag}]" +def _rule_counters( + arg: ArgumentDefinition, + lowered: LoweredArgumentDefinition, +) -> LoweredArgumentDefinition: + """Handle counters, like -vvv for level-3 verbosity.""" + if ( + _markers.UseCounterAction in arg.field.markers + and arg.field.type_or_callable is int + and not arg.field.is_positional() + ): + return dataclasses.replace( + lowered, + metavar=None, + nargs=None, + action="count", + default=0, + required=False, + instantiator=lambda x: x, # argparse will directly give us an int! + ) + return lowered + + def _rule_generate_helptext( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, @@ -465,6 +488,9 @@ def _rule_generate_helptext( # Intentionally not quoted via shlex, since this can't actually be passed # in via the commandline. default_text = f"(fixed to: {default_label})" + elif lowered.action == "count": + # Repeatable argument. + default_text = "(repeatable)" elif lowered.action == "append" and ( default in _fields.MISSING_SINGLETONS or len(cast(tuple, default)) == 0 ): diff --git a/src/tyro/conf/__init__.py b/src/tyro/conf/__init__.py index ef9698523..f58407773 100644 --- a/src/tyro/conf/__init__.py +++ b/src/tyro/conf/__init__.py @@ -21,4 +21,5 @@ from ._markers import Suppress as Suppress from ._markers import SuppressFixed as SuppressFixed from ._markers import UseAppendAction as UseAppendAction +from ._markers import UseCounterAction as UseCounterAction from ._markers import configure as configure diff --git a/src/tyro/conf/_markers.py b/src/tyro/conf/_markers.py index 6d04a38e7..babfc810e 100644 --- a/src/tyro/conf/_markers.py +++ b/src/tyro/conf/_markers.py @@ -123,6 +123,10 @@ `Tuple[T, ...]`, etc), including dictionaries without default values. """ +UseCounterAction = Annotated[T, None] +"""Use "counter" actions for integer arguments. Example usage: `verbose: UseCounterAction[int]`.""" + + CallableType = TypeVar("CallableType", bound=Callable) # Dynamically generate marker singletons. diff --git a/tests/test_conf.py b/tests/test_conf.py index 5321b0551..764b01d62 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -4,6 +4,7 @@ import io import json as json_ import shlex +import sys from typing import Any, Dict, Generic, List, Tuple, Type, TypeVar, Union import pytest @@ -1360,3 +1361,26 @@ class DatasetConfig: helptext = target.getvalue() assert "OptimizerConfig options" in helptext assert "DatasetConfig options" in helptext + + +def test_counter_action() -> None: + def main( + verbosity: tyro.conf.UseCounterAction[int], + aliased_verbosity: Annotated[ + tyro.conf.UseCounterAction[int], tyro.conf.arg(aliases=["-v"]) + ], + ) -> Tuple[int, int]: + """Example showing how to use counter actions. + Args: + verbosity: Verbosity level. + aliased_verbosity: Same as above, but can also be specified with -v, -vv, -vvv, etc. + """ + return verbosity, aliased_verbosity + + assert tyro.cli(main, args=[]) == (0, 0) + assert tyro.cli(main, args="--verbosity --verbosity".split(" ")) == (2, 0) + assert tyro.cli(main, args="--verbosity --verbosity -v".split(" ")) == (2, 1) + if sys.version_info >= (3, 8): + # Doesn't work in Python 3.7 because of argparse limitations. + assert tyro.cli(main, args="--verbosity --verbosity -vv".split(" ")) == (2, 2) + assert tyro.cli(main, args="--verbosity --verbosity -vvv".split(" ")) == (2, 3) From eafae2eaf5402d084d0e956bdc3e9465a67f4c55 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 4 Apr 2024 02:23:53 -0700 Subject: [PATCH 389/491] Support Python 3.12 + PEP 695 (`type` statement, new type parameter syntax) (#135) * Support Python 3.12, add new-style generics example + test * Add support for for `type` statement * Use Python 3.12 for mypy * `tuple` => `Tuple` in test * ruff * Specify Python 3.12 for docs * Python 3.12 for coverage * Suppress mypy errors * Ignore leading comments in docs update script * Update type param example --- .github/workflows/build.yml | 2 +- .github/workflows/coverage.yml | 4 +- .github/workflows/docs.yml | 6 +++ .github/workflows/mypy.yml | 4 +- .github/workflows/publish.yml | 2 +- .github/workflows/pyright.yml | 2 +- .../04_additional/06_generics_py312.rst | 52 ++++++++++++++++++ .../{06_conf.rst => 07_conf.rst} | 8 +-- .../{07_flax.rst => 10_flax.rst} | 8 +-- ...ructors.rst => 11_custom_constructors.rst} | 12 ++--- .../{11_aliases.rst => 12_aliases.rst} | 28 +++++----- .../04_additional/13_type_statement.rst | 52 ++++++++++++++++++ docs/update_example_docs.py | 2 +- examples/04_additional/06_generics_py312.py | 40 ++++++++++++++ .../04_additional/{06_conf.py => 07_conf.py} | 0 .../04_additional/{07_flax.py => 10_flax.py} | 0 ...structors.py => 11_custom_constructors.py} | 0 .../{11_aliases.py => 12_aliases.py} | 0 examples/04_additional/13_type_statement.py | 40 ++++++++++++++ pyproject.toml | 3 +- src/tyro/_instantiators.py | 16 +++--- src/tyro/_resolver.py | 13 +++-- src/tyro/_subcommand_matching.py | 4 +- tests/conftest.py | 6 ++- tests/test_generics_and_serialization.py | 11 ++++ tests/test_new_style_annotations_min_py312.py | 54 +++++++++++++++++++ 26 files changed, 317 insertions(+), 52 deletions(-) create mode 100644 docs/source/examples/04_additional/06_generics_py312.rst rename docs/source/examples/04_additional/{06_conf.rst => 07_conf.rst} (85%) rename docs/source/examples/04_additional/{07_flax.rst => 10_flax.rst} (87%) rename docs/source/examples/04_additional/{10_custom_constructors.rst => 11_custom_constructors.rst} (80%) rename docs/source/examples/04_additional/{11_aliases.rst => 12_aliases.rst} (62%) create mode 100644 docs/source/examples/04_additional/13_type_statement.rst create mode 100644 examples/04_additional/06_generics_py312.py rename examples/04_additional/{06_conf.py => 07_conf.py} (100%) rename examples/04_additional/{07_flax.py => 10_flax.py} (100%) rename examples/04_additional/{10_custom_constructors.py => 11_custom_constructors.py} (100%) rename examples/04_additional/{11_aliases.py => 12_aliases.py} (100%) create mode 100644 examples/04_additional/13_type_statement.py create mode 100644 tests/test_new_style_annotations_min_py312.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 300bf6daf..5263f8f90 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index d07af0bc8..9360a5156 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python 3.10 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.12" - name: Install dependencies run: | pip install --upgrade pip diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b59a66cf6..61391e042 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,6 +12,12 @@ jobs: # Check out source - uses: actions/checkout@v2 + # Set up Python + - name: "Set up Python 3.12" + uses: actions/setup-python@v4 + with: + python-version: "3.12" + # Build documentation - name: Building documentation run: | diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 125eb9529..fd53d6746 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: "Set up Python 3.10" + - name: "Set up Python 3.12" uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.12" - name: Install dependencies run: | pip install --upgrade pip diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e75680dee..8fe49129f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: '3.10' - name: Install dependencies run: | pip install --upgrade pip diff --git a/.github/workflows/pyright.yml b/.github/workflows/pyright.yml index 3f96b89d3..66946c1ed 100644 --- a/.github/workflows/pyright.yml +++ b/.github/workflows/pyright.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v2 diff --git a/docs/source/examples/04_additional/06_generics_py312.rst b/docs/source/examples/04_additional/06_generics_py312.rst new file mode 100644 index 000000000..dd214d8e0 --- /dev/null +++ b/docs/source/examples/04_additional/06_generics_py312.rst @@ -0,0 +1,52 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Generic Types (Python 3.12+ syntax) +========================================== + + +Example of parsing for generic dataclasses using syntax introduced in Python +3.12. Note: this is not compatible with ``from __future__ import annotations``. + + + +.. code-block:: python + :linenos: + + + import dataclasses + + import tyro + + + @dataclasses.dataclass(frozen=True) + class Point3[ScalarType: (int, float)]: + x: ScalarType + y: ScalarType + z: ScalarType + frame_id: str + + + @dataclasses.dataclass(frozen=True) + class Triangle: + a: Point3[float] + b: Point3[float] + c: Point3[float] + + + @dataclasses.dataclass(frozen=True) + class Args[ShapeType]: + shape: ShapeType + + + if __name__ == "__main__": + args = tyro.cli(Args[Triangle]) + print(args) + +------------ + +.. raw:: html + + python 04_additional/06_generics_py312.py --help + +.. program-output:: python ../../examples/04_additional/06_generics_py312.py --help diff --git a/docs/source/examples/04_additional/06_conf.rst b/docs/source/examples/04_additional/07_conf.rst similarity index 85% rename from docs/source/examples/04_additional/06_conf.rst rename to docs/source/examples/04_additional/07_conf.rst index 192c03991..babc7a146 100644 --- a/docs/source/examples/04_additional/06_conf.rst +++ b/docs/source/examples/04_additional/07_conf.rst @@ -52,14 +52,14 @@ Features here are supported, but generally unnecessary and should be used sparin .. raw:: html - python 04_additional/06_conf.py --help + python 04_additional/07_conf.py --help -.. program-output:: python ../../examples/04_additional/06_conf.py --help +.. program-output:: python ../../examples/04_additional/07_conf.py --help ------------ .. raw:: html - python 04_additional/06_conf.py 5 --boolean True + python 04_additional/07_conf.py 5 --boolean True -.. program-output:: python ../../examples/04_additional/06_conf.py 5 --boolean True +.. program-output:: python ../../examples/04_additional/07_conf.py 5 --boolean True diff --git a/docs/source/examples/04_additional/07_flax.rst b/docs/source/examples/04_additional/10_flax.rst similarity index 87% rename from docs/source/examples/04_additional/07_flax.rst rename to docs/source/examples/04_additional/10_flax.rst index d9116016b..8269f51c9 100644 --- a/docs/source/examples/04_additional/07_flax.rst +++ b/docs/source/examples/04_additional/10_flax.rst @@ -63,14 +63,14 @@ directly from ``tyro.cli``. .. raw:: html - python 04_additional/07_flax.py --help + python 04_additional/10_flax.py --help -.. program-output:: python ../../examples/04_additional/07_flax.py --help +.. program-output:: python ../../examples/04_additional/10_flax.py --help ------------ .. raw:: html - python 04_additional/07_flax.py --model.layers 4 + python 04_additional/10_flax.py --model.layers 4 -.. program-output:: python ../../examples/04_additional/07_flax.py --model.layers 4 +.. program-output:: python ../../examples/04_additional/10_flax.py --model.layers 4 diff --git a/docs/source/examples/04_additional/10_custom_constructors.rst b/docs/source/examples/04_additional/11_custom_constructors.rst similarity index 80% rename from docs/source/examples/04_additional/10_custom_constructors.rst rename to docs/source/examples/04_additional/11_custom_constructors.rst index fb3f8880b..9c650ea83 100644 --- a/docs/source/examples/04_additional/10_custom_constructors.rst +++ b/docs/source/examples/04_additional/11_custom_constructors.rst @@ -49,22 +49,22 @@ which makes it easier to load complex objects. .. raw:: html - python 04_additional/10_custom_constructors.py --help + python 04_additional/11_custom_constructors.py --help -.. program-output:: python ../../examples/04_additional/10_custom_constructors.py --help +.. program-output:: python ../../examples/04_additional/11_custom_constructors.py --help ------------ .. raw:: html - python 04_additional/10_custom_constructors.py --dict1.json '{"hello": "world"}' + python 04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' -.. program-output:: python ../../examples/04_additional/10_custom_constructors.py --dict1.json '{"hello": "world"}' +.. program-output:: python ../../examples/04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' ------------ .. raw:: html - python 04_additional/10_custom_constructors.py --dict1.json '{"hello": "world"}`' --dict2.json '{"hello": "world"}' + python 04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}`' --dict2.json '{"hello": "world"}' -.. program-output:: python ../../examples/04_additional/10_custom_constructors.py --dict1.json '{"hello": "world"}`' --dict2.json '{"hello": "world"}' +.. program-output:: python ../../examples/04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}`' --dict2.json '{"hello": "world"}' diff --git a/docs/source/examples/04_additional/11_aliases.rst b/docs/source/examples/04_additional/12_aliases.rst similarity index 62% rename from docs/source/examples/04_additional/11_aliases.rst rename to docs/source/examples/04_additional/12_aliases.rst index 8a13511fc..7505b954c 100644 --- a/docs/source/examples/04_additional/11_aliases.rst +++ b/docs/source/examples/04_additional/12_aliases.rst @@ -45,54 +45,54 @@ Argument aliases .. raw:: html - python 04_additional/11_aliases.py --help + python 04_additional/12_aliases.py --help -.. program-output:: python ../../examples/04_additional/11_aliases.py --help +.. program-output:: python ../../examples/04_additional/12_aliases.py --help ------------ .. raw:: html - python 04_additional/11_aliases.py commit --help + python 04_additional/12_aliases.py commit --help -.. program-output:: python ../../examples/04_additional/11_aliases.py commit --help +.. program-output:: python ../../examples/04_additional/12_aliases.py commit --help ------------ .. raw:: html - python 04_additional/11_aliases.py commit --message hello --all + python 04_additional/12_aliases.py commit --message hello --all -.. program-output:: python ../../examples/04_additional/11_aliases.py commit --message hello --all +.. program-output:: python ../../examples/04_additional/12_aliases.py commit --message hello --all ------------ .. raw:: html - python 04_additional/11_aliases.py commit -m hello -a + python 04_additional/12_aliases.py commit -m hello -a -.. program-output:: python ../../examples/04_additional/11_aliases.py commit -m hello -a +.. program-output:: python ../../examples/04_additional/12_aliases.py commit -m hello -a ------------ .. raw:: html - python 04_additional/11_aliases.py checkout --help + python 04_additional/12_aliases.py checkout --help -.. program-output:: python ../../examples/04_additional/11_aliases.py checkout --help +.. program-output:: python ../../examples/04_additional/12_aliases.py checkout --help ------------ .. raw:: html - python 04_additional/11_aliases.py checkout --branch main + python 04_additional/12_aliases.py checkout --branch main -.. program-output:: python ../../examples/04_additional/11_aliases.py checkout --branch main +.. program-output:: python ../../examples/04_additional/12_aliases.py checkout --branch main ------------ .. raw:: html - python 04_additional/11_aliases.py checkout -b main + python 04_additional/12_aliases.py checkout -b main -.. program-output:: python ../../examples/04_additional/11_aliases.py checkout -b main +.. program-output:: python ../../examples/04_additional/12_aliases.py checkout -b main diff --git a/docs/source/examples/04_additional/13_type_statement.rst b/docs/source/examples/04_additional/13_type_statement.rst new file mode 100644 index 000000000..6f4890fe8 --- /dev/null +++ b/docs/source/examples/04_additional/13_type_statement.rst @@ -0,0 +1,52 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Type aliases (Python 3.12+) +========================================== + + +In Python 3.12, the ``type`` statement is introduced to create type aliases. + + + +.. code-block:: python + :linenos: + + + import dataclasses + + import tyro + + # Lazily-evaluated type alias. + type Field1Type = Inner + + + @dataclasses.dataclass + class Inner: + a: int + b: str + + + @dataclasses.dataclass + class Args: + """Description. + This should show up in the helptext!""" + + field1: Field1Type + """A field.""" + + field2: int = 3 + """A numeric field, with a default value.""" + + + if __name__ == "__main__": + args = tyro.cli(Args) + print(args) + +------------ + +.. raw:: html + + python 04_additional/13_type_statement.py --help + +.. program-output:: python ../../examples/04_additional/13_type_statement.py --help diff --git a/docs/update_example_docs.py b/docs/update_example_docs.py index 093280ac3..43ae770e7 100644 --- a/docs/update_example_docs.py +++ b/docs/update_example_docs.py @@ -49,7 +49,7 @@ def from_path(path: pathlib.Path) -> ExampleMetadata: return ExampleMetadata( index=index, index_with_zero=index_with_zero, - source=source[3:].partition('"""')[2].strip(), + source=source.partition('"""')[2].partition('"""')[2].strip(), title=title, usages=example_usages, description=description.strip(), diff --git a/examples/04_additional/06_generics_py312.py b/examples/04_additional/06_generics_py312.py new file mode 100644 index 000000000..f68624a6a --- /dev/null +++ b/examples/04_additional/06_generics_py312.py @@ -0,0 +1,40 @@ +# mypy: ignore-errors +# +# PEP 695 isn't yet supported in mypy. (April 4, 2024) +"""Generic Types (Python 3.12+ syntax) + +Example of parsing for generic dataclasses using syntax introduced in Python +3.12. Note: this is not compatible with `from __future__ import annotations`. + +Usage: +`python ./05_generics.py --help` +""" + +import dataclasses + +import tyro + + +@dataclasses.dataclass(frozen=True) +class Point3[ScalarType: (int, float)]: + x: ScalarType + y: ScalarType + z: ScalarType + frame_id: str + + +@dataclasses.dataclass(frozen=True) +class Triangle: + a: Point3[float] + b: Point3[float] + c: Point3[float] + + +@dataclasses.dataclass(frozen=True) +class Args[ShapeType]: + shape: ShapeType + + +if __name__ == "__main__": + args = tyro.cli(Args[Triangle]) + print(args) diff --git a/examples/04_additional/06_conf.py b/examples/04_additional/07_conf.py similarity index 100% rename from examples/04_additional/06_conf.py rename to examples/04_additional/07_conf.py diff --git a/examples/04_additional/07_flax.py b/examples/04_additional/10_flax.py similarity index 100% rename from examples/04_additional/07_flax.py rename to examples/04_additional/10_flax.py diff --git a/examples/04_additional/10_custom_constructors.py b/examples/04_additional/11_custom_constructors.py similarity index 100% rename from examples/04_additional/10_custom_constructors.py rename to examples/04_additional/11_custom_constructors.py diff --git a/examples/04_additional/11_aliases.py b/examples/04_additional/12_aliases.py similarity index 100% rename from examples/04_additional/11_aliases.py rename to examples/04_additional/12_aliases.py diff --git a/examples/04_additional/13_type_statement.py b/examples/04_additional/13_type_statement.py new file mode 100644 index 000000000..5f721e8a4 --- /dev/null +++ b/examples/04_additional/13_type_statement.py @@ -0,0 +1,40 @@ +# mypy: ignore-errors +# +# PEP 695 isn't yet supported in mypy. (April 4, 2024) +"""Type aliases (Python 3.12+) + +In Python 3.12, the `type` statement is introduced to create type aliases. + +Usage: +`python ./13_type_statement.py --help` +""" + +import dataclasses + +import tyro + +# Lazily-evaluated type alias. +type Field1Type = Inner + + +@dataclasses.dataclass +class Inner: + a: int + b: str + + +@dataclasses.dataclass +class Args: + """Description. + This should show up in the helptext!""" + + field1: Field1Type + """A field.""" + + field2: int = 3 + """A numeric field, with a default value.""" + + +if __name__ == "__main__": + args = tyro.cli(Args) + print(args) diff --git a/pyproject.toml b/pyproject.toml index 1c9449584..e4b6c0d33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent" ] @@ -63,7 +64,7 @@ tyro = ["py.typed"] profile = "black" [tool.mypy] -python_version = "3.10" +python_version = "3.12" ignore_missing_imports = true warn_unused_configs = true exclude = "^tests/test_py311_generated/.*" diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index 7eb9513aa..115b6f304 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -188,6 +188,14 @@ def instantiator(strings: List[str]) -> None: if typ is os.PathLike: typ = pathlib.Path + # Unwrap NewType + set metavar based on NewType name. + # `isinstance(x, NewType)` doesn't work because NewType isn't a class until + # Python 3.10, so we instead do a duck typing-style check. + metavar = getattr(typ, "__name__", "").upper() + typ, maybe_newtype_name = _resolver.unwrap_newtype_and_aliases(typ) + if maybe_newtype_name is not None: + metavar = maybe_newtype_name.upper() + # Address container types. If a matching container is found, this will recursively # call instantiator_from_type(). container_out = _instantiator_from_container_type( @@ -196,14 +204,6 @@ def instantiator(strings: List[str]) -> None: if container_out is not None: return container_out - # Unwrap NewType + set metavar based on NewType name. - # `isinstance(x, NewType)` doesn't work because NewType isn't a class until - # Python 3.10, so we instead do a duck typing-style check. - metavar = getattr(typ, "__name__", "").upper() - typ, maybe_newtype_name = _resolver.unwrap_newtype(typ) - if maybe_newtype_name is not None: - metavar = maybe_newtype_name.upper() - # Validate that typ is a `(arg: str) -> T` type converter, as expected by argparse. if typ in _builtin_set: pass diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index ae8759f7b..346b4776f 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -26,6 +26,7 @@ Annotated, ForwardRef, Self, + TypeAliasType, get_args, get_origin, get_type_hints, @@ -69,7 +70,7 @@ def resolve_generic_types( cls, annotations = unwrap_annotated(cls) # We'll ignore NewType when getting the origin + args for generics. - origin_cls = get_origin(unwrap_newtype(cls)[0]) + origin_cls = get_origin(unwrap_newtype_and_aliases(cls)[0]) type_from_typevar: Dict[TypeVar, TypeForm[Any]] = {} # Support typing.Self. @@ -88,7 +89,7 @@ def resolve_generic_types( and hasattr(origin_cls.__parameters__, "__len__") ): typevars = origin_cls.__parameters__ - typevar_values = get_args(unwrap_newtype(cls)[0]) + typevar_values = get_args(unwrap_newtype_and_aliases(cls)[0]) assert len(typevars) == len(typevar_values) cls = origin_cls type_from_typevar.update(dict(zip(typevars, typevar_values))) @@ -167,9 +168,13 @@ def type_from_typevar_constraints(typ: TypeOrCallable) -> TypeOrCallable: TypeOrCallableOrNone = TypeVar("TypeOrCallableOrNone", Callable, TypeForm[Any], None) -def unwrap_newtype( +def unwrap_newtype_and_aliases( typ: TypeOrCallableOrNone, ) -> Tuple[TypeOrCallableOrNone, Optional[str]]: + # Handle type aliases, eg via the `type` statement in Python 3.12. + if isinstance(typ, TypeAliasType): + return unwrap_newtype_and_aliases(typ.__value__) # type: ignore + # We'll unwrap NewType annotations here; this is needed before issubclass # checks! # @@ -197,7 +202,7 @@ def unwrap_newtype_and_narrow_subtypes( string default is passed in, we don't want to narrow the type to always be strings!)""" - typ, unused_name = unwrap_newtype(typ) + typ, unused_name = unwrap_newtype_and_aliases(typ) del unused_name try: diff --git a/src/tyro/_subcommand_matching.py b/src/tyro/_subcommand_matching.py index 94cad1c36..16884bcf8 100644 --- a/src/tyro/_subcommand_matching.py +++ b/src/tyro/_subcommand_matching.py @@ -97,11 +97,11 @@ def _get_type_options(typ: _typing.TypeForm) -> Tuple[_typing.TypeForm, ...]: # Check against supertypes. for self_type in self_types: self_type = _resolver.unwrap_annotated(self_type)[0] - self_type, _ = _resolver.unwrap_newtype(self_type) + self_type, _ = _resolver.unwrap_newtype_and_aliases(self_type) ok = False for super_type in super_types: super_type = _resolver.unwrap_annotated(super_type)[0] - self_type, _ = _resolver.unwrap_newtype(self_type) + self_type, _ = _resolver.unwrap_newtype_and_aliases(self_type) if issubclass(self_type, super_type): ok = True if not ok: diff --git a/tests/conftest.py b/tests/conftest.py index 63f1e1b87..31dfb77f0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ import sys +from typing import List -collect_ignore_glob = [] +collect_ignore_glob: List[str] = [] if not sys.version_info >= (3, 8): collect_ignore_glob.append("*_min_py38.py") @@ -11,5 +12,8 @@ if not sys.version_info >= (3, 10): collect_ignore_glob.append("*_min_py310.py") +if not sys.version_info >= (3, 12): + collect_ignore_glob.append("*_min_py312.py") + if not sys.version_info >= (3, 11): collect_ignore_glob.append("test_py311_generated/*.py") diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 6f42315a2..89d3d48ad 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -41,6 +41,17 @@ class TupleGenericVariable(Generic[ScalarType]): ) == TupleGenericVariable((SpecialInt(1), SpecialInt(2), SpecialInt(3))) +def test_tuple_generic_variable_newtype_container() -> None: + @dataclasses.dataclass + class TupleGenericVariable(Generic[ScalarType]): + xyz: Tuple[ScalarType, ...] + + SpecialInt = NewType("SpecialInt", Tuple[int]) + assert tyro.cli( + TupleGenericVariable[SpecialInt], args=["--xyz", "1", "2", "3"] + ) == TupleGenericVariable((SpecialInt((1,)), SpecialInt((2,)), SpecialInt((3,)))) + + def test_tuple_generic_variable_more_newtype() -> None: @dataclasses.dataclass class TupleGenericVariable(Generic[ScalarType]): diff --git a/tests/test_new_style_annotations_min_py312.py b/tests/test_new_style_annotations_min_py312.py new file mode 100644 index 000000000..d04c73f3b --- /dev/null +++ b/tests/test_new_style_annotations_min_py312.py @@ -0,0 +1,54 @@ +# mypy: ignore-errors +# +# PEP 695 isn't yet supported in mypy. (April 4, 2024) +from dataclasses import dataclass + +import tyro + + +def test_simple_generic(): + @dataclass(frozen=True) + class Container[T]: + a: T + b: T + + assert tyro.cli(Container[int], args="--a 1 --b 2".split(" ")) == Container(1, 2) + + +type X = int +type Y = list[int] +type Z = Inner[int] + + +@dataclass(frozen=True) +class Inner[T]: + a: T + b: T + + +def test_generic_with_type_statement_0(): + @dataclass(frozen=True) + class Container[T]: + a: T + b: T + + assert tyro.cli(Container[X], args="--a 1 --b 2".split(" ")) == Container(1, 2) + + +def test_generic_with_type_statement_1(): + @dataclass(frozen=True) + class Container[T]: + a: tuple[X, ...] + b: T + + assert tyro.cli(Container[Y], args="--a 1 --b 2".split(" ")) == Container((1,), [2]) + + +def test_generic_with_type_statement_2(): + @dataclass(frozen=True) + class Container[T]: + a: Z + + assert tyro.cli(Container[Y], args="--a.a 1 --a.b 2".split(" ")) == Container( + Inner(1, 2) + ) From 6720c58c163b04ee5c57b58fcf73cc3ce8c2b820 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 4 Apr 2024 14:52:39 -0700 Subject: [PATCH 390/491] Bump version to 0.8.0 --- docs/source/examples/04_additional/12_aliases.rst | 2 +- docs/source/examples/04_additional/13_type_statement.rst | 2 +- examples/04_additional/12_aliases.py | 2 +- examples/04_additional/13_type_statement.py | 2 +- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/examples/04_additional/12_aliases.rst b/docs/source/examples/04_additional/12_aliases.rst index 7505b954c..f6368e5c3 100644 --- a/docs/source/examples/04_additional/12_aliases.rst +++ b/docs/source/examples/04_additional/12_aliases.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -Argument aliases +Argument Aliases ========================================== diff --git a/docs/source/examples/04_additional/13_type_statement.rst b/docs/source/examples/04_additional/13_type_statement.rst index 6f4890fe8..7406e0825 100644 --- a/docs/source/examples/04_additional/13_type_statement.rst +++ b/docs/source/examples/04_additional/13_type_statement.rst @@ -1,7 +1,7 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -Type aliases (Python 3.12+) +Type Aliases (Python 3.12+) ========================================== diff --git a/examples/04_additional/12_aliases.py b/examples/04_additional/12_aliases.py index 212cffaa2..5ee33de9c 100644 --- a/examples/04_additional/12_aliases.py +++ b/examples/04_additional/12_aliases.py @@ -1,4 +1,4 @@ -"""Argument aliases +"""Argument Aliases :func:`tyro.conf.arg()` can be used to attach aliases to arguments. diff --git a/examples/04_additional/13_type_statement.py b/examples/04_additional/13_type_statement.py index 5f721e8a4..a892bf735 100644 --- a/examples/04_additional/13_type_statement.py +++ b/examples/04_additional/13_type_statement.py @@ -1,7 +1,7 @@ # mypy: ignore-errors # # PEP 695 isn't yet supported in mypy. (April 4, 2024) -"""Type aliases (Python 3.12+) +"""Type Aliases (Python 3.12+) In Python 3.12, the `type` statement is introduced to create type aliases. diff --git a/pyproject.toml b/pyproject.toml index e4b6c0d33..6647d5b88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.7.3" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.0" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index e9ff70179..b262e3e33 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.7.3" +__version__ = "0.8.0" From c3ded1c75f1c97a3862b6e177f78295e3dae5cd1 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 5 Apr 2024 12:44:42 -0700 Subject: [PATCH 391/491] Fix edge case in container narrowing logic (#136) --- src/tyro/_cli.py | 4 ++-- src/tyro/_resolver.py | 8 ++++++-- tests/test_collections.py | 13 +++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index a3be76a7d..c8b0d3478 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -329,9 +329,9 @@ def _cli_impl( if "=" in arg: arg, _, val = arg.partition("=") - fixed = "--" + arg[2:].replace(to_swap_delimeter, delimeter) + "=" + val + fixed = "--" + _strings.replace_delimeter_in_part(arg[2:]) + "=" + val else: - fixed = "--" + arg[2:].replace(to_swap_delimeter, delimeter) + fixed = "--" + _strings.replace_delimeter_in_part(arg[2:]) if ( return_unknown_args and fixed in modified_args diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 346b4776f..082d019df 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -237,13 +237,17 @@ def narrow_collection_types( typ: TypeOrCallable, default_instance: Any ) -> TypeOrCallable: """TypeForm narrowing for containers. Infers types of container contents.""" - if hasattr(default_instance, "__len__") and len(default_instance) == 0: - return typ if typ is list and isinstance(default_instance, list): + if len(default_instance) == 0: + return typ typ = List.__getitem__(Union.__getitem__(tuple(map(type, default_instance)))) # type: ignore elif typ is set and isinstance(default_instance, set): + if len(default_instance) == 0: + return typ typ = Set.__getitem__(Union.__getitem__(tuple(map(type, default_instance)))) # type: ignore elif typ is tuple and isinstance(default_instance, tuple): + if len(default_instance) == 0: + return typ typ = Tuple.__getitem__(tuple(map(type, default_instance))) # type: ignore return typ diff --git a/tests/test_collections.py b/tests/test_collections.py index ce1719dc0..a8782c0e1 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -13,6 +13,7 @@ Sequence, Set, Tuple, + Type, Union, ) @@ -466,6 +467,18 @@ def main(x: tuple = ()) -> Any: assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == ("0", "1", "2", "3") +def test_narrowing_edge_case() -> None: + """https://github.com/brentyi/tyro/issues/136""" + @dataclasses.dataclass + class Config: + _target: Type = dataclasses.field(default_factory=lambda: MyClass) + + class MyClass: + def __len__(self): + return 0 + + assert tyro.cli(Config, args=[]) == Config() + def test_no_type_collections(): assert tyro.cli(dict, args="a b c d".split(" ")) == {"a": "b", "c": "d"} From 3dee2d90d169e087268f0aa95c0dec3d0e729c7d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 5 Apr 2024 12:46:10 -0700 Subject: [PATCH 392/491] Bump version to 0.8.1 --- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6647d5b88..3c889fca6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.0" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.1" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index b262e3e33..770c75ce5 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.0" +__version__ = "0.8.1" From d7d291125738bde9560d96d9ddd2c27942b0a656 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 5 Apr 2024 12:47:44 -0700 Subject: [PATCH 393/491] Run ruff --- src/tyro/_cli.py | 3 --- tests/test_collections.py | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index c8b0d3478..680e238fe 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -324,9 +324,6 @@ def _cli_impl( if not arg.startswith("--"): continue - delimeter = _strings.get_delimeter() - to_swap_delimeter = "-" if delimeter == "_" else "_" - if "=" in arg: arg, _, val = arg.partition("=") fixed = "--" + _strings.replace_delimeter_in_part(arg[2:]) + "=" + val diff --git a/tests/test_collections.py b/tests/test_collections.py index a8782c0e1..e4aa3fe80 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -467,8 +467,10 @@ def main(x: tuple = ()) -> Any: assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == ("0", "1", "2", "3") + def test_narrowing_edge_case() -> None: """https://github.com/brentyi/tyro/issues/136""" + @dataclasses.dataclass class Config: _target: Type = dataclasses.field(default_factory=lambda: MyClass) From 107b05e8c3c43f0f40810abfe6358f5c42d24cbd Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 5 Apr 2024 16:56:16 -0700 Subject: [PATCH 394/491] Bump `typing_extensions` dependency to `>=4.7.0` for `ForwardRef` support related: https://github.com/nerfstudio-project/nerfstudio/issues/3050 --- pyproject.toml | 4 ++-- src/tyro/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3c889fca6..b2c47a7a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.1" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.2" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } @@ -25,7 +25,7 @@ classifiers = [ ] dependencies = [ "docstring-parser>=0.14.1", - "typing-extensions>=4.3.0", + "typing-extensions>=4.7.0", "backports.cached-property>=1.0.2; python_version<'3.8'", "colorama>=0.4.0; platform_system=='Windows'", "rich>=11.1.0", diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index 770c75ce5..7e6a139a5 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.1" +__version__ = "0.8.2" From 2fcdcd9dbab1aaa84ec753f0bffb351c384fc5d0 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 8 Apr 2024 18:05:46 -0700 Subject: [PATCH 395/491] Switch to `uv` for CI --- .github/workflows/build.yml | 6 ++++++ .github/workflows/coverage.yml | 4 ++-- .github/workflows/docs.yml | 6 +++--- .github/workflows/mypy.yml | 4 ++-- .github/workflows/publish.yml | 6 +++--- .github/workflows/pyright.yml | 6 +++--- .github/workflows/ruff.yml | 4 ++-- 7 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5263f8f90..94c1a7d4d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,9 +20,15 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies + if: matrix.python-version == '3.7' run: | pip install --upgrade pip pip install ".[dev]" + - name: Install dependencies + if: matrix.python-version != '3.7' + run: | + pip install uv + uv pip install --system ".[dev]" - name: Test with pytest run: | pytest diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 9360a5156..f3632e003 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,8 +17,8 @@ jobs: python-version: "3.12" - name: Install dependencies run: | - pip install --upgrade pip - pip install -e ".[dev]" + pip install uv + uv pip install --system -e ".[dev]" - name: Generate coverage report run: | pytest --cov=tyro --cov-report=xml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 61391e042..09d4e5975 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -21,9 +21,9 @@ jobs: # Build documentation - name: Building documentation run: | - pip install --upgrade pip - pip install -e ".[dev]" - pip install -r docs/requirements.txt + pip install uv + uv pip install --system -e ".[dev]" + uv pip install --system -r docs/requirements.txt sphinx-build docs/source docs/build -b dirhtml # Deploy diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index fd53d6746..1ed0331d9 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -17,8 +17,8 @@ jobs: python-version: "3.12" - name: Install dependencies run: | - pip install --upgrade pip - pip install -e ".[dev]" + pip install uv + uv pip install --system -e ".[dev]" - name: Test with mypy run: | mypy --install-types --non-interactive . diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8fe49129f..f694231ff 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,9 +20,9 @@ jobs: python-version: '3.10' - name: Install dependencies run: | - pip install --upgrade pip - pip install -e ".[dev]" - pip install build twine + pip install uv + uv pip install --system -e ".[dev]" + uv pip install --system build twine - name: Strip unsupported tags in README run: | sed -i '//,//d' README.md diff --git a/.github/workflows/pyright.yml b/.github/workflows/pyright.yml index 66946c1ed..7da8fd2ce 100644 --- a/.github/workflows/pyright.yml +++ b/.github/workflows/pyright.yml @@ -21,9 +21,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - pip install --upgrade pip - pip install -e ".[dev]" - pip install -r docs/requirements.txt + pip install uv + uv pip install --system -e ".[dev]" + uv pip install --system -r docs/requirements.txt - name: Run pyright run: | pyright . diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 001833420..657d36a7d 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -17,8 +17,8 @@ jobs: python-version: "3.10" - name: Install dependencies run: | - pip install --upgrade pip - pip install -e ".[dev]" + pip install uv + uv pip install --system -e ".[dev]" - name: Ruff check run: | ruff check --output-format github From ec777927e553349720ebeac4bbc38a5fb9a0d976 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 9 Apr 2024 02:09:06 -0700 Subject: [PATCH 396/491] Bump version, add internal mirror of `argparse` This should fix problems with Python 3.11.9 --- pyproject.toml | 12 +- src/tyro/__init__.py | 2 +- src/tyro/_argparse.py | 2672 +++++++++++++++++ src/tyro/_argparse_formatter.py | 36 +- src/tyro/_arguments.py | 3 +- src/tyro/_cli.py | 2 +- src/tyro/_parsers.py | 2 +- tests/test_py311_generated/_generate.py | 2 +- .../test_collections_generated.py | 15 + .../test_conf_generated.py | 24 + ...st_generics_and_serialization_generated.py | 11 + .../test_self_type_generated.py | 36 +- 12 files changed, 2777 insertions(+), 40 deletions(-) create mode 100644 src/tyro/_argparse.py diff --git a/pyproject.toml b/pyproject.toml index b2c47a7a8..aa55c51a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.2" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.3" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } @@ -67,7 +67,10 @@ profile = "black" python_version = "3.12" ignore_missing_imports = true warn_unused_configs = true -exclude = "^tests/test_py311_generated/.*" +exclude = ["^tests/test_py311_generated/.*", "_argparse\\.py"] + +[tool.coverage.run] +omit = ["**/_argparse.py"] [tool.coverage.report] exclude_lines = [ @@ -127,3 +130,8 @@ lint.ignore = [ "PLW0603", # Global statement updates are discouraged. "PLW2901" # For loop variable overwritten. ] +extend-exclude = ["**/_argparse.py"] + +[tool.pyright] +pythonVersion = "3.12" +ignore = ["**/_argparse.py"] diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index 7e6a139a5..65be2014c 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.2" +__version__ = "0.8.3" diff --git a/src/tyro/_argparse.py b/src/tyro/_argparse.py new file mode 100644 index 000000000..f5dcdf0f4 --- /dev/null +++ b/src/tyro/_argparse.py @@ -0,0 +1,2672 @@ +# Mirror of Python's argparse module. +# Should be an exact copy from: +# https://github.com/python/cpython/blob/3.12/Lib/argparse.py + +# Author: Steven J. Bethard . +# New maintainer as of 29 August 2019: Raymond Hettinger + +"""Command-line parsing library + +This module is an optparse-inspired command-line parsing library that: + + - handles both optional and positional arguments + - produces highly informative usage messages + - supports parsers that dispatch to sub-parsers + +The following is a simple usage example that sums integers from the +command-line and writes the result to a file:: + + parser = argparse.ArgumentParser( + description='sum the integers at the command line') + parser.add_argument( + 'integers', metavar='int', nargs='+', type=int, + help='an integer to be summed') + parser.add_argument( + '--log', default=sys.stdout, type=argparse.FileType('w'), + help='the file where the sum should be written') + args = parser.parse_args() + args.log.write('%s' % sum(args.integers)) + args.log.close() + +The module contains the following public classes: + + - ArgumentParser -- The main entry point for command-line parsing. As the + example above shows, the add_argument() method is used to populate + the parser with actions for optional and positional arguments. Then + the parse_args() method is invoked to convert the args at the + command-line into an object with attributes. + + - ArgumentError -- The exception raised by ArgumentParser objects when + there are errors with the parser's actions. Errors raised while + parsing the command-line are caught by ArgumentParser and emitted + as command-line messages. + + - FileType -- A factory for defining types of files to be created. As the + example above shows, instances of FileType are typically passed as + the type= argument of add_argument() calls. + + - Action -- The base class for parser actions. Typically actions are + selected by passing strings like 'store_true' or 'append_const' to + the action= argument of add_argument(). However, for greater + customization of ArgumentParser actions, subclasses of Action may + be defined and passed as the action= argument. + + - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter, + ArgumentDefaultsHelpFormatter -- Formatter classes which + may be passed as the formatter_class= argument to the + ArgumentParser constructor. HelpFormatter is the default, + RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser + not to change the formatting for help text, and + ArgumentDefaultsHelpFormatter adds information about argument defaults + to the help. + +All other classes in this module are considered implementation details. +(Also note that HelpFormatter and RawDescriptionHelpFormatter are only +considered public as object names -- the API of the formatter objects is +still considered an implementation detail.) +""" + +__version__ = '1.1' +__all__ = [ + 'ArgumentParser', + 'ArgumentError', + 'ArgumentTypeError', + 'BooleanOptionalAction', + 'FileType', + 'HelpFormatter', + 'ArgumentDefaultsHelpFormatter', + 'RawDescriptionHelpFormatter', + 'RawTextHelpFormatter', + 'MetavarTypeHelpFormatter', + 'Namespace', + 'Action', + 'ONE_OR_MORE', + 'OPTIONAL', + 'PARSER', + 'REMAINDER', + 'SUPPRESS', + 'ZERO_OR_MORE', +] + + +import os as _os +import re as _re +import sys as _sys + +import warnings + +from gettext import gettext as _, ngettext + +SUPPRESS = '==SUPPRESS==' + +OPTIONAL = '?' +ZERO_OR_MORE = '*' +ONE_OR_MORE = '+' +PARSER = 'A...' +REMAINDER = '...' +_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args' + +# ============================= +# Utility functions and classes +# ============================= + +class _AttributeHolder(object): + """Abstract base class that provides __repr__. + + The __repr__ method returns a string in the format:: + ClassName(attr=name, attr=name, ...) + The attributes are determined either by a class-level attribute, + '_kwarg_names', or by inspecting the instance __dict__. + """ + + def __repr__(self): + type_name = type(self).__name__ + arg_strings = [] + star_args = {} + for arg in self._get_args(): + arg_strings.append(repr(arg)) + for name, value in self._get_kwargs(): + if name.isidentifier(): + arg_strings.append('%s=%r' % (name, value)) + else: + star_args[name] = value + if star_args: + arg_strings.append('**%s' % repr(star_args)) + return '%s(%s)' % (type_name, ', '.join(arg_strings)) + + def _get_kwargs(self): + return list(self.__dict__.items()) + + def _get_args(self): + return [] + + +def _copy_items(items): + if items is None: + return [] + # The copy module is used only in the 'append' and 'append_const' + # actions, and it is needed only when the default value isn't a list. + # Delay its import for speeding up the common case. + if type(items) is list: + return items[:] + import copy + return copy.copy(items) + + +# =============== +# Formatting Help +# =============== + + +class HelpFormatter(object): + """Formatter for generating usage messages and argument help strings. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def __init__(self, + prog, + indent_increment=2, + max_help_position=24, + width=None): + + # default setting for width + if width is None: + import shutil + width = shutil.get_terminal_size().columns + width -= 2 + + self._prog = prog + self._indent_increment = indent_increment + self._max_help_position = min(max_help_position, + max(width - 20, indent_increment * 2)) + self._width = width + + self._current_indent = 0 + self._level = 0 + self._action_max_length = 0 + + self._root_section = self._Section(self, None) + self._current_section = self._root_section + + self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII) + self._long_break_matcher = _re.compile(r'\n\n\n+') + + # =============================== + # Section and indentation methods + # =============================== + def _indent(self): + self._current_indent += self._indent_increment + self._level += 1 + + def _dedent(self): + self._current_indent -= self._indent_increment + assert self._current_indent >= 0, 'Indent decreased below 0.' + self._level -= 1 + + class _Section(object): + + def __init__(self, formatter, parent, heading=None): + self.formatter = formatter + self.parent = parent + self.heading = heading + self.items = [] + + def format_help(self): + # format the indented section + if self.parent is not None: + self.formatter._indent() + join = self.formatter._join_parts + item_help = join([func(*args) for func, args in self.items]) + if self.parent is not None: + self.formatter._dedent() + + # return nothing if the section was empty + if not item_help: + return '' + + # add the heading if the section was non-empty + if self.heading is not SUPPRESS and self.heading is not None: + current_indent = self.formatter._current_indent + heading_text = _('%(heading)s:') % dict(heading=self.heading) + heading = '%*s%s\n' % (current_indent, '', heading_text) + else: + heading = '' + + # join the section-initial newline, the heading and the help + return join(['\n', heading, item_help, '\n']) + + def _add_item(self, func, args): + self._current_section.items.append((func, args)) + + # ======================== + # Message building methods + # ======================== + def start_section(self, heading): + self._indent() + section = self._Section(self, self._current_section, heading) + self._add_item(section.format_help, []) + self._current_section = section + + def end_section(self): + self._current_section = self._current_section.parent + self._dedent() + + def add_text(self, text): + if text is not SUPPRESS and text is not None: + self._add_item(self._format_text, [text]) + + def add_usage(self, usage, actions, groups, prefix=None): + if usage is not SUPPRESS: + args = usage, actions, groups, prefix + self._add_item(self._format_usage, args) + + def add_argument(self, action): + if action.help is not SUPPRESS: + + # find all invocations + get_invocation = self._format_action_invocation + invocations = [get_invocation(action)] + for subaction in self._iter_indented_subactions(action): + invocations.append(get_invocation(subaction)) + + # update the maximum item length + invocation_length = max(map(len, invocations)) + action_length = invocation_length + self._current_indent + self._action_max_length = max(self._action_max_length, + action_length) + + # add the item to the list + self._add_item(self._format_action, [action]) + + def add_arguments(self, actions): + for action in actions: + self.add_argument(action) + + # ======================= + # Help-formatting methods + # ======================= + def format_help(self): + help = self._root_section.format_help() + if help: + help = self._long_break_matcher.sub('\n\n', help) + help = help.strip('\n') + '\n' + return help + + def _join_parts(self, part_strings): + return ''.join([part + for part in part_strings + if part and part is not SUPPRESS]) + + def _format_usage(self, usage, actions, groups, prefix): + if prefix is None: + prefix = _('usage: ') + + # if usage is specified, use that + if usage is not None: + usage = usage % dict(prog=self._prog) + + # if no optionals or positionals are available, usage is just prog + elif usage is None and not actions: + usage = '%(prog)s' % dict(prog=self._prog) + + # if optionals and positionals are available, calculate usage + elif usage is None: + prog = '%(prog)s' % dict(prog=self._prog) + + # split optionals from positionals + optionals = [] + positionals = [] + for action in actions: + if action.option_strings: + optionals.append(action) + else: + positionals.append(action) + + # build full usage string + format = self._format_actions_usage + action_usage = format(optionals + positionals, groups) + usage = ' '.join([s for s in [prog, action_usage] if s]) + + # wrap the usage parts if it's too long + text_width = self._width - self._current_indent + if len(prefix) + len(usage) > text_width: + + # break usage into wrappable parts + part_regexp = ( + r'\(.*?\)+(?=\s|$)|' + r'\[.*?\]+(?=\s|$)|' + r'\S+' + ) + opt_usage = format(optionals, groups) + pos_usage = format(positionals, groups) + opt_parts = _re.findall(part_regexp, opt_usage) + pos_parts = _re.findall(part_regexp, pos_usage) + assert ' '.join(opt_parts) == opt_usage + assert ' '.join(pos_parts) == pos_usage + + # helper for wrapping lines + def get_lines(parts, indent, prefix=None): + lines = [] + line = [] + indent_length = len(indent) + if prefix is not None: + line_len = len(prefix) - 1 + else: + line_len = indent_length - 1 + for part in parts: + if line_len + 1 + len(part) > text_width and line: + lines.append(indent + ' '.join(line)) + line = [] + line_len = indent_length - 1 + line.append(part) + line_len += len(part) + 1 + if line: + lines.append(indent + ' '.join(line)) + if prefix is not None: + lines[0] = lines[0][indent_length:] + return lines + + # if prog is short, follow it with optionals or positionals + if len(prefix) + len(prog) <= 0.75 * text_width: + indent = ' ' * (len(prefix) + len(prog) + 1) + if opt_parts: + lines = get_lines([prog] + opt_parts, indent, prefix) + lines.extend(get_lines(pos_parts, indent)) + elif pos_parts: + lines = get_lines([prog] + pos_parts, indent, prefix) + else: + lines = [prog] + + # if prog is long, put it on its own line + else: + indent = ' ' * len(prefix) + parts = opt_parts + pos_parts + lines = get_lines(parts, indent) + if len(lines) > 1: + lines = [] + lines.extend(get_lines(opt_parts, indent)) + lines.extend(get_lines(pos_parts, indent)) + lines = [prog] + lines + + # join lines into usage + usage = '\n'.join(lines) + + # prefix with 'usage:' + return '%s%s\n\n' % (prefix, usage) + + def _format_actions_usage(self, actions, groups): + # find group indices and identify actions in groups + group_actions = set() + inserts = {} + for group in groups: + if not group._group_actions: + raise ValueError(f'empty group {group}') + + try: + start = actions.index(group._group_actions[0]) + except ValueError: + continue + else: + group_action_count = len(group._group_actions) + end = start + group_action_count + if actions[start:end] == group._group_actions: + + suppressed_actions_count = 0 + for action in group._group_actions: + group_actions.add(action) + if action.help is SUPPRESS: + suppressed_actions_count += 1 + + exposed_actions_count = group_action_count - suppressed_actions_count + if not exposed_actions_count: + continue + + if not group.required: + if start in inserts: + inserts[start] += ' [' + else: + inserts[start] = '[' + if end in inserts: + inserts[end] += ']' + else: + inserts[end] = ']' + elif exposed_actions_count > 1: + if start in inserts: + inserts[start] += ' (' + else: + inserts[start] = '(' + if end in inserts: + inserts[end] += ')' + else: + inserts[end] = ')' + for i in range(start + 1, end): + inserts[i] = '|' + + # collect all actions format strings + parts = [] + for i, action in enumerate(actions): + + # suppressed arguments are marked with None + # remove | separators for suppressed arguments + if action.help is SUPPRESS: + parts.append(None) + if inserts.get(i) == '|': + inserts.pop(i) + elif inserts.get(i + 1) == '|': + inserts.pop(i + 1) + + # produce all arg strings + elif not action.option_strings: + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) + + # if it's in a group, strip the outer [] + if action in group_actions: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + + # add the action string to the list + parts.append(part) + + # produce the first way to invoke the option in brackets + else: + option_string = action.option_strings[0] + + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = action.format_usage() + + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + part = '%s %s' % (option_string, args_string) + + # make it look optional if it's not required or in a group + if not action.required and action not in group_actions: + part = '[%s]' % part + + # add the action string to the list + parts.append(part) + + # insert things at the necessary indices + for i in sorted(inserts, reverse=True): + parts[i:i] = [inserts[i]] + + # join all the action items with spaces + text = ' '.join([item for item in parts if item is not None]) + + # clean up separators for mutually exclusive groups + open = r'[\[(]' + close = r'[\])]' + text = _re.sub(r'(%s) ' % open, r'\1', text) + text = _re.sub(r' (%s)' % close, r'\1', text) + text = _re.sub(r'%s *%s' % (open, close), r'', text) + text = text.strip() + + # return the text + return text + + def _format_text(self, text): + if '%(prog)' in text: + text = text % dict(prog=self._prog) + text_width = max(self._width - self._current_indent, 11) + indent = ' ' * self._current_indent + return self._fill_text(text, text_width, indent) + '\n\n' + + def _format_action(self, action): + # determine the required width and the entry label + help_position = min(self._action_max_length + 2, + self._max_help_position) + help_width = max(self._width - help_position, 11) + action_width = help_position - self._current_indent - 2 + action_header = self._format_action_invocation(action) + + # no help; start on same line and add a final newline + if not action.help: + tup = self._current_indent, '', action_header + action_header = '%*s%s\n' % tup + + # short action name; start on the same line and pad two spaces + elif len(action_header) <= action_width: + tup = self._current_indent, '', action_width, action_header + action_header = '%*s%-*s ' % tup + indent_first = 0 + + # long action name; start on the next line + else: + tup = self._current_indent, '', action_header + action_header = '%*s%s\n' % tup + indent_first = help_position + + # collect the pieces of the action help + parts = [action_header] + + # if there was help for the action, add lines of help text + if action.help and action.help.strip(): + help_text = self._expand_help(action) + if help_text: + help_lines = self._split_lines(help_text, help_width) + parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) + for line in help_lines[1:]: + parts.append('%*s%s\n' % (help_position, '', line)) + + # or add a newline if the description doesn't end with one + elif not action_header.endswith('\n'): + parts.append('\n') + + # if there are any sub-actions, add their help as well + for subaction in self._iter_indented_subactions(action): + parts.append(self._format_action(subaction)) + + # return a single string + return self._join_parts(parts) + + def _format_action_invocation(self, action): + if not action.option_strings: + default = self._get_default_metavar_for_positional(action) + metavar, = self._metavar_formatter(action, default)(1) + return metavar + + else: + parts = [] + + # if the Optional doesn't take a value, format is: + # -s, --long + if action.nargs == 0: + parts.extend(action.option_strings) + + # if the Optional takes a value, format is: + # -s ARGS, --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + for option_string in action.option_strings: + parts.append('%s %s' % (option_string, args_string)) + + return ', '.join(parts) + + def _metavar_formatter(self, action, default_metavar): + if action.metavar is not None: + result = action.metavar + elif action.choices is not None: + choice_strs = [str(choice) for choice in action.choices] + result = '{%s}' % ','.join(choice_strs) + else: + result = default_metavar + + def format(tuple_size): + if isinstance(result, tuple): + return result + else: + return (result, ) * tuple_size + return format + + def _format_args(self, action, default_metavar): + get_metavar = self._metavar_formatter(action, default_metavar) + if action.nargs is None: + result = '%s' % get_metavar(1) + elif action.nargs == OPTIONAL: + result = '[%s]' % get_metavar(1) + elif action.nargs == ZERO_OR_MORE: + metavar = get_metavar(1) + if len(metavar) == 2: + result = '[%s [%s ...]]' % metavar + else: + result = '[%s ...]' % metavar + elif action.nargs == ONE_OR_MORE: + result = '%s [%s ...]' % get_metavar(2) + elif action.nargs == REMAINDER: + result = '...' + elif action.nargs == PARSER: + result = '%s ...' % get_metavar(1) + elif action.nargs == SUPPRESS: + result = '' + else: + try: + formats = ['%s' for _ in range(action.nargs)] + except TypeError: + raise ValueError("invalid nargs value") from None + result = ' '.join(formats) % get_metavar(action.nargs) + return result + + def _expand_help(self, action): + params = dict(vars(action), prog=self._prog) + for name in list(params): + if params[name] is SUPPRESS: + del params[name] + for name in list(params): + if hasattr(params[name], '__name__'): + params[name] = params[name].__name__ + if params.get('choices') is not None: + choices_str = ', '.join([str(c) for c in params['choices']]) + params['choices'] = choices_str + return self._get_help_string(action) % params + + def _iter_indented_subactions(self, action): + try: + get_subactions = action._get_subactions + except AttributeError: + pass + else: + self._indent() + yield from get_subactions() + self._dedent() + + def _split_lines(self, text, width): + text = self._whitespace_matcher.sub(' ', text).strip() + # The textwrap module is used only for formatting help. + # Delay its import for speeding up the common usage of argparse. + import textwrap + return textwrap.wrap(text, width) + + def _fill_text(self, text, width, indent): + text = self._whitespace_matcher.sub(' ', text).strip() + import textwrap + return textwrap.fill(text, width, + initial_indent=indent, + subsequent_indent=indent) + + def _get_help_string(self, action): + return action.help + + def _get_default_metavar_for_optional(self, action): + return action.dest.upper() + + def _get_default_metavar_for_positional(self, action): + return action.dest + + +class RawDescriptionHelpFormatter(HelpFormatter): + """Help message formatter which retains any formatting in descriptions. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _fill_text(self, text, width, indent): + return ''.join(indent + line for line in text.splitlines(keepends=True)) + + +class RawTextHelpFormatter(RawDescriptionHelpFormatter): + """Help message formatter which retains formatting of all help text. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _split_lines(self, text, width): + return text.splitlines() + + +class ArgumentDefaultsHelpFormatter(HelpFormatter): + """Help message formatter which adds default values to argument help. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _get_help_string(self, action): + """ + Add the default value to the option help message. + + ArgumentDefaultsHelpFormatter and BooleanOptionalAction when it isn't + already present. This code will do that, detecting cornercases to + prevent duplicates or cases where it wouldn't make sense to the end + user. + """ + help = action.help + if help is None: + help = '' + + if '%(default)' not in help: + if action.default is not SUPPRESS: + defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] + if action.option_strings or action.nargs in defaulting_nargs: + help += _(' (default: %(default)s)') + return help + + + +class MetavarTypeHelpFormatter(HelpFormatter): + """Help message formatter which uses the argument 'type' as the default + metavar value (instead of the argument 'dest') + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _get_default_metavar_for_optional(self, action): + return action.type.__name__ + + def _get_default_metavar_for_positional(self, action): + return action.type.__name__ + + +# ===================== +# Options and Arguments +# ===================== + +def _get_action_name(argument): + if argument is None: + return None + elif argument.option_strings: + return '/'.join(argument.option_strings) + elif argument.metavar not in (None, SUPPRESS): + return argument.metavar + elif argument.dest not in (None, SUPPRESS): + return argument.dest + elif argument.choices: + return '{' + ','.join(argument.choices) + '}' + else: + return None + + +# Use exceptions from system-level argparse module. +from argparse import ArgumentError, ArgumentTypeError + +# class ArgumentError(Exception): +# """An error from creating or using an argument (optional or positional). +# +# The string value of this exception is the message, augmented with +# information about the argument that caused it. +# """ +# +# def __init__(self, argument, message): +# self.argument_name = _get_action_name(argument) +# self.message = message +# +# def __str__(self): +# if self.argument_name is None: +# format = '%(message)s' +# else: +# format = _('argument %(argument_name)s: %(message)s') +# return format % dict(message=self.message, +# argument_name=self.argument_name) +# +# +# class ArgumentTypeError(Exception): +# """An error from trying to convert a command line string to a type.""" +# pass + + +# ============== +# Action classes +# ============== + +class Action(_AttributeHolder): + """Information about how to convert command line strings to Python objects. + + Action objects are used by an ArgumentParser to represent the information + needed to parse a single argument from one or more strings from the + command line. The keyword arguments to the Action constructor are also + all attributes of Action instances. + + Keyword Arguments: + + - option_strings -- A list of command-line option strings which + should be associated with this action. + + - dest -- The name of the attribute to hold the created object(s) + + - nargs -- The number of command-line arguments that should be + consumed. By default, one argument will be consumed and a single + value will be produced. Other values include: + - N (an integer) consumes N arguments (and produces a list) + - '?' consumes zero or one arguments + - '*' consumes zero or more arguments (and produces a list) + - '+' consumes one or more arguments (and produces a list) + Note that the difference between the default and nargs=1 is that + with the default, a single value will be produced, while with + nargs=1, a list containing a single value will be produced. + + - const -- The value to be produced if the option is specified and the + option uses an action that takes no values. + + - default -- The value to be produced if the option is not specified. + + - type -- A callable that accepts a single string argument, and + returns the converted value. The standard Python types str, int, + float, and complex are useful examples of such callables. If None, + str is used. + + - choices -- A container of values that should be allowed. If not None, + after a command-line argument has been converted to the appropriate + type, an exception will be raised if it is not a member of this + collection. + + - required -- True if the action must always be specified at the + command line. This is only meaningful for optional command-line + arguments. + + - help -- The help string describing the argument. + + - metavar -- The name to be used for the option's argument with the + help string. If None, the 'dest' value will be used as the name. + """ + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + self.option_strings = option_strings + self.dest = dest + self.nargs = nargs + self.const = const + self.default = default + self.type = type + self.choices = choices + self.required = required + self.help = help + self.metavar = metavar + + def _get_kwargs(self): + names = [ + 'option_strings', + 'dest', + 'nargs', + 'const', + 'default', + 'type', + 'choices', + 'required', + 'help', + 'metavar', + ] + return [(name, getattr(self, name)) for name in names] + + def format_usage(self): + return self.option_strings[0] + + def __call__(self, parser, namespace, values, option_string=None): + raise NotImplementedError(_('.__call__() not defined')) + + +# FIXME: remove together with `BooleanOptionalAction` deprecated arguments. +_deprecated_default = object() + +class BooleanOptionalAction(Action): + def __init__(self, + option_strings, + dest, + default=None, + type=_deprecated_default, + choices=_deprecated_default, + required=False, + help=None, + metavar=_deprecated_default): + + _option_strings = [] + for option_string in option_strings: + _option_strings.append(option_string) + + if option_string.startswith('--'): + option_string = '--no-' + option_string[2:] + _option_strings.append(option_string) + + # We need `_deprecated` special value to ban explicit arguments that + # match default value. Like: + # parser.add_argument('-f', action=BooleanOptionalAction, type=int) + for field_name in ('type', 'choices', 'metavar'): + if locals()[field_name] is not _deprecated_default: + warnings._deprecated( + field_name, + "{name!r} is deprecated as of Python 3.12 and will be " + "removed in Python {remove}.", + remove=(3, 14)) + + if type is _deprecated_default: + type = None + if choices is _deprecated_default: + choices = None + if metavar is _deprecated_default: + metavar = None + + super().__init__( + option_strings=_option_strings, + dest=dest, + nargs=0, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar) + + + def __call__(self, parser, namespace, values, option_string=None): + if option_string in self.option_strings: + setattr(namespace, self.dest, not option_string.startswith('--no-')) + + def format_usage(self): + return ' | '.join(self.option_strings) + + +class _StoreAction(Action): + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + if nargs == 0: + raise ValueError('nargs for store actions must be != 0; if you ' + 'have nothing to store, actions such as store ' + 'true or store const may be more appropriate') + if const is not None and nargs != OPTIONAL: + raise ValueError('nargs must be %r to supply const' % OPTIONAL) + super(_StoreAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + const=const, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values) + + +class _StoreConstAction(Action): + + def __init__(self, + option_strings, + dest, + const=None, + default=None, + required=False, + help=None, + metavar=None): + super(_StoreConstAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, self.const) + + +class _StoreTrueAction(_StoreConstAction): + + def __init__(self, + option_strings, + dest, + default=False, + required=False, + help=None): + super(_StoreTrueAction, self).__init__( + option_strings=option_strings, + dest=dest, + const=True, + default=default, + required=required, + help=help) + + +class _StoreFalseAction(_StoreConstAction): + + def __init__(self, + option_strings, + dest, + default=True, + required=False, + help=None): + super(_StoreFalseAction, self).__init__( + option_strings=option_strings, + dest=dest, + const=False, + default=default, + required=required, + help=help) + + +class _AppendAction(Action): + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + if nargs == 0: + raise ValueError('nargs for append actions must be != 0; if arg ' + 'strings are not supplying the value to append, ' + 'the append const action may be more appropriate') + if const is not None and nargs != OPTIONAL: + raise ValueError('nargs must be %r to supply const' % OPTIONAL) + super(_AppendAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + const=const, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + items = getattr(namespace, self.dest, None) + items = _copy_items(items) + items.append(values) + setattr(namespace, self.dest, items) + + +class _AppendConstAction(Action): + + def __init__(self, + option_strings, + dest, + const=None, + default=None, + required=False, + help=None, + metavar=None): + super(_AppendConstAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + items = getattr(namespace, self.dest, None) + items = _copy_items(items) + items.append(self.const) + setattr(namespace, self.dest, items) + + +class _CountAction(Action): + + def __init__(self, + option_strings, + dest, + default=None, + required=False, + help=None): + super(_CountAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + default=default, + required=required, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + count = getattr(namespace, self.dest, None) + if count is None: + count = 0 + setattr(namespace, self.dest, count + 1) + + +class _HelpAction(Action): + + def __init__(self, + option_strings, + dest=SUPPRESS, + default=SUPPRESS, + help=None): + super(_HelpAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + parser.print_help() + parser.exit() + + +class _VersionAction(Action): + + def __init__(self, + option_strings, + version=None, + dest=SUPPRESS, + default=SUPPRESS, + help=None): + if help is None: + help = _("show program's version number and exit") + super(_VersionAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + self.version = version + + def __call__(self, parser, namespace, values, option_string=None): + version = self.version + if version is None: + version = parser.version + formatter = parser._get_formatter() + formatter.add_text(version) + parser._print_message(formatter.format_help(), _sys.stdout) + parser.exit() + + +class _SubParsersAction(Action): + + class _ChoicesPseudoAction(Action): + + def __init__(self, name, aliases, help): + metavar = dest = name + if aliases: + metavar += ' (%s)' % ', '.join(aliases) + sup = super(_SubParsersAction._ChoicesPseudoAction, self) + sup.__init__(option_strings=[], dest=dest, help=help, + metavar=metavar) + + def __init__(self, + option_strings, + prog, + parser_class, + dest=SUPPRESS, + required=False, + help=None, + metavar=None): + + self._prog_prefix = prog + self._parser_class = parser_class + self._name_parser_map = {} + self._choices_actions = [] + + super(_SubParsersAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=PARSER, + choices=self._name_parser_map, + required=required, + help=help, + metavar=metavar) + + def add_parser(self, name, **kwargs): + # set prog from the existing prefix + if kwargs.get('prog') is None: + kwargs['prog'] = '%s %s' % (self._prog_prefix, name) + + aliases = kwargs.pop('aliases', ()) + + if name in self._name_parser_map: + raise ArgumentError(self, _('conflicting subparser: %s') % name) + for alias in aliases: + if alias in self._name_parser_map: + raise ArgumentError( + self, _('conflicting subparser alias: %s') % alias) + + # create a pseudo-action to hold the choice help + if 'help' in kwargs: + help = kwargs.pop('help') + choice_action = self._ChoicesPseudoAction(name, aliases, help) + self._choices_actions.append(choice_action) + + # create the parser and add it to the map + parser = self._parser_class(**kwargs) + self._name_parser_map[name] = parser + + # make parser available under aliases also + for alias in aliases: + self._name_parser_map[alias] = parser + + return parser + + def _get_subactions(self): + return self._choices_actions + + def __call__(self, parser, namespace, values, option_string=None): + parser_name = values[0] + arg_strings = values[1:] + + # set the parser name if requested + if self.dest is not SUPPRESS: + setattr(namespace, self.dest, parser_name) + + # select the parser + try: + parser = self._name_parser_map[parser_name] + except KeyError: + args = {'parser_name': parser_name, + 'choices': ', '.join(self._name_parser_map)} + msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args + raise ArgumentError(self, msg) + + # parse all the remaining options into the namespace + # store any unrecognized options on the object, so that the top + # level parser can decide what to do with them + + # In case this subparser defines new defaults, we parse them + # in a new namespace object and then update the original + # namespace for the relevant parts. + subnamespace, arg_strings = parser.parse_known_args(arg_strings, None) + for key, value in vars(subnamespace).items(): + setattr(namespace, key, value) + + if arg_strings: + vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) + getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) + +class _ExtendAction(_AppendAction): + def __call__(self, parser, namespace, values, option_string=None): + items = getattr(namespace, self.dest, None) + items = _copy_items(items) + items.extend(values) + setattr(namespace, self.dest, items) + +# ============== +# Type classes +# ============== + +class FileType(object): + """Factory for creating file object types + + Instances of FileType are typically passed as type= arguments to the + ArgumentParser add_argument() method. + + Keyword Arguments: + - mode -- A string indicating how the file is to be opened. Accepts the + same values as the builtin open() function. + - bufsize -- The file's desired buffer size. Accepts the same values as + the builtin open() function. + - encoding -- The file's encoding. Accepts the same values as the + builtin open() function. + - errors -- A string indicating how encoding and decoding errors are to + be handled. Accepts the same value as the builtin open() function. + """ + + def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None): + self._mode = mode + self._bufsize = bufsize + self._encoding = encoding + self._errors = errors + + def __call__(self, string): + # the special argument "-" means sys.std{in,out} + if string == '-': + if 'r' in self._mode: + return _sys.stdin.buffer if 'b' in self._mode else _sys.stdin + elif any(c in self._mode for c in 'wax'): + return _sys.stdout.buffer if 'b' in self._mode else _sys.stdout + else: + msg = _('argument "-" with mode %r') % self._mode + raise ValueError(msg) + + # all other arguments are used as file names + try: + return open(string, self._mode, self._bufsize, self._encoding, + self._errors) + except OSError as e: + args = {'filename': string, 'error': e} + message = _("can't open '%(filename)s': %(error)s") + raise ArgumentTypeError(message % args) + + def __repr__(self): + args = self._mode, self._bufsize + kwargs = [('encoding', self._encoding), ('errors', self._errors)] + args_str = ', '.join([repr(arg) for arg in args if arg != -1] + + ['%s=%r' % (kw, arg) for kw, arg in kwargs + if arg is not None]) + return '%s(%s)' % (type(self).__name__, args_str) + +# =========================== +# Optional and Positional Parsing +# =========================== + +class Namespace(_AttributeHolder): + """Simple object for storing attributes. + + Implements equality by attribute names and values, and provides a simple + string representation. + """ + + def __init__(self, **kwargs): + for name in kwargs: + setattr(self, name, kwargs[name]) + + def __eq__(self, other): + if not isinstance(other, Namespace): + return NotImplemented + return vars(self) == vars(other) + + def __contains__(self, key): + return key in self.__dict__ + + +class _ActionsContainer(object): + + def __init__(self, + description, + prefix_chars, + argument_default, + conflict_handler): + super(_ActionsContainer, self).__init__() + + self.description = description + self.argument_default = argument_default + self.prefix_chars = prefix_chars + self.conflict_handler = conflict_handler + + # set up registries + self._registries = {} + + # register actions + self.register('action', None, _StoreAction) + self.register('action', 'store', _StoreAction) + self.register('action', 'store_const', _StoreConstAction) + self.register('action', 'store_true', _StoreTrueAction) + self.register('action', 'store_false', _StoreFalseAction) + self.register('action', 'append', _AppendAction) + self.register('action', 'append_const', _AppendConstAction) + self.register('action', 'count', _CountAction) + self.register('action', 'help', _HelpAction) + self.register('action', 'version', _VersionAction) + self.register('action', 'parsers', _SubParsersAction) + self.register('action', 'extend', _ExtendAction) + + # raise an exception if the conflict handler is invalid + self._get_handler() + + # action storage + self._actions = [] + self._option_string_actions = {} + + # groups + self._action_groups = [] + self._mutually_exclusive_groups = [] + + # defaults storage + self._defaults = {} + + # determines whether an "option" looks like a negative number + self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') + + # whether or not there are any optionals that look like negative + # numbers -- uses a list so it can be shared and edited + self._has_negative_number_optionals = [] + + # ==================== + # Registration methods + # ==================== + def register(self, registry_name, value, object): + registry = self._registries.setdefault(registry_name, {}) + registry[value] = object + + def _registry_get(self, registry_name, value, default=None): + return self._registries[registry_name].get(value, default) + + # ================================== + # Namespace default accessor methods + # ================================== + def set_defaults(self, **kwargs): + self._defaults.update(kwargs) + + # if these defaults match any existing arguments, replace + # the previous default on the object with the new one + for action in self._actions: + if action.dest in kwargs: + action.default = kwargs[action.dest] + + def get_default(self, dest): + for action in self._actions: + if action.dest == dest and action.default is not None: + return action.default + return self._defaults.get(dest, None) + + + # ======================= + # Adding argument actions + # ======================= + def add_argument(self, *args, **kwargs): + """ + add_argument(dest, ..., name=value, ...) + add_argument(option_string, option_string, ..., name=value, ...) + """ + + # if no positional args are supplied or only one is supplied and + # it doesn't look like an option string, parse a positional + # argument + chars = self.prefix_chars + if not args or len(args) == 1 and args[0][0] not in chars: + if args and 'dest' in kwargs: + raise ValueError('dest supplied twice for positional argument') + kwargs = self._get_positional_kwargs(*args, **kwargs) + + # otherwise, we're adding an optional argument + else: + kwargs = self._get_optional_kwargs(*args, **kwargs) + + # if no default was supplied, use the parser-level default + if 'default' not in kwargs: + dest = kwargs['dest'] + if dest in self._defaults: + kwargs['default'] = self._defaults[dest] + elif self.argument_default is not None: + kwargs['default'] = self.argument_default + + # create the action object, and add it to the parser + action_class = self._pop_action_class(kwargs) + if not callable(action_class): + raise ValueError('unknown action "%s"' % (action_class,)) + action = action_class(**kwargs) + + # raise an error if the action type is not callable + type_func = self._registry_get('type', action.type, action.type) + if not callable(type_func): + raise ValueError('%r is not callable' % (type_func,)) + + if type_func is FileType: + raise ValueError('%r is a FileType class object, instance of it' + ' must be passed' % (type_func,)) + + # raise an error if the metavar does not match the type + if hasattr(self, "_get_formatter"): + try: + self._get_formatter()._format_args(action, None) + except TypeError: + raise ValueError("length of metavar tuple does not match nargs") + + return self._add_action(action) + + def add_argument_group(self, *args, **kwargs): + group = _ArgumentGroup(self, *args, **kwargs) + self._action_groups.append(group) + return group + + def add_mutually_exclusive_group(self, **kwargs): + group = _MutuallyExclusiveGroup(self, **kwargs) + self._mutually_exclusive_groups.append(group) + return group + + def _add_action(self, action): + # resolve any conflicts + self._check_conflict(action) + + # add to actions list + self._actions.append(action) + action.container = self + + # index the action by any option strings it has + for option_string in action.option_strings: + self._option_string_actions[option_string] = action + + # set the flag if any option strings look like negative numbers + for option_string in action.option_strings: + if self._negative_number_matcher.match(option_string): + if not self._has_negative_number_optionals: + self._has_negative_number_optionals.append(True) + + # return the created action + return action + + def _remove_action(self, action): + self._actions.remove(action) + + def _add_container_actions(self, container): + # collect groups by titles + title_group_map = {} + for group in self._action_groups: + if group.title in title_group_map: + msg = _('cannot merge actions - two groups are named %r') + raise ValueError(msg % (group.title)) + title_group_map[group.title] = group + + # map each action to its group + group_map = {} + for group in container._action_groups: + + # if a group with the title exists, use that, otherwise + # create a new group matching the container's group + if group.title not in title_group_map: + title_group_map[group.title] = self.add_argument_group( + title=group.title, + description=group.description, + conflict_handler=group.conflict_handler) + + # map the actions to their new group + for action in group._group_actions: + group_map[action] = title_group_map[group.title] + + # add container's mutually exclusive groups + # NOTE: if add_mutually_exclusive_group ever gains title= and + # description= then this code will need to be expanded as above + for group in container._mutually_exclusive_groups: + mutex_group = self.add_mutually_exclusive_group( + required=group.required) + + # map the actions to their new mutex group + for action in group._group_actions: + group_map[action] = mutex_group + + # add all actions to this container or their group + for action in container._actions: + group_map.get(action, self)._add_action(action) + + def _get_positional_kwargs(self, dest, **kwargs): + # make sure required is not specified + if 'required' in kwargs: + msg = _("'required' is an invalid argument for positionals") + raise TypeError(msg) + + # mark positional arguments as required if at least one is + # always required + if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: + kwargs['required'] = True + if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: + kwargs['required'] = True + + # return the keyword arguments with no option strings + return dict(kwargs, dest=dest, option_strings=[]) + + def _get_optional_kwargs(self, *args, **kwargs): + # determine short and long option strings + option_strings = [] + long_option_strings = [] + for option_string in args: + # error on strings that don't start with an appropriate prefix + if not option_string[0] in self.prefix_chars: + args = {'option': option_string, + 'prefix_chars': self.prefix_chars} + msg = _('invalid option string %(option)r: ' + 'must start with a character %(prefix_chars)r') + raise ValueError(msg % args) + + # strings starting with two prefix characters are long options + option_strings.append(option_string) + if len(option_string) > 1 and option_string[1] in self.prefix_chars: + long_option_strings.append(option_string) + + # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' + dest = kwargs.pop('dest', None) + if dest is None: + if long_option_strings: + dest_option_string = long_option_strings[0] + else: + dest_option_string = option_strings[0] + dest = dest_option_string.lstrip(self.prefix_chars) + if not dest: + msg = _('dest= is required for options like %r') + raise ValueError(msg % option_string) + dest = dest.replace('-', '_') + + # return the updated keyword arguments + return dict(kwargs, dest=dest, option_strings=option_strings) + + def _pop_action_class(self, kwargs, default=None): + action = kwargs.pop('action', default) + return self._registry_get('action', action, action) + + def _get_handler(self): + # determine function from conflict handler string + handler_func_name = '_handle_conflict_%s' % self.conflict_handler + try: + return getattr(self, handler_func_name) + except AttributeError: + msg = _('invalid conflict_resolution value: %r') + raise ValueError(msg % self.conflict_handler) + + def _check_conflict(self, action): + + # find all options that conflict with this option + confl_optionals = [] + for option_string in action.option_strings: + if option_string in self._option_string_actions: + confl_optional = self._option_string_actions[option_string] + confl_optionals.append((option_string, confl_optional)) + + # resolve any conflicts + if confl_optionals: + conflict_handler = self._get_handler() + conflict_handler(action, confl_optionals) + + def _handle_conflict_error(self, action, conflicting_actions): + message = ngettext('conflicting option string: %s', + 'conflicting option strings: %s', + len(conflicting_actions)) + conflict_string = ', '.join([option_string + for option_string, action + in conflicting_actions]) + raise ArgumentError(action, message % conflict_string) + + def _handle_conflict_resolve(self, action, conflicting_actions): + + # remove all conflicting options + for option_string, action in conflicting_actions: + + # remove the conflicting option + action.option_strings.remove(option_string) + self._option_string_actions.pop(option_string, None) + + # if the option now has no option string, remove it from the + # container holding it + if not action.option_strings: + action.container._remove_action(action) + + +class _ArgumentGroup(_ActionsContainer): + + def __init__(self, container, title=None, description=None, **kwargs): + # add any missing keyword arguments by checking the container + update = kwargs.setdefault + update('conflict_handler', container.conflict_handler) + update('prefix_chars', container.prefix_chars) + update('argument_default', container.argument_default) + super_init = super(_ArgumentGroup, self).__init__ + super_init(description=description, **kwargs) + + # group attributes + self.title = title + self._group_actions = [] + + # share most attributes with the container + self._registries = container._registries + self._actions = container._actions + self._option_string_actions = container._option_string_actions + self._defaults = container._defaults + self._has_negative_number_optionals = \ + container._has_negative_number_optionals + self._mutually_exclusive_groups = container._mutually_exclusive_groups + + def _add_action(self, action): + action = super(_ArgumentGroup, self)._add_action(action) + self._group_actions.append(action) + return action + + def _remove_action(self, action): + super(_ArgumentGroup, self)._remove_action(action) + self._group_actions.remove(action) + + def add_argument_group(self, *args, **kwargs): + warnings.warn( + "Nesting argument groups is deprecated.", + category=DeprecationWarning, + stacklevel=2 + ) + return super().add_argument_group(*args, **kwargs) + + +class _MutuallyExclusiveGroup(_ArgumentGroup): + + def __init__(self, container, required=False): + super(_MutuallyExclusiveGroup, self).__init__(container) + self.required = required + self._container = container + + def _add_action(self, action): + if action.required: + msg = _('mutually exclusive arguments must be optional') + raise ValueError(msg) + action = self._container._add_action(action) + self._group_actions.append(action) + return action + + def _remove_action(self, action): + self._container._remove_action(action) + self._group_actions.remove(action) + + def add_mutually_exclusive_group(self, *args, **kwargs): + warnings.warn( + "Nesting mutually exclusive groups is deprecated.", + category=DeprecationWarning, + stacklevel=2 + ) + return super().add_mutually_exclusive_group(*args, **kwargs) + + +class ArgumentParser(_AttributeHolder, _ActionsContainer): + """Object for parsing command line strings into Python objects. + + Keyword Arguments: + - prog -- The name of the program (default: + ``os.path.basename(sys.argv[0])``) + - usage -- A usage message (default: auto-generated from arguments) + - description -- A description of what the program does + - epilog -- Text following the argument descriptions + - parents -- Parsers whose arguments should be copied into this one + - formatter_class -- HelpFormatter class for printing help messages + - prefix_chars -- Characters that prefix optional arguments + - fromfile_prefix_chars -- Characters that prefix files containing + additional arguments + - argument_default -- The default value for all arguments + - conflict_handler -- String indicating how to handle conflicts + - add_help -- Add a -h/-help option + - allow_abbrev -- Allow long options to be abbreviated unambiguously + - exit_on_error -- Determines whether or not ArgumentParser exits with + error info when an error occurs + """ + + def __init__(self, + prog=None, + usage=None, + description=None, + epilog=None, + parents=[], + formatter_class=HelpFormatter, + prefix_chars='-', + fromfile_prefix_chars=None, + argument_default=None, + conflict_handler='error', + add_help=True, + allow_abbrev=True, + exit_on_error=True): + + superinit = super(ArgumentParser, self).__init__ + superinit(description=description, + prefix_chars=prefix_chars, + argument_default=argument_default, + conflict_handler=conflict_handler) + + # default setting for prog + if prog is None: + prog = _os.path.basename(_sys.argv[0]) + + self.prog = prog + self.usage = usage + self.epilog = epilog + self.formatter_class = formatter_class + self.fromfile_prefix_chars = fromfile_prefix_chars + self.add_help = add_help + self.allow_abbrev = allow_abbrev + self.exit_on_error = exit_on_error + + add_group = self.add_argument_group + self._positionals = add_group(_('positional arguments')) + self._optionals = add_group(_('options')) + self._subparsers = None + + # register types + def identity(string): + return string + self.register('type', None, identity) + + # add help argument if necessary + # (using explicit default to override global argument_default) + default_prefix = '-' if '-' in prefix_chars else prefix_chars[0] + if self.add_help: + self.add_argument( + default_prefix+'h', default_prefix*2+'help', + action='help', default=SUPPRESS, + help=_('show this help message and exit')) + + # add parent arguments and defaults + for parent in parents: + self._add_container_actions(parent) + try: + defaults = parent._defaults + except AttributeError: + pass + else: + self._defaults.update(defaults) + + # ======================= + # Pretty __repr__ methods + # ======================= + def _get_kwargs(self): + names = [ + 'prog', + 'usage', + 'description', + 'formatter_class', + 'conflict_handler', + 'add_help', + ] + return [(name, getattr(self, name)) for name in names] + + # ================================== + # Optional/Positional adding methods + # ================================== + def add_subparsers(self, **kwargs): + if self._subparsers is not None: + self.error(_('cannot have multiple subparser arguments')) + + # add the parser class to the arguments if it's not present + kwargs.setdefault('parser_class', type(self)) + + if 'title' in kwargs or 'description' in kwargs: + title = _(kwargs.pop('title', 'subcommands')) + description = _(kwargs.pop('description', None)) + self._subparsers = self.add_argument_group(title, description) + else: + self._subparsers = self._positionals + + # prog defaults to the usage message of this parser, skipping + # optional arguments and with no "usage:" prefix + if kwargs.get('prog') is None: + formatter = self._get_formatter() + positionals = self._get_positional_actions() + groups = self._mutually_exclusive_groups + formatter.add_usage(self.usage, positionals, groups, '') + kwargs['prog'] = formatter.format_help().strip() + + # create the parsers action and add it to the positionals list + parsers_class = self._pop_action_class(kwargs, 'parsers') + action = parsers_class(option_strings=[], **kwargs) + self._subparsers._add_action(action) + + # return the created parsers action + return action + + def _add_action(self, action): + if action.option_strings: + self._optionals._add_action(action) + else: + self._positionals._add_action(action) + return action + + def _get_optional_actions(self): + return [action + for action in self._actions + if action.option_strings] + + def _get_positional_actions(self): + return [action + for action in self._actions + if not action.option_strings] + + # ===================================== + # Command line argument parsing methods + # ===================================== + def parse_args(self, args=None, namespace=None): + args, argv = self.parse_known_args(args, namespace) + if argv: + msg = _('unrecognized arguments: %s') + self.error(msg % ' '.join(argv)) + return args + + def parse_known_args(self, args=None, namespace=None): + if args is None: + # args default to the system args + args = _sys.argv[1:] + else: + # make sure that args are mutable + args = list(args) + + # default Namespace built from parser defaults + if namespace is None: + namespace = Namespace() + + # add any action defaults that aren't present + for action in self._actions: + if action.dest is not SUPPRESS: + if not hasattr(namespace, action.dest): + if action.default is not SUPPRESS: + setattr(namespace, action.dest, action.default) + + # add any parser defaults that aren't present + for dest in self._defaults: + if not hasattr(namespace, dest): + setattr(namespace, dest, self._defaults[dest]) + + # parse the arguments and exit if there are any errors + if self.exit_on_error: + try: + namespace, args = self._parse_known_args(args, namespace) + except ArgumentError as err: + self.error(str(err)) + else: + namespace, args = self._parse_known_args(args, namespace) + + if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): + args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) + delattr(namespace, _UNRECOGNIZED_ARGS_ATTR) + return namespace, args + + def _parse_known_args(self, arg_strings, namespace): + # replace arg strings that are file references + if self.fromfile_prefix_chars is not None: + arg_strings = self._read_args_from_files(arg_strings) + + # map all mutually exclusive arguments to the other arguments + # they can't occur with + action_conflicts = {} + for mutex_group in self._mutually_exclusive_groups: + group_actions = mutex_group._group_actions + for i, mutex_action in enumerate(mutex_group._group_actions): + conflicts = action_conflicts.setdefault(mutex_action, []) + conflicts.extend(group_actions[:i]) + conflicts.extend(group_actions[i + 1:]) + + # find all option indices, and determine the arg_string_pattern + # which has an 'O' if there is an option at an index, + # an 'A' if there is an argument, or a '-' if there is a '--' + option_string_indices = {} + arg_string_pattern_parts = [] + arg_strings_iter = iter(arg_strings) + for i, arg_string in enumerate(arg_strings_iter): + + # all args after -- are non-options + if arg_string == '--': + arg_string_pattern_parts.append('-') + for arg_string in arg_strings_iter: + arg_string_pattern_parts.append('A') + + # otherwise, add the arg to the arg strings + # and note the index if it was an option + else: + option_tuple = self._parse_optional(arg_string) + if option_tuple is None: + pattern = 'A' + else: + option_string_indices[i] = option_tuple + pattern = 'O' + arg_string_pattern_parts.append(pattern) + + # join the pieces together to form the pattern + arg_strings_pattern = ''.join(arg_string_pattern_parts) + + # converts arg strings to the appropriate and then takes the action + seen_actions = set() + seen_non_default_actions = set() + + def take_action(action, argument_strings, option_string=None): + seen_actions.add(action) + argument_values = self._get_values(action, argument_strings) + + # error if this argument is not allowed with other previously + # seen arguments, assuming that actions that use the default + # value don't really count as "present" + if argument_values is not action.default: + seen_non_default_actions.add(action) + for conflict_action in action_conflicts.get(action, []): + if conflict_action in seen_non_default_actions: + msg = _('not allowed with argument %s') + action_name = _get_action_name(conflict_action) + raise ArgumentError(action, msg % action_name) + + # take the action if we didn't receive a SUPPRESS value + # (e.g. from a default) + if argument_values is not SUPPRESS: + action(self, namespace, argument_values, option_string) + + # function to convert arg_strings into an optional action + def consume_optional(start_index): + + # get the optional identified at this index + option_tuple = option_string_indices[start_index] + action, option_string, sep, explicit_arg = option_tuple + + # identify additional optionals in the same arg string + # (e.g. -xyz is the same as -x -y -z if no args are required) + match_argument = self._match_argument + action_tuples = [] + while True: + + # if we found no optional action, skip it + if action is None: + extras.append(arg_strings[start_index]) + return start_index + 1 + + # if there is an explicit argument, try to match the + # optional's string arguments to only this + if explicit_arg is not None: + arg_count = match_argument(action, 'A') + + # if the action is a single-dash option and takes no + # arguments, try to parse more single-dash options out + # of the tail of the option string + chars = self.prefix_chars + if ( + arg_count == 0 + and option_string[1] not in chars + and explicit_arg != '' + ): + if sep or explicit_arg[0] in chars: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) + action_tuples.append((action, [], option_string)) + char = option_string[0] + option_string = char + explicit_arg[0] + optionals_map = self._option_string_actions + if option_string in optionals_map: + action = optionals_map[option_string] + explicit_arg = explicit_arg[1:] + if not explicit_arg: + sep = explicit_arg = None + elif explicit_arg[0] == '=': + sep = '=' + explicit_arg = explicit_arg[1:] + else: + sep = '' + else: + extras.append(char + explicit_arg) + stop = start_index + 1 + break + # if the action expect exactly one argument, we've + # successfully matched the option; exit the loop + elif arg_count == 1: + stop = start_index + 1 + args = [explicit_arg] + action_tuples.append((action, args, option_string)) + break + + # error if a double-dash option did not use the + # explicit argument + else: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) + + # if there is no explicit argument, try to match the + # optional's string arguments with the following strings + # if successful, exit the loop + else: + start = start_index + 1 + selected_patterns = arg_strings_pattern[start:] + arg_count = match_argument(action, selected_patterns) + stop = start + arg_count + args = arg_strings[start:stop] + action_tuples.append((action, args, option_string)) + break + + # add the Optional to the list and return the index at which + # the Optional's string args stopped + assert action_tuples + for action, args, option_string in action_tuples: + take_action(action, args, option_string) + return stop + + # the list of Positionals left to be parsed; this is modified + # by consume_positionals() + positionals = self._get_positional_actions() + + # function to convert arg_strings into positional actions + def consume_positionals(start_index): + # match as many Positionals as possible + match_partial = self._match_arguments_partial + selected_pattern = arg_strings_pattern[start_index:] + arg_counts = match_partial(positionals, selected_pattern) + + # slice off the appropriate arg strings for each Positional + # and add the Positional and its args to the list + for action, arg_count in zip(positionals, arg_counts): + args = arg_strings[start_index: start_index + arg_count] + start_index += arg_count + take_action(action, args) + + # slice off the Positionals that we just parsed and return the + # index at which the Positionals' string args stopped + positionals[:] = positionals[len(arg_counts):] + return start_index + + # consume Positionals and Optionals alternately, until we have + # passed the last option string + extras = [] + start_index = 0 + if option_string_indices: + max_option_string_index = max(option_string_indices) + else: + max_option_string_index = -1 + while start_index <= max_option_string_index: + + # consume any Positionals preceding the next option + next_option_string_index = min([ + index + for index in option_string_indices + if index >= start_index]) + if start_index != next_option_string_index: + positionals_end_index = consume_positionals(start_index) + + # only try to parse the next optional if we didn't consume + # the option string during the positionals parsing + if positionals_end_index > start_index: + start_index = positionals_end_index + continue + else: + start_index = positionals_end_index + + # if we consumed all the positionals we could and we're not + # at the index of an option string, there were extra arguments + if start_index not in option_string_indices: + strings = arg_strings[start_index:next_option_string_index] + extras.extend(strings) + start_index = next_option_string_index + + # consume the next optional and any arguments for it + start_index = consume_optional(start_index) + + # consume any positionals following the last Optional + stop_index = consume_positionals(start_index) + + # if we didn't consume all the argument strings, there were extras + extras.extend(arg_strings[stop_index:]) + + # make sure all required actions were present and also convert + # action defaults which were not given as arguments + required_actions = [] + for action in self._actions: + if action not in seen_actions: + if action.required: + required_actions.append(_get_action_name(action)) + else: + # Convert action default now instead of doing it before + # parsing arguments to avoid calling convert functions + # twice (which may fail) if the argument was given, but + # only if it was defined already in the namespace + if (action.default is not None and + isinstance(action.default, str) and + hasattr(namespace, action.dest) and + action.default is getattr(namespace, action.dest)): + setattr(namespace, action.dest, + self._get_value(action, action.default)) + + if required_actions: + self.error(_('the following arguments are required: %s') % + ', '.join(required_actions)) + + # make sure all required groups had one option present + for group in self._mutually_exclusive_groups: + if group.required: + for action in group._group_actions: + if action in seen_non_default_actions: + break + + # if no actions were used, report the error + else: + names = [_get_action_name(action) + for action in group._group_actions + if action.help is not SUPPRESS] + msg = _('one of the arguments %s is required') + self.error(msg % ' '.join(names)) + + # return the updated namespace and the extra arguments + return namespace, extras + + def _read_args_from_files(self, arg_strings): + # expand arguments referencing files + new_arg_strings = [] + for arg_string in arg_strings: + + # for regular arguments, just add them back into the list + if not arg_string or arg_string[0] not in self.fromfile_prefix_chars: + new_arg_strings.append(arg_string) + + # replace arguments referencing files with the file content + else: + try: + with open(arg_string[1:], + encoding=_sys.getfilesystemencoding(), + errors=_sys.getfilesystemencodeerrors()) as args_file: + arg_strings = [] + for arg_line in args_file.read().splitlines(): + for arg in self.convert_arg_line_to_args(arg_line): + arg_strings.append(arg) + arg_strings = self._read_args_from_files(arg_strings) + new_arg_strings.extend(arg_strings) + except OSError as err: + self.error(str(err)) + + # return the modified argument list + return new_arg_strings + + def convert_arg_line_to_args(self, arg_line): + return [arg_line] + + def _match_argument(self, action, arg_strings_pattern): + # match the pattern for this action to the arg strings + nargs_pattern = self._get_nargs_pattern(action) + match = _re.match(nargs_pattern, arg_strings_pattern) + + # raise an exception if we weren't able to find a match + if match is None: + nargs_errors = { + None: _('expected one argument'), + OPTIONAL: _('expected at most one argument'), + ONE_OR_MORE: _('expected at least one argument'), + } + msg = nargs_errors.get(action.nargs) + if msg is None: + msg = ngettext('expected %s argument', + 'expected %s arguments', + action.nargs) % action.nargs + raise ArgumentError(action, msg) + + # return the number of arguments matched + return len(match.group(1)) + + def _match_arguments_partial(self, actions, arg_strings_pattern): + # progressively shorten the actions list by slicing off the + # final actions until we find a match + result = [] + for i in range(len(actions), 0, -1): + actions_slice = actions[:i] + pattern = ''.join([self._get_nargs_pattern(action) + for action in actions_slice]) + match = _re.match(pattern, arg_strings_pattern) + if match is not None: + result.extend([len(string) for string in match.groups()]) + break + + # return the list of arg string counts + return result + + def _parse_optional(self, arg_string): + # if it's an empty string, it was meant to be a positional + if not arg_string: + return None + + # if it doesn't start with a prefix, it was meant to be positional + if not arg_string[0] in self.prefix_chars: + return None + + # if the option string is present in the parser, return the action + if arg_string in self._option_string_actions: + action = self._option_string_actions[arg_string] + return action, arg_string, None, None + + # if it's just a single character, it was meant to be positional + if len(arg_string) == 1: + return None + + # if the option string before the "=" is present, return the action + option_string, sep, explicit_arg = arg_string.partition('=') + if sep and option_string in self._option_string_actions: + action = self._option_string_actions[option_string] + return action, option_string, sep, explicit_arg + + # search through all possible prefixes of the option string + # and all actions in the parser for possible interpretations + option_tuples = self._get_option_tuples(arg_string) + + # if multiple actions match, the option string was ambiguous + if len(option_tuples) > 1: + options = ', '.join([option_string + for action, option_string, sep, explicit_arg in option_tuples]) + args = {'option': arg_string, 'matches': options} + msg = _('ambiguous option: %(option)s could match %(matches)s') + self.error(msg % args) + + # if exactly one action matched, this segmentation is good, + # so return the parsed action + elif len(option_tuples) == 1: + option_tuple, = option_tuples + return option_tuple + + # if it was not found as an option, but it looks like a negative + # number, it was meant to be positional + # unless there are negative-number-like options + if self._negative_number_matcher.match(arg_string): + if not self._has_negative_number_optionals: + return None + + # if it contains a space, it was meant to be a positional + if ' ' in arg_string: + return None + + # it was meant to be an optional but there is no such option + # in this parser (though it might be a valid option in a subparser) + return None, arg_string, None, None + + def _get_option_tuples(self, option_string): + result = [] + + # option strings starting with two prefix characters are only + # split at the '=' + chars = self.prefix_chars + if option_string[0] in chars and option_string[1] in chars: + if self.allow_abbrev: + option_prefix, sep, explicit_arg = option_string.partition('=') + if not sep: + sep = explicit_arg = None + for option_string in self._option_string_actions: + if option_string.startswith(option_prefix): + action = self._option_string_actions[option_string] + tup = action, option_string, sep, explicit_arg + result.append(tup) + + # single character options can be concatenated with their arguments + # but multiple character options always have to have their argument + # separate + elif option_string[0] in chars and option_string[1] not in chars: + option_prefix = option_string + short_option_prefix = option_string[:2] + short_explicit_arg = option_string[2:] + + for option_string in self._option_string_actions: + if option_string == short_option_prefix: + action = self._option_string_actions[option_string] + tup = action, option_string, '', short_explicit_arg + result.append(tup) + elif option_string.startswith(option_prefix): + action = self._option_string_actions[option_string] + tup = action, option_string, None, None + result.append(tup) + + # shouldn't ever get here + else: + self.error(_('unexpected option string: %s') % option_string) + + # return the collected option tuples + return result + + def _get_nargs_pattern(self, action): + # in all examples below, we have to allow for '--' args + # which are represented as '-' in the pattern + nargs = action.nargs + + # the default (None) is assumed to be a single argument + if nargs is None: + nargs_pattern = '(-*A-*)' + + # allow zero or one arguments + elif nargs == OPTIONAL: + nargs_pattern = '(-*A?-*)' + + # allow zero or more arguments + elif nargs == ZERO_OR_MORE: + nargs_pattern = '(-*[A-]*)' + + # allow one or more arguments + elif nargs == ONE_OR_MORE: + nargs_pattern = '(-*A[A-]*)' + + # allow any number of options or arguments + elif nargs == REMAINDER: + nargs_pattern = '([-AO]*)' + + # allow one argument followed by any number of options or arguments + elif nargs == PARSER: + nargs_pattern = '(-*A[-AO]*)' + + # suppress action, like nargs=0 + elif nargs == SUPPRESS: + nargs_pattern = '(-*-*)' + + # all others should be integers + else: + nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) + + # if this is an optional action, -- is not allowed + if action.option_strings: + nargs_pattern = nargs_pattern.replace('-*', '') + nargs_pattern = nargs_pattern.replace('-', '') + + # return the pattern + return nargs_pattern + + # ======================== + # Alt command line argument parsing, allowing free intermix + # ======================== + + def parse_intermixed_args(self, args=None, namespace=None): + args, argv = self.parse_known_intermixed_args(args, namespace) + if argv: + msg = _('unrecognized arguments: %s') + self.error(msg % ' '.join(argv)) + return args + + def parse_known_intermixed_args(self, args=None, namespace=None): + # returns a namespace and list of extras + # + # positional can be freely intermixed with optionals. optionals are + # first parsed with all positional arguments deactivated. The 'extras' + # are then parsed. If the parser definition is incompatible with the + # intermixed assumptions (e.g. use of REMAINDER, subparsers) a + # TypeError is raised. + # + # positionals are 'deactivated' by setting nargs and default to + # SUPPRESS. This blocks the addition of that positional to the + # namespace + + positionals = self._get_positional_actions() + a = [action for action in positionals + if action.nargs in [PARSER, REMAINDER]] + if a: + raise TypeError('parse_intermixed_args: positional arg' + ' with nargs=%s'%a[0].nargs) + + if [action.dest for group in self._mutually_exclusive_groups + for action in group._group_actions if action in positionals]: + raise TypeError('parse_intermixed_args: positional in' + ' mutuallyExclusiveGroup') + + try: + save_usage = self.usage + try: + if self.usage is None: + # capture the full usage for use in error messages + self.usage = self.format_usage()[7:] + for action in positionals: + # deactivate positionals + action.save_nargs = action.nargs + # action.nargs = 0 + action.nargs = SUPPRESS + action.save_default = action.default + action.default = SUPPRESS + namespace, remaining_args = self.parse_known_args(args, + namespace) + for action in positionals: + # remove the empty positional values from namespace + if (hasattr(namespace, action.dest) + and getattr(namespace, action.dest)==[]): + from warnings import warn + warn('Do not expect %s in %s' % (action.dest, namespace)) + delattr(namespace, action.dest) + finally: + # restore nargs and usage before exiting + for action in positionals: + action.nargs = action.save_nargs + action.default = action.save_default + optionals = self._get_optional_actions() + try: + # parse positionals. optionals aren't normally required, but + # they could be, so make sure they aren't. + for action in optionals: + action.save_required = action.required + action.required = False + for group in self._mutually_exclusive_groups: + group.save_required = group.required + group.required = False + namespace, extras = self.parse_known_args(remaining_args, + namespace) + finally: + # restore parser values before exiting + for action in optionals: + action.required = action.save_required + for group in self._mutually_exclusive_groups: + group.required = group.save_required + finally: + self.usage = save_usage + return namespace, extras + + # ======================== + # Value conversion methods + # ======================== + def _get_values(self, action, arg_strings): + # for everything but PARSER, REMAINDER args, strip out first '--' + if not action.option_strings and action.nargs not in [PARSER, REMAINDER]: + try: + arg_strings.remove('--') + except ValueError: + pass + + # optional argument produces a default when not present + if not arg_strings and action.nargs == OPTIONAL: + if action.option_strings: + value = action.const + else: + value = action.default + if isinstance(value, str): + value = self._get_value(action, value) + self._check_value(action, value) + + # when nargs='*' on a positional, if there were no command-line + # args, use the default if it is anything other than None + elif (not arg_strings and action.nargs == ZERO_OR_MORE and + not action.option_strings): + if action.default is not None: + value = action.default + self._check_value(action, value) + else: + # since arg_strings is always [] at this point + # there is no need to use self._check_value(action, value) + value = arg_strings + + # single argument or optional argument produces a single value + elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: + arg_string, = arg_strings + value = self._get_value(action, arg_string) + self._check_value(action, value) + + # REMAINDER arguments convert all values, checking none + elif action.nargs == REMAINDER: + value = [self._get_value(action, v) for v in arg_strings] + + # PARSER arguments convert all values, but check only the first + elif action.nargs == PARSER: + value = [self._get_value(action, v) for v in arg_strings] + self._check_value(action, value[0]) + + # SUPPRESS argument does not put anything in the namespace + elif action.nargs == SUPPRESS: + value = SUPPRESS + + # all other types of nargs produce a list + else: + value = [self._get_value(action, v) for v in arg_strings] + for v in value: + self._check_value(action, v) + + # return the converted value + return value + + def _get_value(self, action, arg_string): + type_func = self._registry_get('type', action.type, action.type) + if not callable(type_func): + msg = _('%r is not callable') + raise ArgumentError(action, msg % type_func) + + # convert the value to the appropriate type + try: + result = type_func(arg_string) + + # ArgumentTypeErrors indicate errors + except ArgumentTypeError as err: + msg = str(err) + raise ArgumentError(action, msg) + + # TypeErrors or ValueErrors also indicate errors + except (TypeError, ValueError): + name = getattr(action.type, '__name__', repr(action.type)) + args = {'type': name, 'value': arg_string} + msg = _('invalid %(type)s value: %(value)r') + raise ArgumentError(action, msg % args) + + # return the converted value + return result + + def _check_value(self, action, value): + # converted value must be one of the choices (if specified) + if action.choices is not None and value not in action.choices: + args = {'value': value, + 'choices': ', '.join(map(repr, action.choices))} + msg = _('invalid choice: %(value)r (choose from %(choices)s)') + raise ArgumentError(action, msg % args) + + # ======================= + # Help-formatting methods + # ======================= + def format_usage(self): + formatter = self._get_formatter() + formatter.add_usage(self.usage, self._actions, + self._mutually_exclusive_groups) + return formatter.format_help() + + def format_help(self): + formatter = self._get_formatter() + + # usage + formatter.add_usage(self.usage, self._actions, + self._mutually_exclusive_groups) + + # description + formatter.add_text(self.description) + + # positionals, optionals and user-defined groups + for action_group in self._action_groups: + formatter.start_section(action_group.title) + formatter.add_text(action_group.description) + formatter.add_arguments(action_group._group_actions) + formatter.end_section() + + # epilog + formatter.add_text(self.epilog) + + # determine help from format above + return formatter.format_help() + + def _get_formatter(self): + return self.formatter_class(prog=self.prog) + + # ===================== + # Help-printing methods + # ===================== + def print_usage(self, file=None): + if file is None: + file = _sys.stdout + self._print_message(self.format_usage(), file) + + def print_help(self, file=None): + if file is None: + file = _sys.stdout + self._print_message(self.format_help(), file) + + def _print_message(self, message, file=None): + if message: + file = file or _sys.stderr + try: + file.write(message) + except (AttributeError, OSError): + pass + + # =============== + # Exiting methods + # =============== + def exit(self, status=0, message=None): + if message: + self._print_message(message, _sys.stderr) + _sys.exit(status) + + def error(self, message): + """error(message: string) + + Prints a usage message incorporating the message to stderr and + exits. + + If you override this in a subclass, it should not return -- it + should either exit or raise an exception. + """ + self.print_usage(_sys.stderr) + args = {'prog': self.prog, 'message': message} + self.exit(2, _('%(prog)s: error: %(message)s\n') % args) diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index 7bf2bfa60..955713d68 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -5,16 +5,13 @@ - Use `rich` for formatting. - Can be themed with an accent color. -This is largely built by fussing around in argparse implementation details, and is -extremely chaotic as a result. - -TODO: the current implementation should be robust given our test coverage, but unideal -long-term. We should just maintain our own fork of argparse. +This is largely built by fussing around in argparse implementation details. It's +chaotic as a result; for stability we mirror argparse at _argparse.py. """ from __future__ import annotations -import argparse +import argparse as argparse_sys import contextlib import dataclasses import difflib @@ -37,6 +34,7 @@ from rich.theme import Theme from typing_extensions import override +from . import _argparse as argparse from . import _arguments, _strings, conf from ._parsers import ParserSpecification @@ -269,7 +267,9 @@ class _ArgumentInfo: global_unrecognized_args: List[str] = [] -class TyroArgumentParser(argparse.ArgumentParser): +# We inherit from both our local mirror of argparse and the upstream one. +# Including the latter is purely for `isinstance()`-style checks. +class TyroArgumentParser(argparse.ArgumentParser, argparse_sys.ArgumentParser): # type: ignore _parser_specification: ParserSpecification _parsing_known_args: bool _args: List[str] @@ -359,7 +359,7 @@ def take_action(action, argument_strings, option_string=None): def consume_optional(start_index): # get the optional identified at this index option_tuple = option_string_indices[start_index] - action, option_string, explicit_arg = option_tuple + action, option_string, sep, explicit_arg = option_tuple # identify additional optionals in the same arg string # (e.g. -xyz is the same as -x -y -z if no args are required) @@ -374,7 +374,6 @@ def consume_optional(start_index): if not self._parsing_known_args: global_unrecognized_args.append(option_string) # - extras.append(arg_strings[start_index]) return start_index + 1 @@ -392,18 +391,27 @@ def consume_optional(start_index): and option_string[1] not in chars and explicit_arg != "" ): + if sep or explicit_arg[0] in chars: + msg = _("ignored explicit argument %r") + raise argparse.ArgumentError(action, msg % explicit_arg) action_tuples.append((action, [], option_string)) char = option_string[0] option_string = char + explicit_arg[0] - new_explicit_arg = explicit_arg[1:] or None optionals_map = self._option_string_actions if option_string in optionals_map: action = optionals_map[option_string] - explicit_arg = new_explicit_arg + explicit_arg = explicit_arg[1:] + if not explicit_arg: + sep = explicit_arg = None + elif explicit_arg[0] == "=": + sep = "=" + explicit_arg = explicit_arg[1:] + else: + sep = "" else: - msg = _("ignored explicit argument %r") - raise argparse.ArgumentError(action, msg % explicit_arg) - + extras.append(char + explicit_arg) + stop = start_index + 1 + break # if the action expect exactly one argument, we've # successfully matched the option; exit the loop elif arg_count == 1: diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index 5c90a0ac9..e92f97cf0 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -3,7 +3,6 @@ from __future__ import annotations -import argparse import dataclasses import enum import functools @@ -30,6 +29,7 @@ import rich.markup import shtab +from . import _argparse as argparse from . import _fields, _instantiators, _resolver, _strings from ._typing import TypeForm from .conf import _markers @@ -48,7 +48,6 @@ _T = TypeVar("_T") -# TODO: refactor! class BooleanOptionalAction(argparse.Action): """Adapted from https://github.com/python/cpython/pull/27672""" diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index 680e238fe..c2746dfc4 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -1,6 +1,5 @@ """Core public API.""" -import argparse import dataclasses import pathlib import sys @@ -21,6 +20,7 @@ import shtab from typing_extensions import Literal +from . import _argparse as argparse from . import ( _argparse_formatter, _arguments, diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 523c9cf28..5a146037f 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -2,7 +2,6 @@ from __future__ import annotations -import argparse import dataclasses from typing import ( Any, @@ -20,6 +19,7 @@ from typing_extensions import Annotated, get_args, get_origin +from . import _argparse as argparse from . import ( _argparse_formatter, _arguments, diff --git a/tests/test_py311_generated/_generate.py b/tests/test_py311_generated/_generate.py index 4e572529a..a78b12adc 100644 --- a/tests/test_py311_generated/_generate.py +++ b/tests/test_py311_generated/_generate.py @@ -41,4 +41,4 @@ subprocess.run(["isort", "--profile=black", str(out_path)], check=True) subprocess.run(["ruff", "format", str(out_path)], check=True) - subprocess.run(["ruff", "--fix", str(out_path)], check=True) + subprocess.run(["ruff", "check", "--fix", str(out_path)], check=True) diff --git a/tests/test_py311_generated/test_collections_generated.py b/tests/test_py311_generated/test_collections_generated.py index ba494b85c..2b35772a2 100644 --- a/tests/test_py311_generated/test_collections_generated.py +++ b/tests/test_py311_generated/test_collections_generated.py @@ -14,6 +14,7 @@ Sequence, Set, Tuple, + Type, ) import pytest @@ -466,6 +467,20 @@ def main(x: tuple = ()) -> Any: assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == ("0", "1", "2", "3") +def test_narrowing_edge_case() -> None: + """https://github.com/brentyi/tyro/issues/136""" + + @dataclasses.dataclass + class Config: + _target: Type = dataclasses.field(default_factory=lambda: MyClass) + + class MyClass: + def __len__(self): + return 0 + + assert tyro.cli(Config, args=[]) == Config() + + def test_no_type_collections(): assert tyro.cli(dict, args="a b c d".split(" ")) == {"a": "b", "c": "d"} assert tyro.cli(list, args="a b c d".split(" ")) == ["a", "b", "c", "d"] diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index cfd7211a2..944536437 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -4,6 +4,7 @@ import io import json as json_ import shlex +import sys from typing import Annotated, Any, Dict, Generic, List, Tuple, Type, TypeVar import pytest @@ -1355,3 +1356,26 @@ class DatasetConfig: helptext = target.getvalue() assert "OptimizerConfig options" in helptext assert "DatasetConfig options" in helptext + + +def test_counter_action() -> None: + def main( + verbosity: tyro.conf.UseCounterAction[int], + aliased_verbosity: Annotated[ + tyro.conf.UseCounterAction[int], tyro.conf.arg(aliases=["-v"]) + ], + ) -> Tuple[int, int]: + """Example showing how to use counter actions. + Args: + verbosity: Verbosity level. + aliased_verbosity: Same as above, but can also be specified with -v, -vv, -vvv, etc. + """ + return verbosity, aliased_verbosity + + assert tyro.cli(main, args=[]) == (0, 0) + assert tyro.cli(main, args="--verbosity --verbosity".split(" ")) == (2, 0) + assert tyro.cli(main, args="--verbosity --verbosity -v".split(" ")) == (2, 1) + if sys.version_info >= (3, 8): + # Doesn't work in Python 3.7 because of argparse limitations. + assert tyro.cli(main, args="--verbosity --verbosity -vv".split(" ")) == (2, 2) + assert tyro.cli(main, args="--verbosity --verbosity -vvv".split(" ")) == (2, 3) diff --git a/tests/test_py311_generated/test_generics_and_serialization_generated.py b/tests/test_py311_generated/test_generics_and_serialization_generated.py index a0897ef27..93eea5277 100644 --- a/tests/test_py311_generated/test_generics_and_serialization_generated.py +++ b/tests/test_py311_generated/test_generics_and_serialization_generated.py @@ -40,6 +40,17 @@ class TupleGenericVariable(Generic[ScalarType]): ) == TupleGenericVariable((SpecialInt(1), SpecialInt(2), SpecialInt(3))) +def test_tuple_generic_variable_newtype_container() -> None: + @dataclasses.dataclass + class TupleGenericVariable(Generic[ScalarType]): + xyz: Tuple[ScalarType, ...] + + SpecialInt = NewType("SpecialInt", Tuple[int]) + assert tyro.cli( + TupleGenericVariable[SpecialInt], args=["--xyz", "1", "2", "3"] + ) == TupleGenericVariable((SpecialInt((1,)), SpecialInt((2,)), SpecialInt((3,)))) + + def test_tuple_generic_variable_more_newtype() -> None: @dataclasses.dataclass class TupleGenericVariable(Generic[ScalarType]): diff --git a/tests/test_py311_generated/test_self_type_generated.py b/tests/test_py311_generated/test_self_type_generated.py index eaffa0314..021ef5b00 100644 --- a/tests/test_py311_generated/test_self_type_generated.py +++ b/tests/test_py311_generated/test_self_type_generated.py @@ -7,7 +7,7 @@ import tyro -class TestClass: +class SomeClass: def __init__(self, a: int, b: int) -> None: self.a = a self.b = b @@ -16,7 +16,7 @@ def method1(self, x: Self) -> None: self.effect = x @classmethod - def method2(cls, x: Self) -> TestClass: + def method2(cls, x: Self) -> SomeClass: return x # Self is not valid in static methods. @@ -27,65 +27,65 @@ def method2(cls, x: Self) -> TestClass: # return x -class TestSubclass(TestClass): ... +class SomeSubclass(SomeClass): ... def test_method() -> None: - x = TestClass(0, 0) + x = SomeClass(0, 0) with pytest.raises(SystemExit): tyro.cli(x.method1, args=[]) assert tyro.cli(x.method1, args="--x.a 3 --x.b 3".split(" ")) is None assert x.effect.a == 3 and x.effect.b == 3 - assert isinstance(x, TestClass) + assert isinstance(x, SomeClass) def test_classmethod() -> None: - x = TestClass(0, 0) + x = SomeClass(0, 0) with pytest.raises(SystemExit): tyro.cli(x.method2, args=[]) with pytest.raises(SystemExit): - tyro.cli(TestClass.method2, args=[]) + tyro.cli(SomeClass.method2, args=[]) y = tyro.cli(x.method2, args="--x.a 3 --x.b 3".split(" ")) assert y.a == 3 assert y.b == 3 - assert isinstance(y, TestClass) + assert isinstance(y, SomeClass) - y = tyro.cli(TestClass.method2, args="--x.a 3 --x.b 3".split(" ")) + y = tyro.cli(SomeClass.method2, args="--x.a 3 --x.b 3".split(" ")) assert y.a == 3 assert y.b == 3 - assert isinstance(y, TestClass) + assert isinstance(y, SomeClass) def test_subclass_method() -> None: - x = TestSubclass(0, 0) + x = SomeSubclass(0, 0) with pytest.raises(SystemExit): tyro.cli(x.method1, args=[]) assert tyro.cli(x.method1, args="--x.a 3 --x.b 3".split(" ")) is None assert x.effect.a == 3 and x.effect.b == 3 - assert isinstance(x, TestSubclass) + assert isinstance(x, SomeSubclass) y = tyro.cli(x.method2, args="--x.a 3 --x.b 3".split(" ")) assert y.a == 3 assert y.b == 3 - assert isinstance(y, TestClass) + assert isinstance(y, SomeClass) def test_subclass_classmethod() -> None: - x = TestSubclass(0, 0) + x = SomeSubclass(0, 0) with pytest.raises(SystemExit): tyro.cli(x.method2, args=[]) with pytest.raises(SystemExit): - tyro.cli(TestSubclass.method2, args=[]) + tyro.cli(SomeSubclass.method2, args=[]) y = tyro.cli(x.method2, args="--x.a 3 --x.b 3".split(" ")) assert y.a == 3 assert y.b == 3 - assert isinstance(y, TestClass) + assert isinstance(y, SomeClass) - y = tyro.cli(TestSubclass.method2, args="--x.a 3 --x.b 3".split(" ")) + y = tyro.cli(SomeSubclass.method2, args="--x.a 3 --x.b 3".split(" ")) assert y.a == 3 assert y.b == 3 - assert isinstance(y, TestClass) + assert isinstance(y, SomeClass) From f373efab42ee96482062c9c50037d85dddf066c9 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 10 May 2024 12:44:01 +0100 Subject: [PATCH 397/491] More tests, quality-of-life improvements for scripts with multiple workers (#142) * Quality-of-life improvements for distributed settings - Added the `enable_console_outputs=` argument, which can be used to prevent duplicate help messages / errors - Move error messages to `stderr` * Add Set[] annotations * Re-generate tests for Python >=3.11 * Add `enable_console_outputs` to tests * Add error suppression test for `enable_console_outputs` * Coverage fix * `enable_console_outputs=False` => `console_outputs=False`, add example * Update dictionaries example * Update version-dependent ignore * Fix tests for Python 3.7 * Example nit --- .../04_additional/02_dictionaries.rst | 17 +-- .../14_suppress_console_outputs.rst | 58 +++++++++ docs/source/index.md | 4 - examples/04_additional/02_dictionaries.py | 10 +- .../14_suppress_console_outputs.py | 43 +++++++ src/tyro/_argparse_formatter.py | 58 +++++---- src/tyro/_cli.py | 103 +++++++++------- src/tyro/_docstrings.py | 4 +- src/tyro/_fields.py | 3 +- src/tyro/_instantiators.py | 2 +- src/tyro/_parsers.py | 1 + tests/conftest.py | 4 + tests/helptext_utils.py | 21 +++- tests/test_collections.py | 2 +- tests/test_conf.py | 34 +++--- tests/test_errors.py | 37 ++++-- tests/test_flax_min_py38.py | 4 +- tests/test_functools.py | 12 +- tests/test_helptext.py | 112 +++++++++--------- tests/test_missing.py | 2 +- tests/test_nested.py | 4 +- tests/test_new_style_annotations_generics.py | 55 +++++++++ tests/test_new_style_annotations_unions.py | 62 ++++++++++ tests/test_partial.py | 8 +- .../test_collections_generated.py | 2 +- .../test_conf_generated.py | 34 +++--- .../test_errors_generated.py | 37 ++++-- .../test_flax_min_py38_generated.py | 4 +- .../test_functools_generated.py | 12 +- .../test_helptext_generated.py | 112 +++++++++--------- .../test_missing_generated.py | 2 +- .../test_nested_generated.py | 4 +- ...ew_style_annotations_generics_generated.py | 54 +++++++++ ...w_style_annotations_min_py312_generated.py | 54 +++++++++ ..._new_style_annotations_unions_generated.py | 61 ++++++++++ .../test_partial_generated.py | 8 +- 36 files changed, 757 insertions(+), 287 deletions(-) create mode 100644 docs/source/examples/04_additional/14_suppress_console_outputs.rst create mode 100644 examples/04_additional/14_suppress_console_outputs.py create mode 100644 tests/test_new_style_annotations_generics.py create mode 100644 tests/test_new_style_annotations_unions.py create mode 100644 tests/test_py311_generated/test_new_style_annotations_generics_generated.py create mode 100644 tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py create mode 100644 tests/test_py311_generated/test_new_style_annotations_unions_generated.py diff --git a/docs/source/examples/04_additional/02_dictionaries.rst b/docs/source/examples/04_additional/02_dictionaries.rst index 7f15d7613..37162ccc9 100644 --- a/docs/source/examples/04_additional/02_dictionaries.rst +++ b/docs/source/examples/04_additional/02_dictionaries.rst @@ -1,12 +1,15 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -Overriding Dictionaries +Dictionaries and TypedDict ========================================== -Dictionary inputs can be specified using either a standard ``Dict[K, V]`` annotation, or a -``TypedDict`` subclass. +Dictionary inputs can be specified using either a standard ``Dict[K, V]`` +annotation, or a ``TypedDict`` subclass. + +For configuring ``TypedDict``\ , we also support ``total={True/False}``\ , +``typing.Required``\ , and ``typing.NotRequired``. @@ -66,14 +69,6 @@ Dictionary inputs can be specified using either a standard ``Dict[K, V]`` annota ------------ -.. raw:: html - - python 04_additional/02_dictionaries.py --typed-dict-a.learning-rate 3e-4 - -.. program-output:: python ../../examples/04_additional/02_dictionaries.py --typed-dict-a.learning-rate 3e-4 - ------------- - .. raw:: html python 04_additional/02_dictionaries.py --typed-dict-a.learning-rate 3e-4 --typed-dict-b.betas 0.9 0.999 diff --git a/docs/source/examples/04_additional/14_suppress_console_outputs.rst b/docs/source/examples/04_additional/14_suppress_console_outputs.rst new file mode 100644 index 000000000..0ec626b5d --- /dev/null +++ b/docs/source/examples/04_additional/14_suppress_console_outputs.rst @@ -0,0 +1,58 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Cleaner Console Outputs for Scripts with Multiple Workers +========================================== + + +The ``console_outputs=`` argument can be set to ``False`` to suppress helptext and +error message printing. + +This is useful in PyTorch for distributed training scripts, where you only want +to print the helptext from the main process: + +.. code-block:: python + + # Hugging Face Accelerate. + args = tyro.cli(Args, console_outputs=accelerator.is_main_process) + + # PyTorch DDP. + args = tyro.cli(Args, console_outputs=(rank == 0)) + + # PyTorch Lightning. + args = tyro.cli(Args, console_outputs=trainer.is_global_zero) + + + +.. code-block:: python + :linenos: + + + import dataclasses + + import tyro + + + @dataclasses.dataclass + class Args: + """Description. + This should show up in the helptext!""" + + field1: int + """A field.""" + + field2: int = 3 + """A numeric field, with a default value.""" + + + if __name__ == "__main__": + args = tyro.cli(Args, console_outputs=False) + print(args) + +------------ + +.. raw:: html + + python 04_additional/14_suppress_console_outputs.py --help + +.. program-output:: python ../../examples/04_additional/14_suppress_console_outputs.py --help diff --git a/docs/source/index.md b/docs/source/index.md index c622118b8..5c9d19457 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -49,10 +49,6 @@ To get started, we recommend browsing the examples to the left. [flax.linen](https://flax.readthedocs.io/en/latest/api_reference/flax.linen.html), and more. - Hate `tyro`? Just remove one line of code, and you're left with beautiful, - type-annotated, and documented vanilla Python that can be used with a range - of other configuration libraries. - 3. **Modularity.** `tyro` supports hierarchical configuration structures, which make it easy to diff --git a/examples/04_additional/02_dictionaries.py b/examples/04_additional/02_dictionaries.py index 1504c3b42..e7d449c3c 100644 --- a/examples/04_additional/02_dictionaries.py +++ b/examples/04_additional/02_dictionaries.py @@ -1,11 +1,13 @@ -"""Overriding Dictionaries +"""Dictionaries and TypedDict -Dictionary inputs can be specified using either a standard `Dict[K, V]` annotation, or a -`TypedDict` subclass. +Dictionary inputs can be specified using either a standard `Dict[K, V]` +annotation, or a `TypedDict` subclass. + +For configuring `TypedDict`, we also support `total={True/False}`, +`typing.Required`, and `typing.NotRequired`. Usage: `python ./02_dictionaries.py --help` -`python ./02_dictionaries.py --typed-dict-a.learning-rate 3e-4` `python ./02_dictionaries.py --typed-dict-a.learning-rate 3e-4 --typed-dict-b.betas 0.9 0.999` `python ./02_dictionaries.py --typed-dict-b.betas 0.9 0.999` """ diff --git a/examples/04_additional/14_suppress_console_outputs.py b/examples/04_additional/14_suppress_console_outputs.py new file mode 100644 index 000000000..a977102cc --- /dev/null +++ b/examples/04_additional/14_suppress_console_outputs.py @@ -0,0 +1,43 @@ +"""Cleaner Console Outputs for Scripts with Multiple Workers + +The `console_outputs=` argument can be set to `False` to suppress helptext and +error message printing. + +This is useful in PyTorch for distributed training scripts, where you only want +to print the helptext from the main process: + +```python +# Hugging Face Accelerate. +args = tyro.cli(Args, console_outputs=accelerator.is_main_process) + +# PyTorch DDP. +args = tyro.cli(Args, console_outputs=(rank == 0)) + +# PyTorch Lightning. +args = tyro.cli(Args, console_outputs=trainer.is_global_zero) +``` + +Usage: +`python ./14_suppress_console_outputs.py --help` +""" + +import dataclasses + +import tyro + + +@dataclasses.dataclass +class Args: + """Description. + This should show up in the helptext!""" + + field1: int + """A field.""" + + field2: int = 3 + """A numeric field, with a default value.""" + + +if __name__ == "__main__": + args = tyro.cli(Args, console_outputs=False) + print(args) diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index 955713d68..10c6eabaf 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -236,9 +236,9 @@ def ansi_context() -> Generator[None, None, None]: def str_from_rich( renderable: RenderableType, width: Optional[int] = None, soft_wrap: bool = False ) -> str: - console = Console(width=width, theme=THEME.as_rich_theme()) - with console.capture() as out: - console.print(renderable, soft_wrap=soft_wrap) + dummy_console = Console(width=width, theme=THEME.as_rich_theme()) + with dummy_console.capture() as out: + dummy_console.print(renderable, soft_wrap=soft_wrap) return out.get().rstrip("\n") @@ -272,11 +272,21 @@ class _ArgumentInfo: class TyroArgumentParser(argparse.ArgumentParser, argparse_sys.ArgumentParser): # type: ignore _parser_specification: ParserSpecification _parsing_known_args: bool + _console_outputs: bool _args: List[str] def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) + @override + def _print_message(self, message, file=None): + if message and self._console_outputs: + file = file or sys.stderr + try: + file.write(message) + except (AttributeError, OSError): # pragma: no cover + pass + @override def _parse_known_args(self, arg_strings, namespace): # pragma: no cover """We override _parse_known_args() to improve error messages in the presence of @@ -571,8 +581,6 @@ def error(self, message: str) -> NoReturn: should either exit or raise an exception. """ - console = Console(theme=THEME.as_rich_theme()) - extra_info: List[RenderableType] = [] global global_unrecognized_args if len(global_unrecognized_args) == 0 and message.startswith( @@ -827,20 +835,24 @@ def get_score(option_string: str) -> float: ) ) - console.print( - Panel( - Group( - f"{message[0].upper() + message[1:]}" if len(message) > 0 else "", - *extra_info, - Rule(style=Style(color="red")), - f"For full helptext, run [bold]{self.prog} --help[/bold]", - ), - title=f"[bold]{message_title}[/bold]", - title_align="left", - border_style=Style(color="bright_red"), - expand=False, + if self._console_outputs: + console = Console(theme=THEME.as_rich_theme(), stderr=True) + console.print( + Panel( + Group( + f"{message[0].upper() + message[1:]}" + if len(message) > 0 + else "", + *extra_info, + Rule(style=Style(color="red")), + f"For full helptext, run [bold]{self.prog} --help[/bold]", + ), + title=f"[bold]{message_title}[/bold]", + title_align="left", + border_style=Style(color="bright_red"), + expand=False, + ) ) - ) sys.exit(2) @@ -943,8 +955,10 @@ def format_help(self): return self._tyro_format_nonroot() def _tyro_format_root(self): - console = Console(width=self.formatter._width, theme=THEME.as_rich_theme()) - with console.capture() as capture: + dummy_console = Console( + width=self.formatter._width, theme=THEME.as_rich_theme() + ) + with dummy_console.capture() as capture: # Get rich renderables from items. top_parts = [] column_parts = [] @@ -1008,8 +1022,8 @@ def _tyro_format_root(self): width=column_width, ) - console.print(Group(*top_parts)) - console.print(columns) + dummy_console.print(Group(*top_parts)) + dummy_console.print(columns) return capture.get() def _format_action(self, action: argparse.Action): diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index c2746dfc4..2190c1755 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -52,6 +52,7 @@ def cli( default: Optional[OutT] = None, return_unknown_args: Literal[False] = False, use_underscores: bool = False, + console_outputs: bool = True, ) -> OutT: ... @@ -65,6 +66,7 @@ def cli( default: Optional[OutT] = None, return_unknown_args: Literal[True], use_underscores: bool = False, + console_outputs: bool = True, ) -> Tuple[OutT, List[str]]: ... @@ -75,12 +77,13 @@ def cli( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - # Note that passing a default makes sense for things like dataclasses, but are not - # supported for general callables. These can, however, be specified in the signature - # of the callable itself. + # Passing a default makes sense for things like dataclasses, but are not + # supported for general callables. These can, however, be specified in the + # signature of the callable itself. default: None = None, return_unknown_args: Literal[False] = False, use_underscores: bool = False, + console_outputs: bool = True, ) -> OutT: ... @@ -91,12 +94,13 @@ def cli( prog: Optional[str] = None, description: Optional[str] = None, args: Optional[Sequence[str]] = None, - # Note that passing a default makes sense for things like dataclasses, but are not - # supported for general callables. These can, however, be specified in the signature - # of the callable itself. + # Passing a default makes sense for things like dataclasses, but are not + # supported for general callables. These can, however, be specified in the + # signature of the callable itself. default: None = None, return_unknown_args: Literal[True], use_underscores: bool = False, + console_outputs: bool = True, ) -> Tuple[OutT, List[str]]: ... @@ -109,12 +113,13 @@ def cli( default: Optional[OutT] = None, return_unknown_args: bool = False, use_underscores: bool = False, + console_outputs: bool = True, **deprecated_kwargs, ) -> Union[OutT, Tuple[OutT, List[str]]]: """Call or instantiate `f`, with inputs populated from an automatically generated CLI interface. - `f` should have type-annotated inputs, and can be a function or type. Note that if + `f` should have type-annotated inputs, and can be a function or type. If `f` is a type, `tyro.cli()` returns an instance. The parser is generated by populating helptext from docstrings and types from @@ -159,16 +164,22 @@ def cli( args: If set, parse arguments from a sequence of strings instead of the commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. default: An instance of `OutT` to use for default values; supported if `f` is a - type like a dataclass or dictionary, but not if `f` is a general callable like - a function or standard class. Helpful for merging CLI arguments with values - loaded from elsewhere. (for example, a config object loaded from a yaml file) + type like a dataclass or dictionary, but not if `f` is a general callable + like a function or standard class. Helpful for merging CLI arguments with + values loaded from elsewhere. (for example, a config object loaded from a + yaml file) return_unknown_args: If True, return a tuple of the output of `f` and a list of unknown arguments. Mirrors the unknown arguments returned from `argparse.ArgumentParser.parse_known_args()`. use_underscores: If True, use underscores as a word delimeter instead of hyphens. - This primarily impacts helptext; underscores and hyphens are treated equivalently - when parsing happens. We default helptext to hyphens to follow the GNU style guide. + This primarily impacts helptext; underscores and hyphens are treated + equivalently when parsing happens. We default helptext to hyphens to follow + the GNU style guide. https://www.gnu.org/software/libc/manual/html_node/Argument-Syntax.html + console_outputs: If set to `False`, parsing errors and help messages will be + supressed. This can be useful for distributed settings, where `tyro.cli()` + is called from multiple workers but we only want console outputs from the + main one. Returns: The output of `f(...)` or an instance `f`. If `f` is a class, the two are @@ -190,6 +201,7 @@ def cli( return_parser=False, return_unknown_args=return_unknown_args, use_underscores=use_underscores, + console_outputs=console_outputs, **deprecated_kwargs, ) @@ -210,6 +222,7 @@ def get_parser( description: Optional[str] = None, default: Optional[OutT] = None, use_underscores: bool = False, + console_outputs: bool = True, ) -> argparse.ArgumentParser: ... @@ -221,18 +234,20 @@ def get_parser( description: Optional[str] = None, default: Optional[OutT] = None, use_underscores: bool = False, + console_outputs: bool = True, ) -> argparse.ArgumentParser: ... def get_parser( f: Union[TypeForm[OutT], Callable[..., OutT]], *, - # Note that we have no `args` argument, since this is only used when + # We have no `args` argument, since this is only used when # parser.parse_args() is called. prog: Optional[str] = None, description: Optional[str] = None, default: Optional[OutT] = None, use_underscores: bool = False, + console_outputs: bool = True, ) -> argparse.ArgumentParser: """Get the `argparse.ArgumentParser` object generated under-the-hood by `tyro.cli()`. Useful for tools like `sphinx-argparse`, `argcomplete`, etc. @@ -251,6 +266,7 @@ def get_parser( return_parser=True, return_unknown_args=False, use_underscores=use_underscores, + console_outputs=console_outputs, ), ) @@ -264,6 +280,7 @@ def _cli_impl( default: Optional[OutT], return_parser: bool, return_unknown_args: bool, + console_outputs: bool, **deprecated_kwargs, ) -> Union[ OutT, @@ -386,6 +403,7 @@ def _cli_impl( ) parser._parser_specification = parser_spec parser._parsing_known_args = return_unknown_args + parser._console_outputs = console_outputs parser._args = args parser_spec.apply(parser) @@ -462,39 +480,40 @@ def _cli_impl( from ._argparse_formatter import THEME - console = Console(theme=THEME.as_rich_theme()) - console.print( - Panel( - Group( - "[bright_red][bold]Error parsing" - f" {e.arg.lowered.name_or_flag if isinstance(e.arg, _arguments.ArgumentDefinition) else e.arg}[/bold]:[/bright_red] {e.message}", - *cast( # Cast to appease mypy... - List[RenderableType], - ( - [] - if not isinstance(e.arg, _arguments.ArgumentDefinition) - or e.arg.lowered.help is None - else [ - Rule(style=Style(color="red")), - "Argument helptext:", - Padding( - Group( - f"{e.arg.lowered.name_or_flag} [bold]{e.arg.lowered.metavar}[/bold]", - e.arg.lowered.help, + if console_outputs: + console = Console(theme=THEME.as_rich_theme(), stderr=True) + console.print( + Panel( + Group( + "[bright_red][bold]Error parsing" + f" {e.arg.lowered.name_or_flag if isinstance(e.arg, _arguments.ArgumentDefinition) else e.arg}[/bold]:[/bright_red] {e.message}", + *cast( # Cast to appease mypy... + List[RenderableType], + ( + [] + if not isinstance(e.arg, _arguments.ArgumentDefinition) + or e.arg.lowered.help is None + else [ + Rule(style=Style(color="red")), + "Argument helptext:", + Padding( + Group( + f"{e.arg.lowered.name_or_flag} [bold]{e.arg.lowered.metavar}[/bold]", + e.arg.lowered.help, + ), + pad=(0, 0, 0, 4), ), - pad=(0, 0, 0, 4), - ), - Rule(style=Style(color="red")), - f"For full helptext, see [bold]{parser.prog} --help[/bold]", - ] + Rule(style=Style(color="red")), + f"For full helptext, see [bold]{parser.prog} --help[/bold]", + ] + ), ), ), - ), - title="[bold]Value error[/bold]", - title_align="left", - border_style=Style(color="red"), + title="[bold]Value error[/bold]", + title_align="left", + border_style=Style(color="red"), + ) ) - ) sys.exit(2) assert len(value_from_prefixed_field_name.keys() - consumed_keywords) == 0, ( diff --git a/src/tyro/_docstrings.py b/src/tyro/_docstrings.py index 88a2262c2..714abe38d 100644 --- a/src/tyro/_docstrings.py +++ b/src/tyro/_docstrings.py @@ -7,7 +7,7 @@ import io import itertools import tokenize -from typing import Callable, Dict, Generic, Hashable, List, Optional, Type, TypeVar +from typing import Callable, Dict, Generic, Hashable, List, Optional, Set, Type, TypeVar import docstring_parser from typing_extensions import get_origin, is_typeddict @@ -281,7 +281,7 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: return None -_callable_description_blocklist = set( +_callable_description_blocklist: Set[Hashable] = set( filter( lambda x: isinstance(x, Hashable), # type: ignore itertools.chain(__builtins__.values(), vars(collections.abc).values()), # type: ignore diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index a9d3d9fc5..beddffa08 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -25,6 +25,7 @@ Iterable, List, Optional, + Set, Tuple, TypeVar, Union, @@ -361,7 +362,7 @@ def resolve(field: FieldDefinition) -> FieldDefinition: Any, PropagatingMissingType, NonpropagatingMissingType, ExcludeFromCallType ] -_known_parsable_types = set( +_known_parsable_types: Set[type] = set( filter( lambda x: isinstance(x, Hashable), # type: ignore itertools.chain( diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index 115b6f304..67bf44f6a 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -103,7 +103,7 @@ class UnsupportedTypeAnnotationError(Exception): """Exception raised when an unsupported type annotation is detected.""" -_builtin_set = set( +_builtin_set: Set[Hashable] = set( filter( lambda x: isinstance(x, Hashable), # type: ignore __builtins__.values(), # type: ignore diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 5a146037f..ca623f0b2 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -629,6 +629,7 @@ def apply( assert isinstance(parent_parser, _argparse_formatter.TyroArgumentParser) subparser._parsing_known_args = parent_parser._parsing_known_args subparser._parser_specification = parent_parser._parser_specification + subparser._console_outputs = parent_parser._console_outputs subparser._args = parent_parser._args subparser_tree_leaves.extend(subparser_def.apply(subparser)) diff --git a/tests/conftest.py b/tests/conftest.py index 31dfb77f0..23d01aae4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,15 +5,19 @@ if not sys.version_info >= (3, 8): collect_ignore_glob.append("*_min_py38.py") + collect_ignore_glob.append("*_min_py38_generated.py") if not sys.version_info >= (3, 9): collect_ignore_glob.append("*_min_py39.py") + collect_ignore_glob.append("*_min_py39_generated.py") if not sys.version_info >= (3, 10): collect_ignore_glob.append("*_min_py310.py") + collect_ignore_glob.append("*_min_py310_generated.py") if not sys.version_info >= (3, 12): collect_ignore_glob.append("*_min_py312.py") + collect_ignore_glob.append("*_min_py312_generated.py") if not sys.version_info >= (3, 11): collect_ignore_glob.append("test_py311_generated/*.py") diff --git a/tests/helptext_utils.py b/tests/helptext_utils.py index 9857b76ac..041f5c58b 100644 --- a/tests/helptext_utils.py +++ b/tests/helptext_utils.py @@ -11,15 +11,25 @@ import tyro._strings -def get_helptext( +def get_helptext_with_checks( f: Callable, args: List[str] = ["--help"], use_underscores: bool = False, default: Any = None, ) -> str: + """Get the helptext for a given tyro with input, while running various + checks along the way.""" + # Should be empty. target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): - tyro.cli(f, args=args, use_underscores=use_underscores, default=default) + tyro.cli( + f, + args=args, + use_underscores=use_underscores, + default=default, + console_outputs=False, + ) + assert target.getvalue() == "" # Check tyro.extras.get_parser(). parser = tyro.extras.get_parser(f, use_underscores=use_underscores) @@ -36,8 +46,10 @@ def get_helptext( # Completion scripts; just smoke test for now. with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): + # `--tyro-print-completion` is deprecated! We should use `--tyro-write-completion` instead. tyro.cli(f, default=default, args=["--tyro-print-completion", "bash"]) with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): + # `--tyro-print-completion` is deprecated! We should use `--tyro-write-completion` instead. tyro.cli(f, default=default, args=["--tyro-print-completion", "zsh"]) with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): tyro.cli( @@ -48,6 +60,11 @@ def get_helptext( f, default=default, args=["--tyro-write-completion", "zsh", os.devnull] ) + # Get the actual helptext. + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(f, args=args, use_underscores=use_underscores, default=default) + # Check helptext with vs without formatting. This can help catch text wrapping bugs # caused by ANSI sequences. target2 = io.StringIO() diff --git a/tests/test_collections.py b/tests/test_collections.py index e4aa3fe80..57a693419 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -79,7 +79,7 @@ class A: assert tyro.cli(A, args=[]) == A(x=(1, 2)) target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(A, args=["1", "2", "3", "4"]) assert "invalid choice" in target.getvalue() diff --git a/tests/test_conf.py b/tests/test_conf.py index 764b01d62..4cdfa4afb 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -8,7 +8,7 @@ from typing import Any, Dict, Generic, List, Tuple, Type, TypeVar, Union import pytest -from helptext_utils import get_helptext +from helptext_utils import get_helptext_with_checks from typing_extensions import Annotated import tyro @@ -32,7 +32,7 @@ class DefaultInstanceSubparser: Union[DefaultInstanceHTTPServer, DefaultInstanceSMTPServer] ] = dataclasses.field(default_factory=DefaultInstanceHTTPServer) - assert "bc" not in get_helptext(DefaultInstanceSubparser) + assert "bc" not in get_helptext_with_checks(DefaultInstanceSubparser) def test_omit_subcommand_prefix() -> None: @@ -414,19 +414,19 @@ class Nested: def main_one(x: Nested = Nested(default_one)) -> None: pass - assert "default: x.subcommand:one" in get_helptext(main_one) + assert "default: x.subcommand:one" in get_helptext_with_checks(main_one) # Match by value. def main_two(x: Nested = Nested(B(9))) -> None: pass - assert "default: x.subcommand:three" in get_helptext(main_two) + assert "default: x.subcommand:three" in get_helptext_with_checks(main_two) # Match by type. def main_three(x: Nested = Nested(B(15))) -> None: pass - assert "default: x.subcommand:one" in get_helptext(main_three) + assert "default: x.subcommand:one" in get_helptext_with_checks(main_three) def test_flag() -> None: @@ -551,7 +551,7 @@ class Struct: def main(x: Any = Struct()): pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x.a" in helptext assert "--x.b" not in helptext @@ -565,7 +565,7 @@ class Struct: def main(x: Any = Struct()): pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x.a" in helptext assert "--x.b" not in helptext @@ -578,7 +578,7 @@ class Struct: def main(x: Any = Struct()): pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x.a" not in helptext assert "--x.b" not in helptext @@ -594,7 +594,7 @@ def b(self, x): def main(x: tyro.conf.SuppressFixed[Any] = Struct()): pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x.a" in helptext assert "--x.b" not in helptext @@ -610,7 +610,7 @@ class Struct: def main(x: Any = Struct()) -> int: return x.a - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "Hello world" in helptext assert "INT" not in helptext assert "NUMBER" in helptext @@ -636,7 +636,7 @@ class Struct: def main(x: Any = Struct()) -> int: return x.a - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "Hello world" in helptext assert "INT" not in helptext assert "NUMBER" in helptext @@ -1050,7 +1050,7 @@ class Config: ) == Config(x={"hello": "world"}) target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Config, args="--x.json 5".split(" ")) error = target.getvalue() @@ -1105,7 +1105,7 @@ class Config: # --x.a and --x.b are required! target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Config, args="--x.c 5".split(" ")) error = target.getvalue() assert "We're missing" in error @@ -1139,7 +1139,7 @@ class Config: # --x.struct.a and --x.struct.b are required! target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Config, args="--x.struct.c 5".split(" ")) error = target.getvalue() assert "We're missing arguments" in error @@ -1173,7 +1173,7 @@ class Config: # --x.struct.a and --x.struct.b are required! target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Config, args="--x.struct.b 5".split(" ")) error = target.getvalue() assert "We're missing arguments" in error @@ -1217,14 +1217,14 @@ class Config: # --x.struct.a and --x.struct.b are required! target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Config, args="--x.struct.b 5".split(" ")) error = target.getvalue() assert "We're missing arguments" in error assert "'a'" in error assert "'b'" not in error - assert "--x.struct.a INT, --all INT, -d INT" in get_helptext(Config) + assert "--x.struct.a INT, --all INT, -d INT" in get_helptext_with_checks(Config) def test_positional_alias() -> None: diff --git a/tests/test_errors.py b/tests/test_errors.py index 30b6900ae..4b522ed35 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -167,7 +167,7 @@ class Class: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Class, args="--reward.trac".split(" ")) error = target.getvalue() @@ -178,6 +178,23 @@ class Class: assert error.count("--help") == 1 +def test_suppress_console_outputs() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + + @dataclasses.dataclass + class Class: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): + tyro.cli(Class, args="--reward.trac".split(" "), console_outputs=False) + + error = target.getvalue() + assert error == "" + + def test_similar_arguments_subcommands() -> None: @dataclasses.dataclass class RewardConfig: @@ -192,7 +209,7 @@ class ClassB: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Union[ClassA, ClassB], args="--reward.trac".split(" ")) # type: ignore error = target.getvalue() @@ -217,7 +234,7 @@ class ClassB: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Union[ClassA, ClassB], args="--fjdkslaj --reward.trac".split(" ")) # type: ignore error = target.getvalue() @@ -243,7 +260,7 @@ class ClassB: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Union[ClassA, ClassB], args="--rd.trac".split(" ")) # type: ignore error = target.getvalue() @@ -269,7 +286,7 @@ class ClassB: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Union[ClassA, ClassB], args="--track".split(" ")) # type: ignore error = target.getvalue() @@ -311,7 +328,7 @@ class ClassB: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Union[ClassA, ClassB], args="--track".split(" ")) # type: ignore error = target.getvalue() @@ -322,7 +339,7 @@ class ClassB: assert error.count("--help") == 21 target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): # --tracked is intentionally between 0.8 ~ 0.9 similarity to track{i} for test # coverage. tyro.cli(RewardConfig, args="--tracked".split(" ")) # type: ignore @@ -374,7 +391,7 @@ class ClassI: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli( # type: ignore Union[ ClassA, ClassB, ClassC, ClassD, ClassE, ClassF, ClassG, ClassH, ClassI @@ -433,7 +450,7 @@ class ClassI: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli( # type: ignore Union[ ClassA, ClassB, ClassC, ClassD, ClassE, ClassF, ClassG, ClassH, ClassI @@ -456,7 +473,7 @@ class Args: flag: bool = False target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli( Args, args="--lag".split(" "), diff --git a/tests/test_flax_min_py38.py b/tests/test_flax_min_py38.py index 056f70949..ab450c77a 100644 --- a/tests/test_flax_min_py38.py +++ b/tests/test_flax_min_py38.py @@ -5,7 +5,7 @@ import jax import pytest from flax import linen as nn -from helptext_utils import get_helptext +from helptext_utils import get_helptext_with_checks from jax import numpy as jnp import tyro @@ -50,7 +50,7 @@ def test_ok(): params = network.init(jax.random.PRNGKey(0), x) assert cast(jax.Array, network.apply(params, x)).shape == (10, 3) - helptext = get_helptext(Classifier) + helptext = get_helptext_with_checks(Classifier) assert "parent" not in helptext assert "name" not in helptext diff --git a/tests/test_functools.py b/tests/test_functools.py index 850c73032..f1a0aaf06 100644 --- a/tests/test_functools.py +++ b/tests/test_functools.py @@ -4,7 +4,7 @@ import functools import pytest -from helptext_utils import get_helptext +from helptext_utils import get_helptext_with_checks import tyro @@ -29,7 +29,7 @@ def main(a: int, b: str) -> str: """Hello!""" return b * a - helptext = get_helptext(functools.partial(main, b="hello world")) + helptext = get_helptext_with_checks(functools.partial(main, b="hello world")) assert "partial" not in helptext assert "Hello!" in helptext assert "hello world" in helptext @@ -42,7 +42,7 @@ class Main: def __init__(self, a: int, b: str) -> None: self.inner = b * a - helptext = get_helptext(functools.partial(Main, b="3")) + helptext = get_helptext_with_checks(functools.partial(Main, b="3")) assert "partial" not in helptext assert "Hello!" in helptext @@ -76,7 +76,7 @@ def wrapper(*args, **kwargs) -> int: assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == 3 - helptext = get_helptext(functools.partial(wrapper, b="3")) + helptext = get_helptext_with_checks(functools.partial(wrapper, b="3")) assert "wraps" not in helptext assert "Hello!" in helptext assert "Argument." in helptext @@ -95,7 +95,7 @@ def wrapper(*args, **kwargs) -> int: assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == 3 - helptext = get_helptext(functools.partial(wrapper, b="3")) + helptext = get_helptext_with_checks(functools.partial(wrapper, b="3")) assert "wraps" not in helptext assert "Hello!" in helptext @@ -127,7 +127,7 @@ def wrapper(*args, **kwargs) -> str: == "hellohellohello" ) - helptext = get_helptext(functools.partial(wrapper, b="3")) + helptext = get_helptext_with_checks(functools.partial(wrapper, b="3")) assert "wraps" not in helptext assert "Hello!" in helptext assert "Second field." in helptext diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 8aebd0880..75c79d178 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -6,7 +6,7 @@ from collections.abc import Callable from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar, Union, cast -from helptext_utils import get_helptext +from helptext_utils import get_helptext_with_checks from torch import nn from typing_extensions import Annotated, Literal, NotRequired, TypedDict @@ -26,7 +26,7 @@ class Helptext: z: int = 3 """Documentation 3""" - helptext = get_helptext(Helptext) + helptext = get_helptext_with_checks(Helptext) assert cast(str, helptext) in helptext assert "x INT" in helptext assert "y INT" in helptext @@ -47,7 +47,7 @@ class Helptext: y: Annotated[int, "ignored"] z: int = 3 - helptext = get_helptext(Helptext) + helptext = get_helptext_with_checks(Helptext) assert cast(str, helptext) in helptext assert "x INT" in helptext assert "y INT" in helptext @@ -76,7 +76,7 @@ class Helptext2: y: Annotated[int, "ignored"] z: int = 3 - helptext = get_helptext(Helptext2) + helptext = get_helptext_with_checks(Helptext2) assert "This docstring should be printed as a description" in helptext assert "Attributes" not in helptext assert "x INT" in helptext @@ -102,7 +102,7 @@ class Helptext3: y: Annotated[int, "ignored"] z: int = 3 - helptext = get_helptext(Helptext3) + helptext = get_helptext_with_checks(Helptext3) assert "This docstring should be printed as a description" in helptext assert "Args" not in helptext assert "x INT" in helptext @@ -137,7 +137,7 @@ def some_method(self) -> None: # noqa class ChildClass(UnrelatedParentClass, ActualParentClass): pass - helptext = get_helptext(ChildClass) + helptext = get_helptext_with_checks(ChildClass) assert "Documentation 1" in helptext assert "Documentation 2" in helptext @@ -167,7 +167,7 @@ def main(x: ParentClass = ChildClass(x=5, y=5)) -> Any: """Main function.""" return x - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "Main function." in helptext assert "Documentation 1" in helptext assert "Documentation 2" in helptext @@ -197,14 +197,14 @@ def main_with_docstring(a: Inner) -> None: def main_no_docstring(a: Inner) -> None: """main_no_docstring.""" - helptext = get_helptext(main_with_docstring) + helptext = get_helptext_with_checks(main_with_docstring) assert "Documented in function" in helptext and str(Inner.__doc__) not in helptext assert "main_with_docstring." in helptext assert "Args:" not in helptext assert "Args:" not in helptext assert "Hello world!" in helptext - helptext = get_helptext(main_no_docstring) + helptext = get_helptext_with_checks(main_no_docstring) assert "Something" in helptext assert "main_no_docstring." in helptext assert "Args:" not in helptext @@ -223,7 +223,7 @@ class HelptextWithVariousDefaults: y: Color = Color.RED z: str = "%" - helptext = get_helptext(HelptextWithVariousDefaults) + helptext = get_helptext_with_checks(HelptextWithVariousDefaults) assert "show this help message and exit" in helptext assert "--x PATH" in helptext assert "(default: /some/path/to/a/file)" in helptext @@ -248,7 +248,7 @@ class HelptextMultiline: """Documentation 3 Next line of documentation 3""" - helptext = get_helptext(HelptextMultiline) + helptext = get_helptext_with_checks(HelptextMultiline) assert "Documentation 1 (required)" in helptext assert "Documentation 2" in helptext assert "documentation 2" in helptext @@ -264,7 +264,7 @@ class HelptextGrouped: y: int z: int = 3 - helptext = get_helptext(HelptextGrouped) + helptext = get_helptext_with_checks(HelptextGrouped) assert "Documentation 1 (required)" in helptext assert "Description of both y and z. (required)" in helptext assert "Description of both y and z. (default: 3)" in helptext @@ -276,7 +276,7 @@ class Config: x: Optional[int] = None """An optional variable.""" - helptext = get_helptext(Config) + helptext = get_helptext_with_checks(Config) assert "--x {None}|INT" in helptext assert "An optional variable. (default: None)" in helptext @@ -294,7 +294,7 @@ class HelptextHardString: # Note that the percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string - helptext = get_helptext(HelptextHardString) + helptext = get_helptext_with_checks(HelptextHardString) assert "--x" in helptext assert "2% milk." in helptext @@ -313,7 +313,7 @@ class Parent: class Child(Parent): pass - helptext = get_helptext(Child) + helptext = get_helptext_with_checks(Child) assert "--x STR" in helptext assert "Helptext." in helptext assert "(default: 'This docstring" in helptext @@ -338,7 +338,7 @@ class Child2(Parent2): """Helptext!""" # fmt: on - helptext = get_helptext(Child2) + helptext = get_helptext_with_checks(Child2) assert "--x STR" in helptext assert "Helptext! (default: 'This" in helptext @@ -348,7 +348,7 @@ def test_tuple_helptext() -> None: class TupleHelptext: x: Tuple[int, str, float] - helptext = get_helptext(TupleHelptext) + helptext = get_helptext_with_checks(TupleHelptext) assert "--x INT STR FLOAT" in helptext @@ -357,7 +357,7 @@ def test_tuple_helptext_defaults() -> None: class TupleHelptextDefaults: x: Tuple[int, str, str] = (5, "hello world", "hello") - helptext = get_helptext(TupleHelptextDefaults) + helptext = get_helptext_with_checks(TupleHelptextDefaults) assert "--x INT STR STR" in helptext assert "(default: 5 'hello world' hello)" in helptext @@ -369,7 +369,7 @@ def test_generic_helptext() -> None: class GenericTupleHelptext(Generic[T]): x: T - helptext = get_helptext(GenericTupleHelptext[int]) + helptext = get_helptext_with_checks(GenericTupleHelptext[int]) assert "--x INT" in helptext @@ -380,7 +380,7 @@ def test_generic_tuple_helptext() -> None: class GenericTupleHelptext(Generic[T]): x: Tuple[T, T, T] - helptext = get_helptext(GenericTupleHelptext[int]) + helptext = get_helptext_with_checks(GenericTupleHelptext[int]) assert "--x INT INT INT" in helptext @@ -391,7 +391,7 @@ def test_generic_list_helptext() -> None: class GenericTupleHelptext(Generic[T]): x: List[T] - helptext = get_helptext(GenericTupleHelptext[int]) + helptext = get_helptext_with_checks(GenericTupleHelptext[int]) assert "--x [INT [INT ...]]" in helptext @@ -401,7 +401,7 @@ class LiteralHelptext: x: Literal[1, 2, 3] """A number.""" - helptext = get_helptext(LiteralHelptext) + helptext = get_helptext_with_checks(LiteralHelptext) assert "--x {1,2,3}" in helptext assert "A number. (required)" in helptext @@ -412,7 +412,7 @@ class OptionalLiteralHelptext: x: Optional[Literal[1, 2, 3]] = None """A number.""" - helptext = get_helptext(OptionalLiteralHelptext) + helptext = get_helptext_with_checks(OptionalLiteralHelptext) assert "--x {None,1,2,3}" in helptext assert "A number. (default: None)" in helptext @@ -445,7 +445,7 @@ class MultipleSubparsers: d: bool = False - helptext = get_helptext(MultipleSubparsers) + helptext = get_helptext_with_checks(MultipleSubparsers) assert "2% milk." in helptext assert "Field a description." in helptext @@ -456,7 +456,7 @@ class MultipleSubparsers: assert "[OPTIONS]" not in helptext assert "[B:SUBCOMMAND2 OPTIONS]" not in helptext - helptext = get_helptext( + helptext = get_helptext_with_checks( MultipleSubparsers, args=["a:subcommand1", "b:subcommand1", "--help"] ) @@ -516,7 +516,7 @@ class MultipleSubparsers: h: bool = False i: bool = False - helptext = get_helptext(MultipleSubparsers) + helptext = get_helptext_with_checks(MultipleSubparsers) assert "2% milk." in helptext assert "Field a description." in helptext @@ -526,7 +526,7 @@ class MultipleSubparsers: assert "[OPTIONS]" in helptext assert "[B:SUBCOMMAND2 OPTIONS]" not in helptext - helptext = get_helptext( + helptext = get_helptext_with_checks( MultipleSubparsers, args=["a:subcommand1", "b:subcommand1", "--help"] ) @@ -554,7 +554,7 @@ class OptionalHelptext: z: Optional[int] = 3 """Documentation 3""" - helptext = get_helptext(OptionalHelptext) + helptext = get_helptext_with_checks(OptionalHelptext) assert cast(str, cast(str, OptionalHelptext.__doc__)[:20]) in helptext assert "2% milk" in helptext assert "--x {None}|INT" in helptext @@ -566,7 +566,7 @@ def test_metavar_0() -> None: def main(x: Union[Literal[0, 1, 2, 3], Tuple[int, int]]) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x {0,1,2,3}|{INT INT}" in helptext @@ -581,7 +581,7 @@ def main( pass # The comma formatting is unfortunate, but matches argparse's default behavior. - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x {0,1,2,3,hey,there,hello}|{[INT [INT ...]]}" in helptext @@ -594,7 +594,7 @@ def main( ) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x {0,1,2,3} INT|STR" in helptext @@ -607,7 +607,7 @@ def main( ) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x {0,1,2,3}|{INT INT}|STR" in helptext @@ -621,7 +621,7 @@ def main( ) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x {0,1,2,3}|{INT INT}|{STR STR STR}|{True}" in helptext @@ -631,7 +631,7 @@ def main( ) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "[--x [{INT INT}|{STR STR} [{INT INT}|{STR STR} ...]]]" in helptext @@ -639,7 +639,7 @@ def test_metavar_6() -> None: def main(x: Dict[Union[Tuple[int, int], Tuple[str, str]], Tuple[int, int]]) -> dict: return x - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert ( "--x [{INT INT}|{STR STR} INT INT [{INT INT}|{STR STR} INT INT ...]]" in helptext @@ -657,7 +657,7 @@ class Something( # But this text should! b: int - helptext = get_helptext(Something) + helptext = get_helptext_with_checks(Something) assert "This text should not" not in helptext assert "But this text should!" in helptext @@ -670,13 +670,13 @@ class Struct: def main(x: Any = Struct()): pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x {fixed}" in helptext def main2(x: Callable = nn.ReLU): pass - helptext = get_helptext(main2) + helptext = get_helptext_with_checks(main2) assert "--x {fixed}" in helptext assert "(fixed to:" in helptext assert "torch" in helptext @@ -686,7 +686,7 @@ def test_pathlike() -> None: def main(x: os.PathLike) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x PATH " in helptext @@ -698,7 +698,7 @@ class Child: def main(child: Child) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--child.x | --child.no-x" in helptext @@ -729,14 +729,14 @@ class MultipleSubparsers: default_factory=SubcommandThree ) - helptext = get_helptext(MultipleSubparsers) + helptext = get_helptext_with_checks(MultipleSubparsers) assert "2% milk." in helptext assert "Field a description." in helptext assert "Field b description." not in helptext assert "Field c description." not in helptext - helptext = get_helptext( + helptext = get_helptext_with_checks( MultipleSubparsers, args=["a:subcommand-one", "b:subcommand-one", "--help"] ) @@ -777,14 +777,14 @@ class MultipleSubparsers: default_factory=SubcommandThree ) - helptext = get_helptext(MultipleSubparsers, use_underscores=True) + helptext = get_helptext_with_checks(MultipleSubparsers, use_underscores=True) assert "2% milk." in helptext assert "Field a description." in helptext assert "Field b description." not in helptext assert "Field c description." not in helptext - helptext = get_helptext( + helptext = get_helptext_with_checks( MultipleSubparsers, args=["a:subcommand_one", "b:subcommand_one", "--help"], use_underscores=True, @@ -813,7 +813,7 @@ class CheckoutCompletion: y: int - help = get_helptext(Union[A, CheckoutCompletion]) # type: ignore + help = get_helptext_with_checks(Union[A, CheckoutCompletion]) # type: ignore assert help.count("checkout-completion") == 3 @@ -830,7 +830,7 @@ class CheckoutCompletio: y: int - help = get_helptext(Union[A, CheckoutCompletio]) # type: ignore + help = get_helptext_with_checks(Union[A, CheckoutCompletio]) # type: ignore assert help.count("checkout-completio") == 3 @@ -847,7 +847,7 @@ class CheckoutCompletionn: y: int - help = get_helptext(Union[A, CheckoutCompletionn]) # type: ignore + help = get_helptext_with_checks(Union[A, CheckoutCompletionn]) # type: ignore assert help.count("checkout-completionn") == 3 @@ -864,7 +864,7 @@ class CmdCheckout012: y: int - help = get_helptext(Union[A, CmdCheckout012]) # type: ignore + help = get_helptext_with_checks(Union[A, CmdCheckout012]) # type: ignore assert help.count("cmd-checkout012") == 3 @@ -875,7 +875,7 @@ class A: x: Tuple[str, str] = ("hello", "world") - help = get_helptext(A) + help = get_helptext_with_checks(A) assert "STR STR" in help assert "hello world" in help assert "('hello', 'world')" not in help @@ -890,7 +890,7 @@ class A: Tuple[str, str], tyro.conf.arg(constructor=lambda x: ("a", "b")) ] = ("hello", "world") - help = get_helptext(A) + help = get_helptext_with_checks(A) assert "STR STR" not in help # Unlike case above, should not be converted to 'hello world'. assert "('hello', 'world')" in help # JSON special case. @@ -908,7 +908,7 @@ class A: Tuple[int, int], tyro.conf.arg(constructor=json.loads, metavar="JSON") ] = (3, 5) - help = get_helptext(A) + help = get_helptext_with_checks(A) assert "STR STR" not in help assert "JSON" in help assert '["hello", "world"]' in help # JSON special case. @@ -925,7 +925,7 @@ def f( del x, y return 5 - help = get_helptext(f, default=3) + help = get_helptext_with_checks(f, default=3) assert 'default if used: ["hello", "world"]' in help assert "default if used: 3" in help @@ -938,7 +938,7 @@ def f( del x, y return 5 - help = get_helptext(f) + help = get_helptext_with_checks(f) assert "repeatable" not in help, help @@ -950,7 +950,7 @@ def f( del x, y return 5 - help = get_helptext(f) + help = get_helptext_with_checks(f) assert "repeatable" in help, help @@ -962,7 +962,7 @@ def f( del x, y return 5 - help = get_helptext(f) + help = get_helptext_with_checks(f) assert "repeatable, appends to: 'hello world' hello" in help, help @@ -970,5 +970,5 @@ def test_typeddict_exclude() -> None: class Special(TypedDict): x: NotRequired[int] - help = get_helptext(Special) + help = get_helptext_with_checks(Special) assert "unset by default" in help, help diff --git a/tests/test_missing.py b/tests/test_missing.py index 223eb2560..3f7f9d428 100644 --- a/tests/test_missing.py +++ b/tests/test_missing.py @@ -15,7 +15,7 @@ def main(a: int = 5, b: int = tyro.MISSING, c: int = 3) -> Tuple[int, int, int]: return a, b, c target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(main, args=[]) message = target.getvalue() assert "Required options" in message diff --git a/tests/test_nested.py b/tests/test_nested.py index 21fa874c1..0d7beb64a 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -3,7 +3,7 @@ import pytest from frozendict import frozendict # type: ignore -from helptext_utils import get_helptext +from helptext_utils import get_helptext_with_checks from typing_extensions import Annotated, Literal import tyro @@ -1161,4 +1161,4 @@ class Args: ] = A("hello") assert tyro.cli(Args, args=[]) == Args(A("hello")) - assert "default: inner:alt" in get_helptext(Args) + assert "default: inner:alt" in get_helptext_with_checks(Args) diff --git a/tests/test_new_style_annotations_generics.py b/tests/test_new_style_annotations_generics.py new file mode 100644 index 000000000..bf365f0f7 --- /dev/null +++ b/tests/test_new_style_annotations_generics.py @@ -0,0 +1,55 @@ +from __future__ import annotations # Should enable support for all versions of Python. + +from typing import Any, Optional, Union + +import pytest +from typing_extensions import Literal + +import tyro + + +def test_list() -> None: + def main(x: list[bool]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] + + +def test_tuple() -> None: + def main(x: tuple[bool, str]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "True", "False"]) == (True, "False") + + +def test_tuple_variable() -> None: + def main(x: tuple[Union[bool, str], ...]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "True", "Wrong"]) == (True, "Wrong") + + +def test_super_nested() -> None: + def main( + x: Optional[ + list[ + tuple[ + Optional[int], + Literal[3, 4], + Union[tuple[int, int], tuple[str, str]], + ] + ] + ] = None, + ) -> Any: + return x + + assert tyro.cli(main, args=[]) is None + assert tyro.cli(main, args="--x None".split(" ")) is None + assert tyro.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] + assert tyro.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] + assert tyro.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ + (2, 3, ("x", "2")), + (2, 3, (1, 2)), + ] + with pytest.raises(SystemExit): + tyro.cli(main, args=["--help"]) diff --git a/tests/test_new_style_annotations_unions.py b/tests/test_new_style_annotations_unions.py new file mode 100644 index 000000000..22b38ab72 --- /dev/null +++ b/tests/test_new_style_annotations_unions.py @@ -0,0 +1,62 @@ +from __future__ import annotations # Should enable support for all versions of Python. + +from typing import Any + +import pytest +from typing_extensions import Literal + +import tyro + + +def test_union_basic(): + def main(x: int | str) -> int | str: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == 6 + assert tyro.cli(main, args=["--x", "five"]) == "five" + + +def test_union_with_list(): + def main(x: int | str | list[bool]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == 6 + assert tyro.cli(main, args=["--x", "five"]) == "five" + assert tyro.cli(main, args=["--x", "True"]) == "True" + assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] + + +def test_union_literal(): + def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == "6" + assert tyro.cli(main, args=["--x", "five"]) == "five" + + +def test_super_nested(): + def main( + x: None + | list[ + tuple[ + None | int, + Literal[3, 4], + tuple[int, int] | tuple[str, str], + ] + ] = None, + ) -> Any: + return x + + assert tyro.cli(main, args=[]) is None + assert tyro.cli(main, args="--x None".split(" ")) is None + assert tyro.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] + assert tyro.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] + assert tyro.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ + (2, 3, ("x", "2")), + (2, 3, (1, 2)), + ] + with pytest.raises(SystemExit): + tyro.cli(main, args=["--help"]) diff --git a/tests/test_partial.py b/tests/test_partial.py index 86a017451..89fe78552 100644 --- a/tests/test_partial.py +++ b/tests/test_partial.py @@ -3,7 +3,7 @@ import dataclasses import functools -from helptext_utils import get_helptext +from helptext_utils import get_helptext_with_checks import tyro @@ -28,7 +28,7 @@ def main(a: int, b: str) -> str: """Hello!""" return b * a - helptext = get_helptext(functools.partial(main, b="3")) + helptext = get_helptext_with_checks(functools.partial(main, b="3")) assert "partial" not in helptext assert "Hello!" in helptext @@ -40,7 +40,7 @@ class Main: def __init__(self, a: int, b: str) -> None: self.inner = b * a - helptext = get_helptext(functools.partial(Main, b="3")) + helptext = get_helptext_with_checks(functools.partial(Main, b="3")) assert "partial" not in helptext assert "Hello!" in helptext @@ -56,6 +56,6 @@ class Config: ConfigWithDefaults = functools.partial(functools.partial(Config, a=3), c=5) assert tyro.cli(ConfigWithDefaults, args=["--b", "4"]) == Config(3, 4, 5) - helptext = get_helptext(ConfigWithDefaults) + helptext = get_helptext_with_checks(ConfigWithDefaults) assert "partial" not in helptext assert "Hello!" in helptext diff --git a/tests/test_py311_generated/test_collections_generated.py b/tests/test_py311_generated/test_collections_generated.py index 2b35772a2..ff03ac8df 100644 --- a/tests/test_py311_generated/test_collections_generated.py +++ b/tests/test_py311_generated/test_collections_generated.py @@ -78,7 +78,7 @@ class A: assert tyro.cli(A, args=[]) == A(x=(1, 2)) target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(A, args=["1", "2", "3", "4"]) assert "invalid choice" in target.getvalue() diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index 944536437..bb737fb07 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -8,7 +8,7 @@ from typing import Annotated, Any, Dict, Generic, List, Tuple, Type, TypeVar import pytest -from helptext_utils import get_helptext +from helptext_utils import get_helptext_with_checks import tyro @@ -31,7 +31,7 @@ class DefaultInstanceSubparser: DefaultInstanceHTTPServer | DefaultInstanceSMTPServer ] = dataclasses.field(default_factory=DefaultInstanceHTTPServer) - assert "bc" not in get_helptext(DefaultInstanceSubparser) + assert "bc" not in get_helptext_with_checks(DefaultInstanceSubparser) def test_omit_subcommand_prefix() -> None: @@ -411,19 +411,19 @@ class Nested: def main_one(x: Nested = Nested(default_one)) -> None: pass - assert "default: x.subcommand:one" in get_helptext(main_one) + assert "default: x.subcommand:one" in get_helptext_with_checks(main_one) # Match by value. def main_two(x: Nested = Nested(B(9))) -> None: pass - assert "default: x.subcommand:three" in get_helptext(main_two) + assert "default: x.subcommand:three" in get_helptext_with_checks(main_two) # Match by type. def main_three(x: Nested = Nested(B(15))) -> None: pass - assert "default: x.subcommand:one" in get_helptext(main_three) + assert "default: x.subcommand:one" in get_helptext_with_checks(main_three) def test_flag() -> None: @@ -548,7 +548,7 @@ class Struct: def main(x: Any = Struct()): pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x.a" in helptext assert "--x.b" not in helptext @@ -562,7 +562,7 @@ class Struct: def main(x: Any = Struct()): pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x.a" in helptext assert "--x.b" not in helptext @@ -575,7 +575,7 @@ class Struct: def main(x: Any = Struct()): pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x.a" not in helptext assert "--x.b" not in helptext @@ -591,7 +591,7 @@ def b(self, x): def main(x: tyro.conf.SuppressFixed[Any] = Struct()): pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x.a" in helptext assert "--x.b" not in helptext @@ -607,7 +607,7 @@ class Struct: def main(x: Any = Struct()) -> int: return x.a - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "Hello world" in helptext assert "INT" not in helptext assert "NUMBER" in helptext @@ -633,7 +633,7 @@ class Struct: def main(x: Any = Struct()) -> int: return x.a - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "Hello world" in helptext assert "INT" not in helptext assert "NUMBER" in helptext @@ -1047,7 +1047,7 @@ class Config: ) == Config(x={"hello": "world"}) target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Config, args="--x.json 5".split(" ")) error = target.getvalue() @@ -1102,7 +1102,7 @@ class Config: # --x.a and --x.b are required! target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Config, args="--x.c 5".split(" ")) error = target.getvalue() assert "We're missing" in error @@ -1136,7 +1136,7 @@ class Config: # --x.struct.a and --x.struct.b are required! target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Config, args="--x.struct.c 5".split(" ")) error = target.getvalue() assert "We're missing arguments" in error @@ -1170,7 +1170,7 @@ class Config: # --x.struct.a and --x.struct.b are required! target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Config, args="--x.struct.b 5".split(" ")) error = target.getvalue() assert "We're missing arguments" in error @@ -1214,14 +1214,14 @@ class Config: # --x.struct.a and --x.struct.b are required! target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Config, args="--x.struct.b 5".split(" ")) error = target.getvalue() assert "We're missing arguments" in error assert "'a'" in error assert "'b'" not in error - assert "--x.struct.a INT, --all INT, -d INT" in get_helptext(Config) + assert "--x.struct.a INT, --all INT, -d INT" in get_helptext_with_checks(Config) def test_positional_alias() -> None: diff --git a/tests/test_py311_generated/test_errors_generated.py b/tests/test_py311_generated/test_errors_generated.py index e48201d8f..08facc3ca 100644 --- a/tests/test_py311_generated/test_errors_generated.py +++ b/tests/test_py311_generated/test_errors_generated.py @@ -166,7 +166,7 @@ class Class: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Class, args="--reward.trac".split(" ")) error = target.getvalue() @@ -177,6 +177,23 @@ class Class: assert error.count("--help") == 1 +def test_suppress_console_outputs() -> None: + @dataclasses.dataclass + class RewardConfig: + track: bool + + @dataclasses.dataclass + class Class: + reward: RewardConfig + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): + tyro.cli(Class, args="--reward.trac".split(" "), console_outputs=False) + + error = target.getvalue() + assert error == "" + + def test_similar_arguments_subcommands() -> None: @dataclasses.dataclass class RewardConfig: @@ -191,7 +208,7 @@ class ClassB: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(ClassA | ClassB, args="--reward.trac".split(" ")) # type: ignore error = target.getvalue() @@ -216,7 +233,7 @@ class ClassB: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(ClassA | ClassB, args="--fjdkslaj --reward.trac".split(" ")) # type: ignore error = target.getvalue() @@ -242,7 +259,7 @@ class ClassB: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(ClassA | ClassB, args="--rd.trac".split(" ")) # type: ignore error = target.getvalue() @@ -268,7 +285,7 @@ class ClassB: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(ClassA | ClassB, args="--track".split(" ")) # type: ignore error = target.getvalue() @@ -310,7 +327,7 @@ class ClassB: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(ClassA | ClassB, args="--track".split(" ")) # type: ignore error = target.getvalue() @@ -321,7 +338,7 @@ class ClassB: assert error.count("--help") == 21 target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): # --tracked is intentionally between 0.8 ~ 0.9 similarity to track{i} for test # coverage. tyro.cli(RewardConfig, args="--tracked".split(" ")) # type: ignore @@ -373,7 +390,7 @@ class ClassI: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli( # type: ignore ClassA | ClassB @@ -438,7 +455,7 @@ class ClassI: reward: RewardConfig target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli( # type: ignore ClassA | ClassB @@ -467,7 +484,7 @@ class Args: flag: bool = False target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli( Args, args="--lag".split(" "), diff --git a/tests/test_py311_generated/test_flax_min_py38_generated.py b/tests/test_py311_generated/test_flax_min_py38_generated.py index 056f70949..ab450c77a 100644 --- a/tests/test_py311_generated/test_flax_min_py38_generated.py +++ b/tests/test_py311_generated/test_flax_min_py38_generated.py @@ -5,7 +5,7 @@ import jax import pytest from flax import linen as nn -from helptext_utils import get_helptext +from helptext_utils import get_helptext_with_checks from jax import numpy as jnp import tyro @@ -50,7 +50,7 @@ def test_ok(): params = network.init(jax.random.PRNGKey(0), x) assert cast(jax.Array, network.apply(params, x)).shape == (10, 3) - helptext = get_helptext(Classifier) + helptext = get_helptext_with_checks(Classifier) assert "parent" not in helptext assert "name" not in helptext diff --git a/tests/test_py311_generated/test_functools_generated.py b/tests/test_py311_generated/test_functools_generated.py index 850c73032..f1a0aaf06 100644 --- a/tests/test_py311_generated/test_functools_generated.py +++ b/tests/test_py311_generated/test_functools_generated.py @@ -4,7 +4,7 @@ import functools import pytest -from helptext_utils import get_helptext +from helptext_utils import get_helptext_with_checks import tyro @@ -29,7 +29,7 @@ def main(a: int, b: str) -> str: """Hello!""" return b * a - helptext = get_helptext(functools.partial(main, b="hello world")) + helptext = get_helptext_with_checks(functools.partial(main, b="hello world")) assert "partial" not in helptext assert "Hello!" in helptext assert "hello world" in helptext @@ -42,7 +42,7 @@ class Main: def __init__(self, a: int, b: str) -> None: self.inner = b * a - helptext = get_helptext(functools.partial(Main, b="3")) + helptext = get_helptext_with_checks(functools.partial(Main, b="3")) assert "partial" not in helptext assert "Hello!" in helptext @@ -76,7 +76,7 @@ def wrapper(*args, **kwargs) -> int: assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == 3 - helptext = get_helptext(functools.partial(wrapper, b="3")) + helptext = get_helptext_with_checks(functools.partial(wrapper, b="3")) assert "wraps" not in helptext assert "Hello!" in helptext assert "Argument." in helptext @@ -95,7 +95,7 @@ def wrapper(*args, **kwargs) -> int: assert tyro.cli(functools.partial(wrapper, a=3), args=["--b", "hi"]) == 3 - helptext = get_helptext(functools.partial(wrapper, b="3")) + helptext = get_helptext_with_checks(functools.partial(wrapper, b="3")) assert "wraps" not in helptext assert "Hello!" in helptext @@ -127,7 +127,7 @@ def wrapper(*args, **kwargs) -> str: == "hellohellohello" ) - helptext = get_helptext(functools.partial(wrapper, b="3")) + helptext = get_helptext_with_checks(functools.partial(wrapper, b="3")) assert "wraps" not in helptext assert "Hello!" in helptext assert "Second field." in helptext diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py index d945804b3..075fdac46 100644 --- a/tests/test_py311_generated/test_helptext_generated.py +++ b/tests/test_py311_generated/test_helptext_generated.py @@ -19,7 +19,7 @@ cast, ) -from helptext_utils import get_helptext +from helptext_utils import get_helptext_with_checks from torch import nn import tyro @@ -38,7 +38,7 @@ class Helptext: z: int = 3 """Documentation 3""" - helptext = get_helptext(Helptext) + helptext = get_helptext_with_checks(Helptext) assert cast(str, helptext) in helptext assert "x INT" in helptext assert "y INT" in helptext @@ -59,7 +59,7 @@ class Helptext: y: Annotated[int, "ignored"] z: int = 3 - helptext = get_helptext(Helptext) + helptext = get_helptext_with_checks(Helptext) assert cast(str, helptext) in helptext assert "x INT" in helptext assert "y INT" in helptext @@ -88,7 +88,7 @@ class Helptext2: y: Annotated[int, "ignored"] z: int = 3 - helptext = get_helptext(Helptext2) + helptext = get_helptext_with_checks(Helptext2) assert "This docstring should be printed as a description" in helptext assert "Attributes" not in helptext assert "x INT" in helptext @@ -114,7 +114,7 @@ class Helptext3: y: Annotated[int, "ignored"] z: int = 3 - helptext = get_helptext(Helptext3) + helptext = get_helptext_with_checks(Helptext3) assert "This docstring should be printed as a description" in helptext assert "Args" not in helptext assert "x INT" in helptext @@ -149,7 +149,7 @@ def some_method(self) -> None: # noqa class ChildClass(UnrelatedParentClass, ActualParentClass): pass - helptext = get_helptext(ChildClass) + helptext = get_helptext_with_checks(ChildClass) assert "Documentation 1" in helptext assert "Documentation 2" in helptext @@ -179,7 +179,7 @@ def main(x: ParentClass = ChildClass(x=5, y=5)) -> Any: """Main function.""" return x - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "Main function." in helptext assert "Documentation 1" in helptext assert "Documentation 2" in helptext @@ -209,14 +209,14 @@ def main_with_docstring(a: Inner) -> None: def main_no_docstring(a: Inner) -> None: """main_no_docstring.""" - helptext = get_helptext(main_with_docstring) + helptext = get_helptext_with_checks(main_with_docstring) assert "Documented in function" in helptext and str(Inner.__doc__) not in helptext assert "main_with_docstring." in helptext assert "Args:" not in helptext assert "Args:" not in helptext assert "Hello world!" in helptext - helptext = get_helptext(main_no_docstring) + helptext = get_helptext_with_checks(main_no_docstring) assert "Something" in helptext assert "main_no_docstring." in helptext assert "Args:" not in helptext @@ -235,7 +235,7 @@ class HelptextWithVariousDefaults: y: Color = Color.RED z: str = "%" - helptext = get_helptext(HelptextWithVariousDefaults) + helptext = get_helptext_with_checks(HelptextWithVariousDefaults) assert "show this help message and exit" in helptext assert "--x PATH" in helptext assert "(default: /some/path/to/a/file)" in helptext @@ -260,7 +260,7 @@ class HelptextMultiline: """Documentation 3 Next line of documentation 3""" - helptext = get_helptext(HelptextMultiline) + helptext = get_helptext_with_checks(HelptextMultiline) assert "Documentation 1 (required)" in helptext assert "Documentation 2" in helptext assert "documentation 2" in helptext @@ -276,7 +276,7 @@ class HelptextGrouped: y: int z: int = 3 - helptext = get_helptext(HelptextGrouped) + helptext = get_helptext_with_checks(HelptextGrouped) assert "Documentation 1 (required)" in helptext assert "Description of both y and z. (required)" in helptext assert "Description of both y and z. (default: 3)" in helptext @@ -288,7 +288,7 @@ class Config: x: Optional[int] = None """An optional variable.""" - helptext = get_helptext(Config) + helptext = get_helptext_with_checks(Config) assert "--x {None}|INT" in helptext assert "An optional variable. (default: None)" in helptext @@ -306,7 +306,7 @@ class HelptextHardString: # Note that the percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string - helptext = get_helptext(HelptextHardString) + helptext = get_helptext_with_checks(HelptextHardString) assert "--x" in helptext assert "2% milk." in helptext @@ -325,7 +325,7 @@ class Parent: class Child(Parent): pass - helptext = get_helptext(Child) + helptext = get_helptext_with_checks(Child) assert "--x STR" in helptext assert "Helptext." in helptext assert "(default: 'This docstring" in helptext @@ -350,7 +350,7 @@ class Child2(Parent2): """Helptext!""" # fmt: on - helptext = get_helptext(Child2) + helptext = get_helptext_with_checks(Child2) assert "--x STR" in helptext assert "Helptext! (default: 'This" in helptext @@ -360,7 +360,7 @@ def test_tuple_helptext() -> None: class TupleHelptext: x: Tuple[int, str, float] - helptext = get_helptext(TupleHelptext) + helptext = get_helptext_with_checks(TupleHelptext) assert "--x INT STR FLOAT" in helptext @@ -369,7 +369,7 @@ def test_tuple_helptext_defaults() -> None: class TupleHelptextDefaults: x: Tuple[int, str, str] = (5, "hello world", "hello") - helptext = get_helptext(TupleHelptextDefaults) + helptext = get_helptext_with_checks(TupleHelptextDefaults) assert "--x INT STR STR" in helptext assert "(default: 5 'hello world' hello)" in helptext @@ -381,7 +381,7 @@ def test_generic_helptext() -> None: class GenericTupleHelptext(Generic[T]): x: T - helptext = get_helptext(GenericTupleHelptext[int]) + helptext = get_helptext_with_checks(GenericTupleHelptext[int]) assert "--x INT" in helptext @@ -392,7 +392,7 @@ def test_generic_tuple_helptext() -> None: class GenericTupleHelptext(Generic[T]): x: Tuple[T, T, T] - helptext = get_helptext(GenericTupleHelptext[int]) + helptext = get_helptext_with_checks(GenericTupleHelptext[int]) assert "--x INT INT INT" in helptext @@ -403,7 +403,7 @@ def test_generic_list_helptext() -> None: class GenericTupleHelptext(Generic[T]): x: List[T] - helptext = get_helptext(GenericTupleHelptext[int]) + helptext = get_helptext_with_checks(GenericTupleHelptext[int]) assert "--x [INT [INT ...]]" in helptext @@ -413,7 +413,7 @@ class LiteralHelptext: x: Literal[1, 2, 3] """A number.""" - helptext = get_helptext(LiteralHelptext) + helptext = get_helptext_with_checks(LiteralHelptext) assert "--x {1,2,3}" in helptext assert "A number. (required)" in helptext @@ -424,7 +424,7 @@ class OptionalLiteralHelptext: x: Optional[Literal[1, 2, 3]] = None """A number.""" - helptext = get_helptext(OptionalLiteralHelptext) + helptext = get_helptext_with_checks(OptionalLiteralHelptext) assert "--x {None,1,2,3}" in helptext assert "A number. (default: None)" in helptext @@ -457,7 +457,7 @@ class MultipleSubparsers: d: bool = False - helptext = get_helptext(MultipleSubparsers) + helptext = get_helptext_with_checks(MultipleSubparsers) assert "2% milk." in helptext assert "Field a description." in helptext @@ -468,7 +468,7 @@ class MultipleSubparsers: assert "[OPTIONS]" not in helptext assert "[B:SUBCOMMAND2 OPTIONS]" not in helptext - helptext = get_helptext( + helptext = get_helptext_with_checks( MultipleSubparsers, args=["a:subcommand1", "b:subcommand1", "--help"] ) @@ -528,7 +528,7 @@ class MultipleSubparsers: h: bool = False i: bool = False - helptext = get_helptext(MultipleSubparsers) + helptext = get_helptext_with_checks(MultipleSubparsers) assert "2% milk." in helptext assert "Field a description." in helptext @@ -538,7 +538,7 @@ class MultipleSubparsers: assert "[OPTIONS]" in helptext assert "[B:SUBCOMMAND2 OPTIONS]" not in helptext - helptext = get_helptext( + helptext = get_helptext_with_checks( MultipleSubparsers, args=["a:subcommand1", "b:subcommand1", "--help"] ) @@ -566,7 +566,7 @@ class OptionalHelptext: z: Optional[int] = 3 """Documentation 3""" - helptext = get_helptext(OptionalHelptext) + helptext = get_helptext_with_checks(OptionalHelptext) assert cast(str, cast(str, OptionalHelptext.__doc__)[:20]) in helptext assert "2% milk" in helptext assert "--x {None}|INT" in helptext @@ -578,7 +578,7 @@ def test_metavar_0() -> None: def main(x: Literal[0, 1, 2, 3] | Tuple[int, int]) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x {0,1,2,3}|{INT INT}" in helptext @@ -589,7 +589,7 @@ def main( pass # The comma formatting is unfortunate, but matches argparse's default behavior. - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x {0,1,2,3,hey,there,hello}|{[INT [INT ...]]}" in helptext @@ -602,7 +602,7 @@ def main( ) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x {0,1,2,3} INT|STR" in helptext @@ -612,7 +612,7 @@ def main( ) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x {0,1,2,3}|{INT INT}|STR" in helptext @@ -622,7 +622,7 @@ def main( ) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x {0,1,2,3}|{INT INT}|{STR STR STR}|{True}" in helptext @@ -632,7 +632,7 @@ def main( ) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "[--x [{INT INT}|{STR STR} [{INT INT}|{STR STR} ...]]]" in helptext @@ -640,7 +640,7 @@ def test_metavar_6() -> None: def main(x: Dict[Tuple[int, int] | Tuple[str, str], Tuple[int, int]]) -> dict: return x - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert ( "--x [{INT INT}|{STR STR} INT INT [{INT INT}|{STR STR} INT INT ...]]" in helptext @@ -658,7 +658,7 @@ class Something( # But this text should! b: int - helptext = get_helptext(Something) + helptext = get_helptext_with_checks(Something) assert "This text should not" not in helptext assert "But this text should!" in helptext @@ -671,13 +671,13 @@ class Struct: def main(x: Any = Struct()): pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x {fixed}" in helptext def main2(x: Callable = nn.ReLU): pass - helptext = get_helptext(main2) + helptext = get_helptext_with_checks(main2) assert "--x {fixed}" in helptext assert "(fixed to:" in helptext assert "torch" in helptext @@ -687,7 +687,7 @@ def test_pathlike() -> None: def main(x: os.PathLike) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--x PATH " in helptext @@ -699,7 +699,7 @@ class Child: def main(child: Child) -> None: pass - helptext = get_helptext(main) + helptext = get_helptext_with_checks(main) assert "--child.x | --child.no-x" in helptext @@ -730,14 +730,14 @@ class MultipleSubparsers: default_factory=SubcommandThree ) - helptext = get_helptext(MultipleSubparsers) + helptext = get_helptext_with_checks(MultipleSubparsers) assert "2% milk." in helptext assert "Field a description." in helptext assert "Field b description." not in helptext assert "Field c description." not in helptext - helptext = get_helptext( + helptext = get_helptext_with_checks( MultipleSubparsers, args=["a:subcommand-one", "b:subcommand-one", "--help"] ) @@ -778,14 +778,14 @@ class MultipleSubparsers: default_factory=SubcommandThree ) - helptext = get_helptext(MultipleSubparsers, use_underscores=True) + helptext = get_helptext_with_checks(MultipleSubparsers, use_underscores=True) assert "2% milk." in helptext assert "Field a description." in helptext assert "Field b description." not in helptext assert "Field c description." not in helptext - helptext = get_helptext( + helptext = get_helptext_with_checks( MultipleSubparsers, args=["a:subcommand_one", "b:subcommand_one", "--help"], use_underscores=True, @@ -814,7 +814,7 @@ class CheckoutCompletion: y: int - help = get_helptext(A | CheckoutCompletion) # type: ignore + help = get_helptext_with_checks(A | CheckoutCompletion) # type: ignore assert help.count("checkout-completion") == 3 @@ -831,7 +831,7 @@ class CheckoutCompletio: y: int - help = get_helptext(A | CheckoutCompletio) # type: ignore + help = get_helptext_with_checks(A | CheckoutCompletio) # type: ignore assert help.count("checkout-completio") == 3 @@ -848,7 +848,7 @@ class CheckoutCompletionn: y: int - help = get_helptext(A | CheckoutCompletionn) # type: ignore + help = get_helptext_with_checks(A | CheckoutCompletionn) # type: ignore assert help.count("checkout-completionn") == 3 @@ -865,7 +865,7 @@ class CmdCheckout012: y: int - help = get_helptext(A | CmdCheckout012) # type: ignore + help = get_helptext_with_checks(A | CmdCheckout012) # type: ignore assert help.count("cmd-checkout012") == 3 @@ -876,7 +876,7 @@ class A: x: Tuple[str, str] = ("hello", "world") - help = get_helptext(A) + help = get_helptext_with_checks(A) assert "STR STR" in help assert "hello world" in help assert "('hello', 'world')" not in help @@ -891,7 +891,7 @@ class A: Tuple[str, str], tyro.conf.arg(constructor=lambda x: ("a", "b")) ] = ("hello", "world") - help = get_helptext(A) + help = get_helptext_with_checks(A) assert "STR STR" not in help # Unlike case above, should not be converted to 'hello world'. assert "('hello', 'world')" in help # JSON special case. @@ -909,7 +909,7 @@ class A: Tuple[int, int], tyro.conf.arg(constructor=json.loads, metavar="JSON") ] = (3, 5) - help = get_helptext(A) + help = get_helptext_with_checks(A) assert "STR STR" not in help assert "JSON" in help assert '["hello", "world"]' in help # JSON special case. @@ -926,7 +926,7 @@ def f( del x, y return 5 - help = get_helptext(f, default=3) + help = get_helptext_with_checks(f, default=3) assert 'default if used: ["hello", "world"]' in help assert "default if used: 3" in help @@ -939,7 +939,7 @@ def f( del x, y return 5 - help = get_helptext(f) + help = get_helptext_with_checks(f) assert "repeatable" not in help, help @@ -951,7 +951,7 @@ def f( del x, y return 5 - help = get_helptext(f) + help = get_helptext_with_checks(f) assert "repeatable" in help, help @@ -963,7 +963,7 @@ def f( del x, y return 5 - help = get_helptext(f) + help = get_helptext_with_checks(f) assert "repeatable, appends to: 'hello world' hello" in help, help @@ -971,5 +971,5 @@ def test_typeddict_exclude() -> None: class Special(TypedDict): x: NotRequired[int] - help = get_helptext(Special) + help = get_helptext_with_checks(Special) assert "unset by default" in help, help diff --git a/tests/test_py311_generated/test_missing_generated.py b/tests/test_py311_generated/test_missing_generated.py index 223eb2560..3f7f9d428 100644 --- a/tests/test_py311_generated/test_missing_generated.py +++ b/tests/test_py311_generated/test_missing_generated.py @@ -15,7 +15,7 @@ def main(a: int = 5, b: int = tyro.MISSING, c: int = 3) -> Tuple[int, int, int]: return a, b, c target = io.StringIO() - with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(main, args=[]) message = target.getvalue() assert "Required options" in message diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index 389e35b2b..5381c1f79 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -13,7 +13,7 @@ import pytest from frozendict import frozendict # type: ignore -from helptext_utils import get_helptext +from helptext_utils import get_helptext_with_checks import tyro @@ -1170,4 +1170,4 @@ class Args: ) assert tyro.cli(Args, args=[]) == Args(A("hello")) - assert "default: inner:alt" in get_helptext(Args) + assert "default: inner:alt" in get_helptext_with_checks(Args) diff --git a/tests/test_py311_generated/test_new_style_annotations_generics_generated.py b/tests/test_py311_generated/test_new_style_annotations_generics_generated.py new file mode 100644 index 000000000..a1d5203c5 --- /dev/null +++ b/tests/test_py311_generated/test_new_style_annotations_generics_generated.py @@ -0,0 +1,54 @@ +from __future__ import annotations # Should enable support for all versions of Python. + +from typing import Any, Literal, Optional + +import pytest + +import tyro + + +def test_list() -> None: + def main(x: list[bool]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] + + +def test_tuple() -> None: + def main(x: tuple[bool, str]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "True", "False"]) == (True, "False") + + +def test_tuple_variable() -> None: + def main(x: tuple[bool | str, ...]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "True", "Wrong"]) == (True, "Wrong") + + +def test_super_nested() -> None: + def main( + x: Optional[ + list[ + tuple[ + Optional[int], + Literal[3, 4], + tuple[int, int] | tuple[str, str], + ] + ] + ] = None, + ) -> Any: + return x + + assert tyro.cli(main, args=[]) is None + assert tyro.cli(main, args="--x None".split(" ")) is None + assert tyro.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] + assert tyro.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] + assert tyro.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ + (2, 3, ("x", "2")), + (2, 3, (1, 2)), + ] + with pytest.raises(SystemExit): + tyro.cli(main, args=["--help"]) diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py new file mode 100644 index 000000000..d04c73f3b --- /dev/null +++ b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py @@ -0,0 +1,54 @@ +# mypy: ignore-errors +# +# PEP 695 isn't yet supported in mypy. (April 4, 2024) +from dataclasses import dataclass + +import tyro + + +def test_simple_generic(): + @dataclass(frozen=True) + class Container[T]: + a: T + b: T + + assert tyro.cli(Container[int], args="--a 1 --b 2".split(" ")) == Container(1, 2) + + +type X = int +type Y = list[int] +type Z = Inner[int] + + +@dataclass(frozen=True) +class Inner[T]: + a: T + b: T + + +def test_generic_with_type_statement_0(): + @dataclass(frozen=True) + class Container[T]: + a: T + b: T + + assert tyro.cli(Container[X], args="--a 1 --b 2".split(" ")) == Container(1, 2) + + +def test_generic_with_type_statement_1(): + @dataclass(frozen=True) + class Container[T]: + a: tuple[X, ...] + b: T + + assert tyro.cli(Container[Y], args="--a 1 --b 2".split(" ")) == Container((1,), [2]) + + +def test_generic_with_type_statement_2(): + @dataclass(frozen=True) + class Container[T]: + a: Z + + assert tyro.cli(Container[Y], args="--a.a 1 --a.b 2".split(" ")) == Container( + Inner(1, 2) + ) diff --git a/tests/test_py311_generated/test_new_style_annotations_unions_generated.py b/tests/test_py311_generated/test_new_style_annotations_unions_generated.py new file mode 100644 index 000000000..ccb1920ea --- /dev/null +++ b/tests/test_py311_generated/test_new_style_annotations_unions_generated.py @@ -0,0 +1,61 @@ +from __future__ import annotations # Should enable support for all versions of Python. + +from typing import Any, Literal + +import pytest + +import tyro + + +def test_union_basic(): + def main(x: int | str) -> int | str: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == 6 + assert tyro.cli(main, args=["--x", "five"]) == "five" + + +def test_union_with_list(): + def main(x: int | str | list[bool]) -> Any: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == 6 + assert tyro.cli(main, args=["--x", "five"]) == "five" + assert tyro.cli(main, args=["--x", "True"]) == "True" + assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] + + +def test_union_literal(): + def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: + return x + + assert tyro.cli(main, args=["--x", "5"]) == 5 + assert tyro.cli(main, args=["--x", "6"]) == "6" + assert tyro.cli(main, args=["--x", "five"]) == "five" + + +def test_super_nested(): + def main( + x: None + | list[ + tuple[ + None | int, + Literal[3, 4], + tuple[int, int] | tuple[str, str], + ] + ] = None, + ) -> Any: + return x + + assert tyro.cli(main, args=[]) is None + assert tyro.cli(main, args="--x None".split(" ")) is None + assert tyro.cli(main, args="--x None 3 2 2".split(" ")) == [(None, 3, (2, 2))] + assert tyro.cli(main, args="--x 2 3 x 2".split(" ")) == [(2, 3, ("x", "2"))] + assert tyro.cli(main, args="--x 2 3 x 2 2 3 1 2".split(" ")) == [ + (2, 3, ("x", "2")), + (2, 3, (1, 2)), + ] + with pytest.raises(SystemExit): + tyro.cli(main, args=["--help"]) diff --git a/tests/test_py311_generated/test_partial_generated.py b/tests/test_py311_generated/test_partial_generated.py index 86a017451..89fe78552 100644 --- a/tests/test_py311_generated/test_partial_generated.py +++ b/tests/test_py311_generated/test_partial_generated.py @@ -3,7 +3,7 @@ import dataclasses import functools -from helptext_utils import get_helptext +from helptext_utils import get_helptext_with_checks import tyro @@ -28,7 +28,7 @@ def main(a: int, b: str) -> str: """Hello!""" return b * a - helptext = get_helptext(functools.partial(main, b="3")) + helptext = get_helptext_with_checks(functools.partial(main, b="3")) assert "partial" not in helptext assert "Hello!" in helptext @@ -40,7 +40,7 @@ class Main: def __init__(self, a: int, b: str) -> None: self.inner = b * a - helptext = get_helptext(functools.partial(Main, b="3")) + helptext = get_helptext_with_checks(functools.partial(Main, b="3")) assert "partial" not in helptext assert "Hello!" in helptext @@ -56,6 +56,6 @@ class Config: ConfigWithDefaults = functools.partial(functools.partial(Config, a=3), c=5) assert tyro.cli(ConfigWithDefaults, args=["--b", "4"]) == Config(3, 4, 5) - helptext = get_helptext(ConfigWithDefaults) + helptext = get_helptext_with_checks(ConfigWithDefaults) assert "partial" not in helptext assert "Hello!" in helptext From 622906799d6f87536f414383aec13c09eb0e2cd2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 10 May 2024 12:53:54 +0100 Subject: [PATCH 398/491] Bump version --- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aa55c51a1..231bbd7e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.3" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.4" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index 65be2014c..5f94556b7 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.3" +__version__ = "0.8.4" From 868c980e0e85054ee14fb9a177a51a44a011ce96 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 12 May 2024 03:10:44 +0100 Subject: [PATCH 399/491] Support `typing.ReadOnly`, fix `typing.Final` edge case --- src/tyro/_instantiators.py | 25 ++++++++++--------- src/tyro/_resolver.py | 14 +++++++++++ tests/test_nested.py | 17 ++++++++++++- tests/test_py311_generated/_generate.py | 21 ++++++++++++++-- .../test_nested_generated.py | 16 ++++++++++++ ...t_typeddict_readonly_min_py38_generated.py | 23 +++++++++++++++++ tests/test_typeddict_readonly_min_py38.py | 21 ++++++++++++++++ 7 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 tests/test_py311_generated/test_typeddict_readonly_min_py38_generated.py create mode 100644 tests/test_typeddict_readonly_min_py38.py diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index 67bf44f6a..0cf9fe5a5 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -56,7 +56,7 @@ overload, ) -from typing_extensions import Annotated, Final, Literal, get_args, get_origin +from typing_extensions import Annotated, Literal, get_args, get_origin from . import _resolver @@ -196,6 +196,8 @@ def instantiator(strings: List[str]) -> None: if maybe_newtype_name is not None: metavar = maybe_newtype_name.upper() + typ, _ = _resolver.unwrap_annotated(typ) + # Address container types. If a matching container is found, this will recursively # call instantiator_from_type(). container_out = _instantiator_from_container_type( @@ -204,6 +206,13 @@ def instantiator(strings: List[str]) -> None: if container_out is not None: return container_out + # At this point: Annotated[] and containers like list[T] are all handled, + # any types with a non-None origin aren't supported. + if get_origin(typ) is not None: + raise UnsupportedTypeAnnotationError( # pragma: no cover + f"Unsupported type {typ} with origin {get_origin(typ)}" + ) + # Validate that typ is a `(arg: str) -> T` type converter, as expected by argparse. if typ in _builtin_set: pass @@ -319,7 +328,7 @@ def _instantiator_from_container_type( type_from_typevar: Dict[TypeVar, TypeForm[Any]], markers: FrozenSet[_markers.Marker], ) -> Optional[Tuple[Instantiator, InstantiatorMetadata]]: - """Attempt to create an instantiator from a container type. Returns `None` is no + """Attempt to create an instantiator from a container type. Returns `None` if no container type is found.""" # Default generic types to strings. @@ -333,14 +342,9 @@ def _instantiator_from_container_type( typ = Set[str] type_origin = get_origin(typ) - if type_origin is None: + if type_origin in (None, Annotated): return None - # Unwrap Annotated and Final types. - if type_origin in (Annotated, Final): - contained_type = get_args(typ)[0] - return instantiator_from_type(contained_type, type_from_typevar, markers) - for make, matched_origins in { _instantiator_from_sequence: ( collections.abc.Sequence, @@ -356,10 +360,7 @@ def _instantiator_from_container_type( }.items(): if type_origin in matched_origins: return make(typ, type_from_typevar, markers) - - raise UnsupportedTypeAnnotationError( # pragma: no cover - f"Unsupported type {typ} with origin {type_origin}" - ) + return None def _instantiator_from_tuple( diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 082d019df..a8d2ed8c6 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -24,6 +24,7 @@ from typing_extensions import ( Annotated, + Final, ForwardRef, Self, TypeAliasType, @@ -266,6 +267,19 @@ def unwrap_annotated( - Annotated[int, 1], int => (int, (1,)) - Annotated[int, "1"], int => (int, ()) """ + + # `Final` and `ReadOnly` types are ignored in tyro. + try: + # Can only import ReadOnly in typing_extensions>=4.9.0, which isn't + # supported by Python 3.7. + from typing_extensions import ReadOnly + + while get_origin(typ) in (Final, ReadOnly): + typ = get_args(typ)[0] + except ImportError: # pragma: no cover + while get_origin(typ) is Final: + typ = get_args(typ)[0] + # Check for __tyro_markers__ from @configure. targets = tuple( x diff --git a/tests/test_nested.py b/tests/test_nested.py index 0d7beb64a..b9ef69327 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -4,7 +4,7 @@ import pytest from frozendict import frozendict # type: ignore from helptext_utils import get_helptext_with_checks -from typing_extensions import Annotated, Literal +from typing_extensions import Annotated, Final, Literal import tyro @@ -39,6 +39,21 @@ class Nested: tyro.cli(Nested, args=["--x", "1"]) +def test_nested_final() -> None: + @dataclasses.dataclass + class B: + y: int + + @dataclasses.dataclass + class Nested: + x: int + b: Final[B] # type: ignore + + assert tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) + with pytest.raises(SystemExit): + tyro.cli(Nested, args=["--x", "1"]) + + def test_nested_accidental_underscores() -> None: @dataclasses.dataclass class B: diff --git a/tests/test_py311_generated/_generate.py b/tests/test_py311_generated/_generate.py index a78b12adc..6fcc279a0 100644 --- a/tests/test_py311_generated/_generate.py +++ b/tests/test_py311_generated/_generate.py @@ -3,9 +3,19 @@ import pathlib import subprocess +from concurrent.futures import ThreadPoolExecutor -for test_path in pathlib.Path(__file__).absolute().parent.parent.glob("test_*.py"): - content = test_path.read_text().replace("typing_extensions", "typing") + +def generate_from_path(test_path: pathlib.Path) -> None: + content = test_path.read_text() + content = content.replace("typing_extensions", "typing") + + if "ReadOnly" in content: + content = content.replace("ReadOnly,", "", 1) + content = content.replace( + "\nfrom typing import ", + "\nfrom typing_extensions import ReadOnly\nfrom typing import ", + ) while "Union[" in content: new_content, _, b = content.partition("Union[") @@ -42,3 +52,10 @@ subprocess.run(["isort", "--profile=black", str(out_path)], check=True) subprocess.run(["ruff", "format", str(out_path)], check=True) subprocess.run(["ruff", "check", "--fix", str(out_path)], check=True) + + +with ThreadPoolExecutor(max_workers=8) as executor: + executor.map( + generate_from_path, + pathlib.Path(__file__).absolute().parent.parent.glob("test_*.py"), + ) diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index 5381c1f79..16ae5bc05 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -2,6 +2,7 @@ from typing import ( Annotated, Any, + Final, Generic, Literal, Mapping, @@ -48,6 +49,21 @@ class Nested: tyro.cli(Nested, args=["--x", "1"]) +def test_nested_final() -> None: + @dataclasses.dataclass + class B: + y: int + + @dataclasses.dataclass + class Nested: + x: int + b: Final[B] + + assert tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) + with pytest.raises(SystemExit): + tyro.cli(Nested, args=["--x", "1"]) + + def test_nested_accidental_underscores() -> None: @dataclasses.dataclass class B: diff --git a/tests/test_py311_generated/test_typeddict_readonly_min_py38_generated.py b/tests/test_py311_generated/test_typeddict_readonly_min_py38_generated.py new file mode 100644 index 000000000..b91cb7cc9 --- /dev/null +++ b/tests/test_py311_generated/test_typeddict_readonly_min_py38_generated.py @@ -0,0 +1,23 @@ +from typing import TypedDict + +import pytest +from typing_extensions import ReadOnly + +import tyro + + +def test_read_only_typeddict() -> None: + class ManyTypesTypedDict(TypedDict): + i: ReadOnly[int] # type: ignore + s: ReadOnly[ReadOnly[str]] # type: ignore + + assert tyro.cli( + ManyTypesTypedDict, + args="--i 5 --s 5".split(" "), + ) == dict(i=5, s="5") + + with pytest.raises(SystemExit): + tyro.cli(ManyTypesTypedDict, args="--i 5".split(" ")) + + with pytest.raises(SystemExit): + tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) diff --git a/tests/test_typeddict_readonly_min_py38.py b/tests/test_typeddict_readonly_min_py38.py new file mode 100644 index 000000000..ae26777bf --- /dev/null +++ b/tests/test_typeddict_readonly_min_py38.py @@ -0,0 +1,21 @@ +import pytest +from typing_extensions import ReadOnly, TypedDict + +import tyro + + +def test_read_only_typeddict() -> None: + class ManyTypesTypedDict(TypedDict): + i: ReadOnly[int] # type: ignore + s: ReadOnly[ReadOnly[str]] # type: ignore + + assert tyro.cli( + ManyTypesTypedDict, + args="--i 5 --s 5".split(" "), + ) == dict(i=5, s="5") + + with pytest.raises(SystemExit): + tyro.cli(ManyTypesTypedDict, args="--i 5".split(" ")) + + with pytest.raises(SystemExit): + tyro.cli(ManyTypesTypedDict, args="--s 5".split(" ")) From f1707f23543db6e9e94a54fb1130c039ef4dc122 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 13 May 2024 15:23:17 +0100 Subject: [PATCH 400/491] Typo fixes + wording in marker docstrings Signed-off-by: Brent Yi --- src/tyro/conf/_markers.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/tyro/conf/_markers.py b/src/tyro/conf/_markers.py index babfc810e..c9a0f122b 100644 --- a/src/tyro/conf/_markers.py +++ b/src/tyro/conf/_markers.py @@ -10,7 +10,8 @@ # Note that all Annotated[T, None] values are just for static checkers. The real marker # singletons are instantiated dynamically below. # -# An alias could ideally be made, but SpecialForm aliases are not well supported by static analysis tools. +# An alias could ideally be made, but SpecialForm aliases aren't well supported by +# type checkers. T = TypeVar("T") @@ -37,8 +38,8 @@ Fixed = Annotated[T, None] """A type `T` can be annotated as `Fixed[T]` to prevent `tyro.cli` from parsing it; a -default value should be set instead. Note that fields with defaults that can't be parsed -will also be marked as fixed automatically.""" +default value should be set instead. Fields that can't be parsed with defaults will also +be marked as fixed automatically.""" Suppress = Annotated[T, None] """A type `T` can be annotated as `Suppress[T]` to prevent `tyro.cli` from parsing it, and @@ -71,8 +72,8 @@ python x.py {--root options} s1 {--s1 options} s2 {--s2 options} ``` -This can be frustrating because the resulting CLI is sensitive the exact positioning and -ordering of options. +This can be frustrating because the resulting CLI is sensitive to the positioning of +options. To consolidate subcommands, we push arguments to the end, after all subcommands: ``` @@ -80,7 +81,7 @@ ``` This is more robust to reordering of options, ensuring that any new options can simply -be placed at the end of the command> +be placed at the end of the command. """ OmitSubcommandPrefixes = Annotated[T, None] @@ -92,10 +93,10 @@ cmd: Union[NestedTypeA, NestedTypeB] By default, `--cmd.arg` may be generated as a flag for each dataclass in the union. -If subcommand prefixes are omitted, we would instead simply have `--arg`. +If subcommand prefixes are omitted, we would instead have `--arg`. By default, `cmd:nested-type-a` and `cmd:nested-type-b` may be generated as subcommand. -If subcommand prefixes are omitted, we would instead simply have `nested-type-a` and +If subcommand prefixes are omitted, we would instead have `nested-type-a` and `nested-type-b`. """ @@ -113,14 +114,14 @@ UseAppendAction = Annotated[T, None] """Use "append" actions for variable-length arguments. -Given an annotation like `x: List[int]`, this means that `x = [0, 1, 2]` can be set via +Given an annotation like `x: list[int]`, this means that `x = [0, 1, 2]` can be set via the CLI syntax `--x 0 --x 1 --x 2` instead of the default of `--x 0 1 2`. The resulting syntax may be more user-friendly; for `tyro`, it also enables support for -otherwise ambiguous annotations like `List[List[int]]`. +otherwise ambiguous annotations like `list[list[int]]`. -Can be applied to all variable-length sequences (`List[T]`, `Sequence[T]`, -`Tuple[T, ...]`, etc), including dictionaries without default values. +Can be applied to all variable-length sequences (`list[T]`, `Sequence[T]`, +`tuple[T, ...]`, etc), including dictionaries without default values. """ UseCounterAction = Annotated[T, None] From ed9e5e4fc8ea8025dfb1f3df82764fefa5c8c8a2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 16 May 2024 13:58:34 +0100 Subject: [PATCH 401/491] Cleaner stack traces for errors in `tyro.cli()` --- src/tyro/_calling.py | 56 ++++++++++++------- src/tyro/_cli.py | 24 +++++--- src/tyro/_fields.py | 2 +- .../test_nested_generated.py | 2 +- 4 files changed, 53 insertions(+), 31 deletions(-) diff --git a/src/tyro/_calling.py b/src/tyro/_calling.py index d4fc46d8e..30f51becf 100644 --- a/src/tyro/_calling.py +++ b/src/tyro/_calling.py @@ -5,6 +5,7 @@ import dataclasses import itertools +from functools import partial from typing import Any, Callable, Dict, List, Sequence, Set, Tuple, TypeVar, Union from typing_extensions import get_args @@ -25,16 +26,22 @@ class InstantiationError(Exception): T = TypeVar("T") -def call_from_args( +def callable_with_args( f: Callable[..., T], parser_definition: _parsers.ParserSpecification, default_instance: Union[T, _fields.NonpropagatingMissingType], value_from_prefixed_field_name: Dict[str, Any], field_name_prefix: str, -) -> Tuple[T, Set[str]]: - """Call `f` with arguments specified by a dictionary of values from argparse. +) -> Tuple[Callable[[], T], Set[str]]: + """Populate `f` with arguments specified by a dictionary of values from argparse. - Returns the output of `f` and a set of used arguments.""" + Returns a partialed version of `f` with arguments populated, and a set of + used arguments. + + We return a `Callable[[], OutT]` instead of `T` directly for aesthetic + reasons; it lets use reduce layers in stack traces for errors from + functions passed to `tyro`. + """ positional_args: List[Any] = [] kwargs: Dict[str, Any] = {} @@ -119,13 +126,15 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: # Nested callable. if _resolver.unwrap_origin_strip_extras(field_type) is Union: field_type = type(field.default) - value, consumed_keywords_child = call_from_args( + get_value, consumed_keywords_child = callable_with_args( field_type, parser_definition.child_from_prefix[prefixed_field_name], field.default, value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, ) + value = get_value() + del get_value consumed_keywords |= consumed_keywords_child else: # Unions over dataclasses (subparsers). This is the only other option. @@ -152,7 +161,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: chosen_f = subparser_def.options[ list(subparser_def.parser_from_name.keys()).index(subparser_name) ] - value, consumed_keywords_child = call_from_args( + get_value, consumed_keywords_child = callable_with_args( chosen_f, subparser_def.parser_from_name[subparser_name], ( @@ -163,6 +172,8 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, ) + value = get_value() + del get_value consumed_keywords |= consumed_keywords_child if value is _fields.EXCLUDE_FROM_CALL: @@ -191,7 +202,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: if any(is_missing_list): if not any_arguments_provided: # No arguments were provided in this group. - return default_instance, consumed_keywords # type: ignore + return lambda: default_instance, consumed_keywords # type: ignore message = "either all arguments must be provided or none of them." if len(kwargs) > 0: @@ -221,32 +232,35 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: # Triggered when support_single_arg_types=True is used. assert len(kwargs) == 0 assert len(positional_args) == 1 - return positional_args[0], consumed_keywords # type: ignore + return lambda: positional_args[0], consumed_keywords # type: ignore else: assert len(positional_args) == 0 - return unwrapped_f(kwargs.values()), consumed_keywords # type: ignore + return partial(unwrapped_f, kwargs.values()), consumed_keywords # type: ignore elif unwrapped_f is dict: if len(positional_args) > 0: # Triggered when support_single_arg_types=True is used. assert len(kwargs) == 0 assert len(positional_args) == 1 - return unwrapped_f(*positional_args), consumed_keywords # type: ignore + return partial(unwrapped_f, *positional_args), consumed_keywords # type: ignore else: assert len(positional_args) == 0 - return kwargs, consumed_keywords # type: ignore + return lambda: kwargs, consumed_keywords # type: ignore else: if field_name_prefix == "": # Don't catch any errors for the "root" field. If main() in tyro.cli(main) # raises a ValueError, this shouldn't be caught. - return unwrapped_f(*positional_args, **kwargs), consumed_keywords # type: ignore + return partial(unwrapped_f, *positional_args, **kwargs), consumed_keywords # type: ignore else: # Try to catch ValueErrors raised by field constructors. - try: - return unwrapped_f(*positional_args, **kwargs), consumed_keywords # type: ignore - # If unwrapped_f raises a ValueError, wrap the message with a more informative - # InstantiationError if possible. - except ValueError as e: - raise InstantiationError( - e.args[0], - field_name_prefix, - ) + def with_instantiation_error(): + try: + return unwrapped_f(*positional_args, **kwargs) + # If unwrapped_f raises a ValueError, wrap the message with a more informative + # InstantiationError if possible. + except ValueError as e: + raise InstantiationError( + e.args[0], + field_name_prefix, + ) + + return with_instantiation_error, consumed_keywords # type: ignore diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index 2190c1755..2982033bd 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -209,9 +209,12 @@ def cli( _unsafe_cache.clear_cache() if return_unknown_args: - return cast(Tuple[OutT, List[str]], output) + assert isinstance(output, tuple) + run_with_args_from_cli = output[0] + return run_with_args_from_cli(), output[1] else: - return cast(OutT, output) + run_with_args_from_cli = cast(Callable[[], OutT], output) + return run_with_args_from_cli() @overload @@ -285,7 +288,7 @@ def _cli_impl( ) -> Union[ OutT, argparse.ArgumentParser, - Tuple[OutT, List[str]], + Tuple[Callable[[], OutT], List[str]], ]: """Helper for stitching the `tyro` pipeline together.""" if "default_instance" in deprecated_kwargs: @@ -461,7 +464,7 @@ def _cli_impl( try: # Attempt to call `f` using whatever was passed in. - out, consumed_keywords = _calling.call_from_args( + get_out, consumed_keywords = _calling.callable_with_args( f, parser_spec, default_instance_internal, @@ -469,7 +472,11 @@ def _cli_impl( field_name_prefix="", ) except _calling.InstantiationError as e: - assert isinstance(e, _calling.InstantiationError) + # Print prettier errors. + # Note that this doesn't catch errors raised directly by get_out(), + # since that's called later! This is intentional, because we do less + # error handling for the root callable. Relevant: the + # `field_name_prefix == ""` condition in `callable_with_args()`! # Emulate argparse's error behavior when invalid arguments are passed in. from rich.console import Console, Group, RenderableType @@ -522,14 +529,15 @@ def _cli_impl( ) if dummy_wrapped: - out = getattr(out, _strings.dummy_field_name) + get_wrapped_out = get_out + get_out = lambda: getattr(get_wrapped_out(), _strings.dummy_field_name) # noqa if return_unknown_args: assert unknown_args is not None, "Should have parsed with `parse_known_args()`" # If we're parsed unknown args, we should return the original args, not # the fixed ones. unknown_args = [modified_args.get(arg, arg) for arg in unknown_args] - return out, unknown_args + return get_out, unknown_args # type: ignore else: assert unknown_args is None, "Should have parsed with `parse_args()`" - return out + return get_out # type: ignore diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index beddffa08..0b269c10d 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -932,7 +932,7 @@ def _try_field_list_from_general_callable( # Return error message. return out - # If a default is provided: . + # If a default is provided: either all or none of the arguments must be passed in. if default_instance not in MISSING_SINGLETONS: for i, field in enumerate(out): out[i] = field.add_markers((_markers._OPTIONAL_GROUP,)) diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index 16ae5bc05..dca4d0841 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -57,7 +57,7 @@ class B: @dataclasses.dataclass class Nested: x: int - b: Final[B] + b: Final[B] # type: ignore assert tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) with pytest.raises(SystemExit): From 6527d1d09834055c5dd55a07af40fd5769e742d7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 22 May 2024 14:37:47 +0100 Subject: [PATCH 402/491] Docs tweaks, run `prettier` --- .github/workflows/coverage.yml | 1 - .github/workflows/docs.yml | 1 - .github/workflows/publish.yml | 41 +++++++++---------- README.md | 26 ++++++++---- docs/source/building_configuration_systems.md | 18 -------- docs/source/index.md | 5 +-- docs/source/installation.md | 6 +-- docs/source/your_first_cli.md | 20 ++++----- 8 files changed, 51 insertions(+), 67 deletions(-) delete mode 100644 docs/source/building_configuration_systems.md diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f3632e003..b00e9eef8 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -31,4 +31,3 @@ jobs: name: codecov-umbrella fail_ci_if_error: true verbose: true - diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 09d4e5975..4d339befb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,7 +8,6 @@ jobs: docs: runs-on: ubuntu-latest steps: - # Check out source - uses: actions/checkout@v2 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f694231ff..deca26a88 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,27 +9,26 @@ on: jobs: deploy: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - name: Install dependencies - run: | - pip install uv - uv pip install --system -e ".[dev]" - uv pip install --system build twine - - name: Strip unsupported tags in README - run: | - sed -i '//,//d' README.md - - name: Build and publish - env: - PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} - PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python -m build - twine upload --username $PYPI_USERNAME --password $PYPI_PASSWORD dist/* + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install dependencies + run: | + pip install uv + uv pip install --system -e ".[dev]" + uv pip install --system build twine + - name: Strip unsupported tags in README + run: | + sed -i '//,//d' README.md + - name: Build and publish + env: + PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} + PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python -m build + twine upload --username $PYPI_USERNAME --password $PYPI_PASSWORD dist/* diff --git a/README.md b/README.md index 3bc8a2482..5e1d8ec55 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,9 @@ For advanced users, it also supports: - **Subcommands**, as well as choosing between and overriding values in configuration objects. -- **Completion script generation** for `bash`, `zsh`, and `tcsh`. -- **Fine-grained configuration** via PEP 529 runtime annotations - (`tyro.conf.*`). +- **Shell completion** for `bash`, `zsh`, and `tcsh`. +- **Fine-grained configuration** via [PEP + 593](https://peps.python.org/pep-0593/) annotations (`tyro.conf.*`). For examples and the API reference, see our [documentation](https://brentyi.github.io/tyro). @@ -94,7 +94,7 @@ facilitating type safety and modularity for larger projects. Examples: />

Library for distributed training of large language models in JAX.Train very large language models in Jax.
@@ -106,7 +106,7 @@ facilitating type safety and modularity for larger projects. Examples: /> Interface for processing composite Wavefront OBJ files for Mujoco.Interface for processing OBJ files for Mujoco.
@@ -159,12 +159,24 @@ facilitating type safety and modularity for larger projects. Examples: Collection of tracked experiments for reinforcement learning.
+ + vwxyzjn/cleanrl +
GitHub star count + + 		Single-file implementation of deep RL algorithms.
### Alternatives -`tyro` bakes many opinions into its design decisions. If any of them don't make -sense, feel free to file an issue! +`tyro` is an opinionated library. If any design decisions don't make sense, +feel free to file an issue! You might also consider one of many alternative libraries. Some that we particularly like: diff --git a/docs/source/building_configuration_systems.md b/docs/source/building_configuration_systems.md deleted file mode 100644 index 6c0cfc441..000000000 --- a/docs/source/building_configuration_systems.md +++ /dev/null @@ -1,18 +0,0 @@ -# Building configuration systems - -Beyond building simple command-line interfaces, :func:`tyro.cli()` is designed -to scale to larger configuration systems such as those typically built with -libraries like [hydra](https://github.com/facebookresearch/hydra). - -For a live example, see -[nerfstudio](https://github.com/nerfstudio-project/nerfstudio/). Notably, -`nerfstudio`'s configuration system is implemented entirely in Python, without -YAML, and has full tab completion support in your terminal. - -For overriding a dynamically loaded configuration object (typically a -dataclass), the `default=` parameter of :func:`tyro.cli()` can be used. If you -have multiple default configuration objects and need to select one to pass in as -a default -- this might be a YAML file or config instance -- an environment -variable or manually parsed (via `sys.argv`) positional argument can be used. - -For concrete examples, see our "Config Management" documentation. diff --git a/docs/source/index.md b/docs/source/index.md index 5c9d19457..8610b441b 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -23,8 +23,8 @@ For advanced users, it also supports: - **Subcommands**, as well as choosing between and overriding values in configuration objects. - **Completion script generation** for `bash`, `zsh`, and `tcsh`. -- **Fine-grained configuration** via PEP 529 runtime annotations - (`tyro.conf.*`). +- **Fine-grained configuration** via [PEP + 593](https://peps.python.org/pep-0593/) annotations (`tyro.conf.*`). To get started, we recommend browsing the examples to the left. @@ -120,7 +120,6 @@ To get started, we recommend browsing the examples to the left. goals_and_alternatives helptext_generation tab_completion - building_configuration_systems .. toctree:: diff --git a/docs/source/installation.md b/docs/source/installation.md index 180bcb113..3ffe61bfd 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -2,8 +2,7 @@ ## Standard -Installation is supported on Python >=3.7 via pip. This is typically all that's -required. +Installation is supported on Python >=3.7 via pip. ```bash pip install tyro @@ -22,7 +21,4 @@ python -m pip install -e ".[dev]" # Run tests. pytest - -# Check types. -mypy --install-types . ``` diff --git a/docs/source/your_first_cli.md b/docs/source/your_first_cli.md index 241f79810..f73dbfba7 100644 --- a/docs/source/your_first_cli.md +++ b/docs/source/your_first_cli.md @@ -1,7 +1,7 @@ # Your first CLI -For getting started with `tyro`, consider the simple `argparse`-based -command-line interface: +To get started with `tyro`, consider the simple `argparse`-based command-line +interface: ```python """Sum two numbers from argparse.""" @@ -17,14 +17,13 @@ total = args.a + args.b print(total) ``` -This pattern is dramatically cleaner than manually parsing `sys.argv`, but has -several issues: it lacks type checking and IDE support (consider: jumping to -definitions, finding references, docstrings, refactoring and renaming tools), -requires a significant amount of parsing-specific boilerplate, and becomes -difficult to manage for larger projects. +This is dramatically cleaner than manually parsing `sys.argv`, but has several +issues: it requires a significant amount of parsing-specific boilerplate, lacks +type checking and IDE support (consider: jumping to definitions, finding +references, docstrings, refactoring and renaming tools), and becomes difficult +to manage for larger projects. -The basic goal of :func:`tyro.cli()` is to provide a wrapper for `argparse` that -solves these issues. +:func:`tyro.cli()` aims to solve these issues. **(1) Command-line interfaces from functions.** @@ -57,8 +56,7 @@ tyro.cli(add) # Returns `None`. **(2) Command-line interfaces from config objects.** -A class in Python can be treated as a function that returns an instance. This -makes it easy to populate explicit configuration structures: +A class in Python can be treated as a function that returns an instance: ```python """Sum two numbers by instantiating a dataclass with tyro.""" From de5be4abdf4e680b06ca92e717b12365cdb17989 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 8 Jun 2024 19:53:54 -0700 Subject: [PATCH 403/491] Profiling + performance improvements (~15%) --- src/tyro/_argparse_formatter.py | 13 ++ src/tyro/_arguments.py | 205 ++++++++++++------------------ src/tyro/_docstrings.py | 49 +++---- src/tyro/_instantiators.py | 12 +- src/tyro/_parsers.py | 4 +- src/tyro/_resolver.py | 42 ++++-- src/tyro/_strings.py | 25 ++-- src/tyro/_subcommand_matching.py | 4 +- src/tyro/extras/_serialization.py | 2 +- 9 files changed, 166 insertions(+), 190 deletions(-) diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index 10c6eabaf..90c724a1e 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -278,6 +278,19 @@ class TyroArgumentParser(argparse.ArgumentParser, argparse_sys.ArgumentParser): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) + @override + def _check_value(self, action, value): + """We override _check_value to ignore sentinel values defined by tyro. + + This solves a choices error raised by argparse in a very specific edge case: + literals in containers as positional arguments. + """ + from ._fields import MISSING_SINGLETONS + + if value in MISSING_SINGLETONS: + return + return super()._check_value(action, value) + @override def _print_message(self, message, file=None): if message and self._console_outputs: diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index e92f97cf0..9a1ff1ea5 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -5,7 +5,6 @@ import dataclasses import enum -import functools import itertools import json import shlex @@ -15,11 +14,9 @@ Callable, Dict, Iterable, - List, Mapping, Optional, Sequence, - Set, Tuple, TypeVar, Union, @@ -187,25 +184,21 @@ def add_argument( @cached_property def lowered(self) -> LoweredArgumentDefinition: """Lowered argument definition, generated by applying a sequence of rules.""" - rules = ( - _rule_handle_defaults, - _rule_handle_boolean_flags, - _rule_recursive_instantiator_from_type, - _rule_convert_defaults_to_strings, - _rule_counters, - _rule_generate_helptext, - _rule_set_name_or_flag_and_dest, - _rule_positional_special_handling, - _rule_static_cast_choices_to_patched_list, - ) - return functools.reduce( - lambda lowered, rule: rule(self, lowered), - rules, - LoweredArgumentDefinition(), - ) + # Each rule will mutate the lowered object. This is (unfortunately) + # much faster than a functional approach. + lowered = LoweredArgumentDefinition() + _rule_handle_defaults(self, lowered) + _rule_handle_boolean_flags(self, lowered) + _rule_recursive_instantiator_from_type(self, lowered) + _rule_convert_defaults_to_strings(self, lowered) + _rule_counters(self, lowered) + _rule_generate_helptext(self, lowered) + _rule_set_name_or_flag_and_dest(self, lowered) + _rule_positional_special_handling(self, lowered) + return lowered -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass class LoweredArgumentDefinition: """Contains fields meant to be passed directly into argparse.""" @@ -231,7 +224,7 @@ def is_fixed(self) -> bool: required: Optional[bool] = None action: Optional[Any] = None nargs: Optional[Union[int, str]] = None - choices: Optional[Union[Set[str], List[str]]] = None + choices: Optional[Tuple[str, ...]] = None # Note: unlike in vanilla argparse, our metavar is always a string. We handle # sequences, multiple arguments, etc, manually. metavar: Optional[str] = None @@ -241,7 +234,7 @@ def is_fixed(self) -> bool: def _rule_handle_defaults( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, -) -> LoweredArgumentDefinition: +) -> None: """Set `required=False` if a default value is set.""" # Mark lowered as required if a default is set. @@ -249,22 +242,25 @@ def _rule_handle_defaults( arg.field.default in _fields.MISSING_SINGLETONS and _markers._OPTIONAL_GROUP not in arg.field.markers ): - return dataclasses.replace(lowered, default=None, required=True) + lowered.default = None + lowered.required = True + return - return dataclasses.replace(lowered, default=arg.field.default) + lowered.default = arg.field.default + return def _rule_handle_boolean_flags( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, -) -> LoweredArgumentDefinition: +) -> None: if ( _resolver.apply_type_from_typevar( arg.field.type_or_callable, arg.type_from_typevar ) is not bool - ): # type: ignore - return lowered + ): + return if ( arg.field.default in _fields.MISSING_SINGLETONS @@ -273,14 +269,12 @@ def _rule_handle_boolean_flags( or _markers.Fixed in arg.field.markers ): # Treat bools as a normal parameter. - return lowered + return elif arg.field.default in (True, False): # Default `False` => --flag passed in flips to `True`. - return dataclasses.replace( - lowered, - action=BooleanOptionalAction, - instantiator=lambda x: x, # argparse will directly give us a bool! - ) + lowered.action = BooleanOptionalAction + lowered.instantiator = lambda x: x # argparse will directly give us a bool! + return assert False, ( f"Expected a boolean as a default for {arg.field.intern_name}, but got" @@ -291,7 +285,7 @@ def _rule_handle_boolean_flags( def _rule_recursive_instantiator_from_type( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, -) -> LoweredArgumentDefinition: +) -> None: """The bulkiest bit: recursively analyze the type annotation and use it to determine how to instantiate it given some string from the commandline. @@ -301,15 +295,13 @@ def _rule_recursive_instantiator_from_type( bit more flexible, and lets us handle more complex types like enums and multi-type tuples.""" if _markers.Fixed in arg.field.markers: - return dataclasses.replace( - lowered, - instantiator=None, - metavar="{fixed}", - required=False, - default=_fields.MISSING_PROP, - ) + lowered.instantiator = None + lowered.metavar = "{fixed}" + lowered.required = False + lowered.default = _fields.MISSING_PROP + return if lowered.instantiator is not None: - return lowered + return try: instantiator, metadata = _instantiators.instantiator_from_type( arg.field.type_or_callable, @@ -326,12 +318,10 @@ def _rule_recursive_instantiator_from_type( else: # For fields with a default, we'll get by even if there's no instantiator # available. - return dataclasses.replace( - lowered, - metavar="{fixed}", - required=False, - default=_fields.MISSING_PROP, - ) + lowered.metavar = "{fixed}" + lowered.required = False + lowered.default = _fields.MISSING_PROP + return if metadata.action == "append": @@ -342,31 +332,27 @@ def append_instantiator(x: Any) -> Any: return type(out)(arg.field.default) + out - return dataclasses.replace( - lowered, - instantiator=append_instantiator, - default=None, - choices=metadata.choices, - nargs=metadata.nargs, - metavar=metadata.metavar, - action=metadata.action, - required=False, - ) + lowered.instantiator = append_instantiator + lowered.default = None + lowered.choices = metadata.choices + lowered.nargs = metadata.nargs + lowered.metavar = metadata.metavar + lowered.action = metadata.action + lowered.required = False + return else: - return dataclasses.replace( - lowered, - instantiator=instantiator, - choices=metadata.choices, - nargs=metadata.nargs, - metavar=metadata.metavar, - action=metadata.action, - ) + lowered.instantiator = instantiator + lowered.choices = metadata.choices + lowered.nargs = metadata.nargs + lowered.metavar = metadata.metavar + lowered.action = metadata.action + return def _rule_convert_defaults_to_strings( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, -) -> LoweredArgumentDefinition: +) -> None: """Sets all default values to strings, as required as input to our instantiator functions. Special-cased for enums.""" @@ -387,9 +373,10 @@ def as_str(x: Any) -> Tuple[str, ...]: or lowered.default in _fields.MISSING_SINGLETONS or lowered.action is not None ): - return lowered + return else: - return dataclasses.replace(lowered, default=as_str(lowered.default)) + lowered.default = as_str(lowered.default) + return # This can be turned off when we don't want rich-based formatting. (notably for @@ -408,36 +395,34 @@ def _rich_tag_if_enabled(x: str, tag: str) -> str: def _rule_counters( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, -) -> LoweredArgumentDefinition: +) -> None: """Handle counters, like -vvv for level-3 verbosity.""" if ( _markers.UseCounterAction in arg.field.markers and arg.field.type_or_callable is int and not arg.field.is_positional() ): - return dataclasses.replace( - lowered, - metavar=None, - nargs=None, - action="count", - default=0, - required=False, - instantiator=lambda x: x, # argparse will directly give us an int! - ) - return lowered + lowered.metavar = None + lowered.nargs = None + lowered.action = "count" + lowered.default = 0 + lowered.required = False + lowered.instantiator = lambda x: x # argparse will directly give us an int! + return def _rule_generate_helptext( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, -) -> LoweredArgumentDefinition: +) -> None: """Generate helptext from docstring, argument name, default values.""" # If the suppress marker is attached, hide the argument. if _markers.Suppress in arg.field.markers or ( _markers.SuppressFixed in arg.field.markers and lowered.is_fixed() ): - return dataclasses.replace(lowered, help=argparse.SUPPRESS) + lowered.help = argparse.SUPPRESS + return help_parts = [] @@ -526,13 +511,14 @@ def _rule_generate_helptext( # Note that the percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string - return dataclasses.replace(lowered, help=" ".join(help_parts).replace("%", "%%")) + lowered.help = " ".join(help_parts).replace("%", "%%") + return def _rule_set_name_or_flag_and_dest( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, -) -> LoweredArgumentDefinition: +) -> None: name_or_flag = _strings.make_field_name( [arg.extern_prefix, arg.field.extern_name] if arg.field.argconf.prefix_name @@ -559,19 +545,16 @@ def _rule_set_name_or_flag_and_dest( assert name_or_flag.startswith(strip_prefix), name_or_flag name_or_flag = "--" + name_or_flag[len(strip_prefix) :] - return dataclasses.replace( - lowered, - name_or_flag=name_or_flag, - dest=_strings.make_field_name([arg.intern_prefix, arg.field.intern_name]), - ) + lowered.name_or_flag = name_or_flag + lowered.dest = _strings.make_field_name([arg.intern_prefix, arg.field.intern_name]) def _rule_positional_special_handling( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, -) -> LoweredArgumentDefinition: +) -> None: if not arg.field.is_positional(): - return lowered + return None metavar = lowered.metavar if lowered.required: @@ -587,37 +570,11 @@ def _rule_positional_special_handling( # If lowered.nargs is either + or an int. nargs = "*" - return dataclasses.replace( - lowered, - name_or_flag=_strings.make_field_name( - [arg.intern_prefix, arg.field.intern_name] - ), - dest=None, - required=None, # Can't be passed in for positionals. - metavar=metavar, - nargs=nargs, - ) - - -class _PatchedList(list): - """Custom list type, for avoiding "default not in choices" errors when the default - is set to MISSING_NONPROP. - - This solves a choices error raised by argparse in a very specific edge case: - literals in containers as positional arguments.""" - - def __init__(self, li): - super(_PatchedList, self).__init__(li) - - def __contains__(self, x: Any) -> bool: - return list.__contains__(self, x) or x is _fields.MISSING_NONPROP - - -def _rule_static_cast_choices_to_patched_list( - arg: ArgumentDefinition, - lowered: LoweredArgumentDefinition, -) -> LoweredArgumentDefinition: - return dataclasses.replace( - lowered, - choices=_PatchedList(lowered.choices) if lowered.choices is not None else None, + lowered.name_or_flag = _strings.make_field_name( + [arg.intern_prefix, arg.field.intern_name] ) + lowered.dest = None + lowered.required = None # Can't be passed in for positionals. + lowered.metavar = metavar + lowered.nargs = nargs + return diff --git a/src/tyro/_docstrings.py b/src/tyro/_docstrings.py index 714abe38d..841f3b504 100644 --- a/src/tyro/_docstrings.py +++ b/src/tyro/_docstrings.py @@ -154,45 +154,38 @@ def get_class_tokenization_with_field( return tokenization +@functools.lru_cache(maxsize=1024) +def parse_docstring_from_object(obj: object) -> Dict[str, str]: + return { + doc.arg_name: doc.description + for doc in docstring_parser.parse_from_object(obj).params + if doc.description is not None + } + + @_unsafe_cache.unsafe_cache(1024) def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: """Get docstring for a field in a class.""" - docstring = inspect.getdoc(cls) - if docstring is not None: - for param_doc in docstring_parser.parse(docstring).params: - if param_doc.arg_name == field_name: - return ( - _strings.remove_single_line_breaks(param_doc.description) - if param_doc.description is not None - else None - ) + # NoneType will break docstring_parser. + if cls is type(None): + return None + + # Try to parse using docstring_parser. + for cls_search in cls.__mro__: + if cls_search.__module__ == "builtins": + continue # Skip `object`, `Callable`, `tuple`, etc. + docstring = parse_docstring_from_object(cls_search).get(field_name, None) + if docstring is not None: + return _strings.remove_single_line_breaks(docstring) + # If docstring_parser failed, let's try looking for comments. tokenization = get_class_tokenization_with_field(cls, field_name) if tokenization is None: # Currently only happens for dynamic dataclasses. return None field_data = tokenization.field_data_from_name[field_name] - # Check for docstring-style comment. This should be on the next logical line. - logical_line = field_data.logical_line + 1 - if ( - logical_line in tokenization.tokens_from_logical_line - and len(tokenization.tokens_from_logical_line[logical_line]) >= 1 - ): - first_token = tokenization.tokens_from_logical_line[logical_line][0] - first_token_content = first_token.content.strip() - - # Found a docstring! - if ( - first_token.token_type == tokenize.STRING - and first_token_content.startswith('"""') - and first_token_content.endswith('"""') - ): - return _strings.remove_single_line_breaks( - _strings.dedent(first_token_content[3:-3]) - ) - # Check for comment on the same line as the field. final_token_on_line = tokenization.tokens_from_logical_line[ field_data.logical_line diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index 0cf9fe5a5..c1a7f96df 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -91,7 +91,7 @@ class InstantiatorMetadata: # Unlike in vanilla argparse, our metavar is always a string. We handle # sequences, multiple arguments, etc, manually. metavar: str - choices: Optional[Set[str]] + choices: Optional[Tuple[str, ...]] action: Optional[Literal["append"]] def check_choices(self, strings: List[str]) -> None: @@ -176,9 +176,7 @@ def instantiator(strings: List[str]) -> None: return instantiator, InstantiatorMetadata( nargs=1, metavar="{None}", - choices={ - "None", - }, + choices=("None",), action=None, ) @@ -196,7 +194,7 @@ def instantiator(strings: List[str]) -> None: if maybe_newtype_name is not None: metavar = maybe_newtype_name.upper() - typ, _ = _resolver.unwrap_annotated(typ) + typ = _resolver.unwrap_annotated(typ) # Address container types. If a matching container is found, this will recursively # call instantiator_from_type(). @@ -268,7 +266,7 @@ def instantiator_base_case(strings: List[str]) -> Any: if auto_choices is None else "{" + ",".join(map(str, auto_choices)) + "}" ), - choices=None if auto_choices is None else set(auto_choices), + choices=None if auto_choices is None else auto_choices, action=None, ) @@ -691,7 +689,7 @@ def _instantiator_from_literal( InstantiatorMetadata( nargs=1, metavar="{" + ",".join(str_choices) + "}", - choices=set(str_choices), + choices=str_choices, action=None, ), ) diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index ca623f0b2..50088c39e 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -388,7 +388,7 @@ def from_field( extern_prefix: str, ) -> Optional[SubparsersSpecification]: # Union of classes should create subparsers. - typ = _resolver.unwrap_annotated(field.type_or_callable)[0] + typ = _resolver.unwrap_annotated(field.type_or_callable) if get_origin(typ) is not Union: return None @@ -509,7 +509,7 @@ def from_field( # Strip the subcommand config from the option type. # Relevant: https://github.com/brentyi/tyro/pull/117 - option_origin, annotations = _resolver.unwrap_annotated(option) # type: ignore + option_origin, annotations = _resolver.unwrap_annotated(option, Any) # type: ignore annotations = tuple( a for a in annotations diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index a8d2ed8c6..dc4cb7d64 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -20,6 +20,7 @@ TypeVar, Union, cast, + overload, ) from typing_extensions import ( @@ -43,7 +44,7 @@ def unwrap_origin_strip_extras(typ: TypeOrCallable) -> TypeOrCallable: """Returns the origin, ignoring typing.Annotated, of typ if it exists. Otherwise, returns typ.""" # TODO: Annotated[] handling should be revisited... - typ, _ = unwrap_annotated(typ) + typ = unwrap_annotated(typ) origin = get_origin(typ) if origin is not None: @@ -68,7 +69,7 @@ def resolve_generic_types( # ^We need this `if` statement for an obscure edge case: when `cls` is a # function with `__tyro_markers__` set, we don't want/need to return # Annotated[func, markers]. - cls, annotations = unwrap_annotated(cls) + cls, annotations = unwrap_annotated(cls, Any) # We'll ignore NewType when getting the origin + args for generics. origin_cls = get_origin(unwrap_newtype_and_aliases(cls)[0]) @@ -79,9 +80,9 @@ def resolve_generic_types( if hasattr(cls, "__self__"): self_type = getattr(cls, "__self__") if inspect.isclass(self_type): - type_from_typevar[cast(TypeVar, Self)] = self_type + type_from_typevar[cast(TypeVar, Self)] = self_type # type: ignore else: - type_from_typevar[cast(TypeVar, Self)] = self_type.__class__ + type_from_typevar[cast(TypeVar, Self)] = self_type.__class__ # type: ignore if ( # Apply some heuristics for generic types. Should revisit this. @@ -214,7 +215,7 @@ def unwrap_newtype_and_narrow_subtypes( # it doesn't really make sense to parse this case. return typ - superclass = unwrap_annotated(typ)[0] + superclass = unwrap_annotated(typ) # For Python 3.10. if get_origin(superclass) is Union: @@ -256,10 +257,24 @@ def narrow_collection_types( MetadataType = TypeVar("MetadataType") +@overload def unwrap_annotated( typ: TypeOrCallable, - search_type: TypeForm[MetadataType] = cast(TypeForm[Any], Any), -) -> Tuple[TypeOrCallable, Tuple[MetadataType, ...]]: + search_type: Union[TypeForm[MetadataType], object], +) -> Tuple[TypeOrCallable, Tuple[MetadataType, ...]]: ... + + +@overload +def unwrap_annotated( + typ: TypeOrCallable, + search_type: None = None, +) -> TypeOrCallable: ... + + +def unwrap_annotated( + typ: TypeOrCallable, + search_type: Union[TypeForm[MetadataType], object, None] = None, +) -> Union[Tuple[TypeOrCallable, Tuple[MetadataType, ...]], TypeOrCallable]: """Helper for parsing typing.Annotated types. Examples: @@ -280,11 +295,18 @@ def unwrap_annotated( while get_origin(typ) is Final: typ = get_args(typ)[0] + # Don't search for any annotations. + if search_type is None: + if not hasattr(typ, "__metadata__"): + return typ + else: + return get_args(typ)[0] + # Check for __tyro_markers__ from @configure. targets = tuple( x for x in getattr(typ, "__tyro_markers__", tuple()) - if search_type is Any or isinstance(x, search_type) + if search_type is Any or isinstance(x, search_type) # type: ignore ) assert isinstance(targets, tuple) if not hasattr(typ, "__metadata__"): @@ -297,14 +319,14 @@ def unwrap_annotated( targets += tuple( x for x in targets + args[1:] - if search_type is Any or isinstance(x, search_type) + if search_type is Any or isinstance(x, search_type) # type: ignore ) # Check for __tyro_markers__ in unwrapped type. targets += tuple( x for x in getattr(args[0], "__tyro_markers__", tuple()) - if search_type is Any or isinstance(x, search_type) + if search_type is Any or isinstance(x, search_type) # type: ignore ) return args[0], targets # type: ignore diff --git a/src/tyro/_strings.py b/src/tyro/_strings.py index 91a950662..e2fc7c09f 100644 --- a/src/tyro/_strings.py +++ b/src/tyro/_strings.py @@ -4,7 +4,7 @@ import functools import re import textwrap -from typing import Iterable, List, Sequence, Tuple, Type +from typing import List, Sequence, Tuple, Type from typing_extensions import Literal, get_args, get_origin @@ -14,10 +14,6 @@ DELIMETER: Literal["-", "_"] = "-" -def _strip_dummy_field_names(parts: Iterable[str]) -> Iterable[str]: - return filter(lambda name: len(name) > 0 and name != dummy_field_name, parts) - - @contextlib.contextmanager def delimeter_context(delimeter: Literal["-", "_"]): """Context for setting the delimeter. Determines if `field_a` is populated as @@ -37,13 +33,8 @@ def get_delimeter() -> Literal["-", "_"]: def replace_delimeter_in_part(p: str) -> str: """Replace hyphens with underscores (or vice versa) except when at the start.""" if get_delimeter() == "-": - num_underscore_prefix = 0 - for i in range(len(p)): - if p[i] == "_": - num_underscore_prefix += 1 - else: - break - p = "_" * num_underscore_prefix + (p[num_underscore_prefix:].replace("_", "-")) + stripped = p.lstrip("_") + p = p[: len(p) - len(stripped)] + stripped.replace("_", "-") else: p = p.replace("-", "_") return p @@ -56,10 +47,12 @@ def make_field_name(parts: Sequence[str]) -> str: ('parents', '1', '_child_node') => 'parents.1._child-node' ('parents', '1', 'middle._child_node') => 'parents.1.middle._child-node' """ - out: List[str] = [] - for p in _strip_dummy_field_names(parts): - out.extend(map(replace_delimeter_in_part, p.split("."))) - return ".".join(out) + out = ".".join(parts) + return ".".join( + replace_delimeter_in_part(part) + for part in out.split(".") + if len(part) > 0 and part != dummy_field_name + ) def make_subparser_dest(name: str) -> str: diff --git a/src/tyro/_subcommand_matching.py b/src/tyro/_subcommand_matching.py index 16884bcf8..e9a45a425 100644 --- a/src/tyro/_subcommand_matching.py +++ b/src/tyro/_subcommand_matching.py @@ -96,11 +96,11 @@ def _get_type_options(typ: _typing.TypeForm) -> Tuple[_typing.TypeForm, ...]: # Check against supertypes. for self_type in self_types: - self_type = _resolver.unwrap_annotated(self_type)[0] + self_type = _resolver.unwrap_annotated(self_type) self_type, _ = _resolver.unwrap_newtype_and_aliases(self_type) ok = False for super_type in super_types: - super_type = _resolver.unwrap_annotated(super_type)[0] + super_type = _resolver.unwrap_annotated(super_type) self_type, _ = _resolver.unwrap_newtype_and_aliases(self_type) if issubclass(self_type, super_type): ok = True diff --git a/src/tyro/extras/_serialization.py b/src/tyro/extras/_serialization.py index 6ae4771e1..b478f9e3c 100644 --- a/src/tyro/extras/_serialization.py +++ b/src/tyro/extras/_serialization.py @@ -35,7 +35,7 @@ def _get_contained_special_types_from_type( else _parent_contained_dataclasses ) - cls, _ = _resolver.unwrap_annotated(cls) + cls = _resolver.unwrap_annotated(cls) cls, type_from_typevar = _resolver.resolve_generic_types(cls) contained_special_types = {cls} From 84991dadbbd734268f364118c3f0130f7f2358fa Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 9 Jun 2024 02:48:29 -0700 Subject: [PATCH 404/491] Profiling, more performance improvements --- src/tyro/_fields.py | 101 +++++++++++------- src/tyro/_parsers.py | 4 +- src/tyro/_resolver.py | 38 ++++--- tests/test_missing_optional_packages.py | 9 +- tests/test_py311_generated/_generate.py | 8 +- ...est_missing_optional_packages_generated.py | 9 +- 6 files changed, 105 insertions(+), 64 deletions(-) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 0b269c10d..1e02f6cfd 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -20,7 +20,6 @@ Any, Callable, Dict, - FrozenSet, Hashable, Iterable, List, @@ -56,7 +55,7 @@ from .conf import _confstruct, _markers -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass class FieldDefinition: intern_name: str extern_name: str @@ -69,7 +68,7 @@ class FieldDefinition: # tyro.conf.subcommand(default=...). is_default_from_default_instance: bool helptext: Optional[str] - markers: FrozenSet[Any] + markers: Set[Any] custom_constructor: bool argconf: _confstruct._ArgConfiguration @@ -134,7 +133,7 @@ def make( default=default, is_default_from_default_instance=is_default_from_default_instance, helptext=helptext, - markers=frozenset(inferred_markers).union(markers), + markers=set(inferred_markers).union(markers), custom_constructor=argconf.constructor_factory is not None, argconf=argconf, call_argname=call_argname_override @@ -296,9 +295,7 @@ def field_list_from_callable( is_default_from_default_instance=True, helptext="", custom_constructor=False, - markers=frozenset( - (_markers.Positional, _markers._PositionalCall) - ), + markers={_markers.Positional, _markers._PositionalCall}, argconf=_confstruct._ArgConfiguration( None, None, None, None, None, None ), @@ -414,15 +411,16 @@ def _try_field_list_from_callable( # Try field generation from class inputs. if cls is not None: - for match, field_list_from_class in ( - (is_typeddict, _field_list_from_typeddict), - (_resolver.is_namedtuple, _field_list_from_namedtuple), - (_resolver.is_dataclass, _field_list_from_dataclass), - (_is_attrs, _field_list_from_attrs), - (_is_pydantic, _field_list_from_pydantic), - ): - if match(cls): - return field_list_from_class(cls, default_instance) + if is_typeddict(cls): + return _field_list_from_typeddict(cls, default_instance) + if _resolver.is_namedtuple(cls): + return _field_list_from_namedtuple(cls, default_instance) + if _resolver.is_dataclass(cls): + return _field_list_from_dataclass(cls, default_instance) + if _is_attrs(cls): + return _field_list_from_attrs(cls, default_instance) + if _is_pydantic(cls): + return _field_list_from_pydantic(cls, default_instance) # Standard container types. These are different because they can be nested structures # if they contain other nested types (eg Tuple[Struct, Struct]), or treated as @@ -608,26 +606,26 @@ def _field_list_from_dataclass( return field_list -# Support attrs and pydantic if they're installed. -try: - import pydantic -except ImportError: - if not TYPE_CHECKING: - pydantic = None # type: ignore - else: - import pydantic +def _is_pydantic(cls: TypeForm[Any]) -> bool: + # pydantic will already be imported if it's used. + if "pydantic" not in sys.modules.keys(): + return False -try: - from pydantic import v1 as pydantic_v1 -except ImportError: - if not TYPE_CHECKING: - pydantic_v1 = None # type: ignore - else: - from pydantic import v1 as pydantic_v1 + # Jump through a bunch of hoops to minimize unnecessary imports. + import pydantic + try: + if "pydantic.v1" in sys.modules.keys(): + from pydantic import v1 as pydantic_v1 + else: + pydantic_v1 = None + except ImportError: + if TYPE_CHECKING: + from pydantic import v1 as pydantic_v1 + else: + pydantic_v1 = None -def _is_pydantic(cls: TypeForm[Any]) -> bool: - if pydantic is not None and issubclass(cls, pydantic.BaseModel): + if issubclass(cls, pydantic.BaseModel): return True if pydantic_v1 is not None and issubclass(cls, pydantic_v1.BaseModel): return True @@ -637,7 +635,18 @@ def _is_pydantic(cls: TypeForm[Any]) -> bool: def _field_list_from_pydantic( cls: TypeForm[Any], default_instance: DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - assert pydantic is not None + import pydantic + + try: + if "pydantic.v1" in sys.modules.keys(): + from pydantic import v1 as pydantic_v1 + else: + pydantic_v1 = None + except ImportError: + if TYPE_CHECKING: + from pydantic import v1 as pydantic_v1 + else: + pydantic_v1 = None # Handle pydantic models. field_list = [] @@ -697,20 +706,25 @@ def _field_list_from_pydantic( return field_list -try: - import attr -except ImportError: - attr = None # type: ignore +def _is_attrs(cls: TypeForm[Any]) -> bool: + # attr will already be imported if it's used. + if "attr" not in sys.modules.keys(): + return False + try: + import attr + except ImportError: + # This is needed for the mock import test in + # test_missing_optional_packages.py to pass... + return False -def _is_attrs(cls: TypeForm[Any]) -> bool: - return attr is not None and attr.has(cls) + return attr.has(cls) def _field_list_from_attrs( cls: TypeForm[Any], default_instance: DefaultInstance ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - assert attr is not None + import attr # Resolve forward references in-place, if any exist. attr.resolve_types(cls) @@ -1105,6 +1119,11 @@ def _get_dataclass_field_default( return MISSING_NONPROP, False +if TYPE_CHECKING: + import pydantic as pydantic + import pydantic.v1 as pydantic_v1 + + def _get_pydantic_v1_field_default( name: str, field: pydantic_v1.fields.ModelField, diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 50088c39e..0b0d4087d 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -85,9 +85,7 @@ def from_callable_or_type( support_single_arg_types=support_single_arg_types, ) for i in range(len(field_list)): - field_list[i] = dataclasses.replace( - field_list[i], markers=field_list[i].markers | set(markers) - ) + field_list[i].markers |= set(markers) # Cycle detection. # diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index dc4cb7d64..62571c532 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -271,6 +271,19 @@ def unwrap_annotated( ) -> TypeOrCallable: ... +# `Final` and `ReadOnly` types are ignored in tyro. +try: + # Can only import ReadOnly in typing_extensions>=4.9.0, which isn't + # supported by Python 3.7. + from typing_extensions import ReadOnly + + STRIP_TYPES = {Final, ReadOnly} +except ImportError: + STRIP_TYPES = { + Final, + } + + def unwrap_annotated( typ: TypeOrCallable, search_type: Union[TypeForm[MetadataType], object, None] = None, @@ -284,16 +297,8 @@ def unwrap_annotated( """ # `Final` and `ReadOnly` types are ignored in tyro. - try: - # Can only import ReadOnly in typing_extensions>=4.9.0, which isn't - # supported by Python 3.7. - from typing_extensions import ReadOnly - - while get_origin(typ) in (Final, ReadOnly): - typ = get_args(typ)[0] - except ImportError: # pragma: no cover - while get_origin(typ) is Final: - typ = get_args(typ)[0] + while get_origin(typ) in STRIP_TYPES: + typ = get_args(typ)[0] # Don't search for any annotations. if search_type is None: @@ -303,11 +308,14 @@ def unwrap_annotated( return get_args(typ)[0] # Check for __tyro_markers__ from @configure. - targets = tuple( - x - for x in getattr(typ, "__tyro_markers__", tuple()) - if search_type is Any or isinstance(x, search_type) # type: ignore - ) + if search_type is Any: + targets = getattr(typ, "__tyro_markers__", tuple()) + else: + targets = tuple( + x + for x in getattr(typ, "__tyro_markers__", tuple()) + if isinstance(x, search_type) # type: ignore + ) assert isinstance(targets, tuple) if not hasattr(typ, "__metadata__"): return typ, targets # type: ignore diff --git a/tests/test_missing_optional_packages.py b/tests/test_missing_optional_packages.py index a093baba0..1e3afcdec 100644 --- a/tests/test_missing_optional_packages.py +++ b/tests/test_missing_optional_packages.py @@ -1,4 +1,5 @@ import builtins +import dataclasses import pytest @@ -17,7 +18,13 @@ def mocked_import(name, *args, **kwargs): def test_missing_optional_packages(hide_optional_packages): - def main(x: int) -> int: + @dataclasses.dataclass + class Extra: + a: int = 3 + b: str = "5" + + def main(x: int, extra: Extra) -> int: + del extra return x import tyro diff --git a/tests/test_py311_generated/_generate.py b/tests/test_py311_generated/_generate.py index 6fcc279a0..77c93968a 100644 --- a/tests/test_py311_generated/_generate.py +++ b/tests/test_py311_generated/_generate.py @@ -55,7 +55,9 @@ def generate_from_path(test_path: pathlib.Path) -> None: with ThreadPoolExecutor(max_workers=8) as executor: - executor.map( - generate_from_path, - pathlib.Path(__file__).absolute().parent.parent.glob("test_*.py"), + list( + executor.map( + generate_from_path, + pathlib.Path(__file__).absolute().parent.parent.glob("test_*.py"), + ) ) diff --git a/tests/test_py311_generated/test_missing_optional_packages_generated.py b/tests/test_py311_generated/test_missing_optional_packages_generated.py index a093baba0..1e3afcdec 100644 --- a/tests/test_py311_generated/test_missing_optional_packages_generated.py +++ b/tests/test_py311_generated/test_missing_optional_packages_generated.py @@ -1,4 +1,5 @@ import builtins +import dataclasses import pytest @@ -17,7 +18,13 @@ def mocked_import(name, *args, **kwargs): def test_missing_optional_packages(hide_optional_packages): - def main(x: int) -> int: + @dataclasses.dataclass + class Extra: + a: int = 3 + b: str = "5" + + def main(x: int, extra: Extra) -> int: + del extra return x import tyro From 0969dc86e8b06ddf16a28cf0fba61444a1ca0e5b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 10 Jun 2024 00:21:11 -0700 Subject: [PATCH 405/491] Bump docstring_parser, fix missing import test --- pyproject.toml | 2 +- src/tyro/_fields.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 231bbd7e8..a5a82dc66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ "Operating System :: OS Independent" ] dependencies = [ - "docstring-parser>=0.14.1", + "docstring-parser>=0.16", "typing-extensions>=4.7.0", "backports.cached-property>=1.0.2; python_version<'3.8'", "colorama>=0.4.0; platform_system=='Windows'", diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 1e02f6cfd..72db4e95b 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -42,6 +42,7 @@ is_typeddict, ) +from . import conf # Avoid circular import. from . import ( _docstrings, _instantiators, @@ -49,7 +50,6 @@ _singleton, _strings, _unsafe_cache, - conf, # Avoid circular import. ) from ._typing import TypeForm from .conf import _confstruct, _markers @@ -609,10 +609,14 @@ def _field_list_from_dataclass( def _is_pydantic(cls: TypeForm[Any]) -> bool: # pydantic will already be imported if it's used. if "pydantic" not in sys.modules.keys(): + # This is needed for the mock import test in + # test_missing_optional_packages.py to pass. return False - # Jump through a bunch of hoops to minimize unnecessary imports. - import pydantic + try: + import pydantic + except ImportError: + return False try: if "pydantic.v1" in sys.modules.keys(): @@ -715,7 +719,7 @@ def _is_attrs(cls: TypeForm[Any]) -> bool: import attr except ImportError: # This is needed for the mock import test in - # test_missing_optional_packages.py to pass... + # test_missing_optional_packages.py to pass. return False return attr.has(cls) From 22300760f717dd3dfadecff36b0646fb00e9ce34 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 10 Jun 2024 00:28:44 -0700 Subject: [PATCH 406/491] Typing fixes --- src/tyro/_fields.py | 4 ++-- src/tyro/_instantiators.py | 23 +++++++++++------------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 72db4e95b..48c1535b4 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -42,7 +42,6 @@ is_typeddict, ) -from . import conf # Avoid circular import. from . import ( _docstrings, _instantiators, @@ -50,6 +49,7 @@ _singleton, _strings, _unsafe_cache, + conf, # Avoid circular import. ) from ._typing import TypeForm from .conf import _confstruct, _markers @@ -662,7 +662,7 @@ def _field_list_from_pydantic( # We do a conditional cast because the pydantic.v1 module won't # actually exist in legacy versions of pydantic. if TYPE_CHECKING: - cls_cast = cast(pydantic_v1.BaseModel, cls) + cls_cast = cast(pydantic_v1.BaseModel, cls) # type: ignore else: cls_cast = cls for pd1_field in cls_cast.__fields__.values(): diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index c1a7f96df..dad0d30b1 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -42,7 +42,6 @@ Any, Callable, Dict, - FrozenSet, Hashable, Iterable, List, @@ -151,7 +150,7 @@ def is_type_string_converter(typ: Union[Callable, TypeForm[Any]]) -> bool: def instantiator_from_type( typ: Union[TypeForm[Any], Callable], type_from_typevar: Dict[TypeVar, TypeForm[Any]], - markers: FrozenSet[_markers.Marker], + markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: """Recursive helper for parsing type annotations. @@ -276,7 +275,7 @@ def _instantiator_from_type_inner( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal["fixed_length"], - markers: FrozenSet[_markers.Marker], + markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: ... @@ -285,7 +284,7 @@ def _instantiator_from_type_inner( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal[False], - markers: FrozenSet[_markers.Marker], + markers: Set[_markers.Marker], ) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: ... @@ -294,7 +293,7 @@ def _instantiator_from_type_inner( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal[True], - markers: FrozenSet[_markers.Marker], + markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: ... @@ -302,7 +301,7 @@ def _instantiator_from_type_inner( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal["fixed_length", True, False], - markers: FrozenSet[_markers.Marker], + markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: """Thin wrapper over instantiator_from_type, with some extra asserts for catching errors.""" @@ -324,7 +323,7 @@ def _instantiator_from_type_inner( def _instantiator_from_container_type( typ: TypeForm[Any], type_from_typevar: Dict[TypeVar, TypeForm[Any]], - markers: FrozenSet[_markers.Marker], + markers: Set[_markers.Marker], ) -> Optional[Tuple[Instantiator, InstantiatorMetadata]]: """Attempt to create an instantiator from a container type. Returns `None` if no container type is found.""" @@ -364,7 +363,7 @@ def _instantiator_from_container_type( def _instantiator_from_tuple( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], - markers: FrozenSet[_markers.Marker], + markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: types = get_args(typ) typeset = set(types) # Note that sets are unordered. @@ -449,7 +448,7 @@ def _join_union_metavars(metavars: Iterable[str]) -> str: def _instantiator_from_union( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], - markers: FrozenSet[_markers.Marker], + markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: options = list(get_args(typ)) if NoneType in options: @@ -526,7 +525,7 @@ def union_instantiator(strings: List[str]) -> Any: def _instantiator_from_dict( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], - markers: FrozenSet[_markers.Marker], + markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: key_type, val_type = get_args(typ) key_instantiator, key_meta = _instantiator_from_type_inner( @@ -608,7 +607,7 @@ def dict_instantiator(strings: List[str]) -> Any: def _instantiator_from_sequence( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], - markers: FrozenSet[_markers.Marker], + markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: """Instantiator for variable-length sequences: list, sets, Tuple[T, ...], etc.""" container_type = get_origin(typ) @@ -678,7 +677,7 @@ def sequence_instantiator(strings: List[str]) -> Any: def _instantiator_from_literal( typ: TypeForm, type_from_typevar: Dict[TypeVar, TypeForm[Any]], - markers: FrozenSet[_markers.Marker], + markers: Set[_markers.Marker], ) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: choices = get_args(typ) str_choices = tuple(x.name if isinstance(x, enum.Enum) else str(x) for x in choices) From c886390c68ac3659ea5b735ea0bfad99fb82b0b9 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 10 Jun 2024 22:09:37 -0700 Subject: [PATCH 407/491] Fix mypy errors --- src/tyro/_fields.py | 4 +-- src/tyro/_parsers.py | 4 +-- src/tyro/_resolver.py | 70 ++++++++++++++++++++++++------------------- 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 48c1535b4..11728c337 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -622,7 +622,7 @@ def _is_pydantic(cls: TypeForm[Any]) -> bool: if "pydantic.v1" in sys.modules.keys(): from pydantic import v1 as pydantic_v1 else: - pydantic_v1 = None + pydantic_v1 = None # type: ignore except ImportError: if TYPE_CHECKING: from pydantic import v1 as pydantic_v1 @@ -645,7 +645,7 @@ def _field_list_from_pydantic( if "pydantic.v1" in sys.modules.keys(): from pydantic import v1 as pydantic_v1 else: - pydantic_v1 = None + pydantic_v1 = None # type: ignore except ImportError: if TYPE_CHECKING: from pydantic import v1 as pydantic_v1 diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 0b0d4087d..8d341f55d 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -416,7 +416,7 @@ def from_field( options[i] = Annotated.__class_getitem__( # type: ignore ( found_subcommand_configs[0].constructor_factory(), - *_resolver.unwrap_annotated(option, Any)[1], # type: ignore + *_resolver.unwrap_annotated(option, "all")[1], ) ) @@ -507,7 +507,7 @@ def from_field( # Strip the subcommand config from the option type. # Relevant: https://github.com/brentyi/tyro/pull/117 - option_origin, annotations = _resolver.unwrap_annotated(option, Any) # type: ignore + option_origin, annotations = _resolver.unwrap_annotated(option, "all") annotations = tuple( a for a in annotations diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 62571c532..25e91d66b 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -27,6 +27,7 @@ Annotated, Final, ForwardRef, + Literal, Self, TypeAliasType, get_args, @@ -69,7 +70,7 @@ def resolve_generic_types( # ^We need this `if` statement for an obscure edge case: when `cls` is a # function with `__tyro_markers__` set, we don't want/need to return # Annotated[func, markers]. - cls, annotations = unwrap_annotated(cls, Any) + cls, annotations = unwrap_annotated(cls, "all") # We'll ignore NewType when getting the origin + args for generics. origin_cls = get_origin(unwrap_newtype_and_aliases(cls)[0]) @@ -254,39 +255,43 @@ def narrow_collection_types( return typ +# `Final` and `ReadOnly` types are ignored in tyro. +try: + # Can only import ReadOnly in typing_extensions>=4.9.0, which isn't + # supported by Python 3.7. + from typing_extensions import ReadOnly + + STRIP_WRAPPER_TYPES = {Final, ReadOnly} +except ImportError: + STRIP_WRAPPER_TYPES = {Final} + MetadataType = TypeVar("MetadataType") @overload def unwrap_annotated( typ: TypeOrCallable, - search_type: Union[TypeForm[MetadataType], object], + search_type: TypeForm[MetadataType], ) -> Tuple[TypeOrCallable, Tuple[MetadataType, ...]]: ... @overload def unwrap_annotated( typ: TypeOrCallable, - search_type: None = None, -) -> TypeOrCallable: ... + search_type: Literal["all"], +) -> Tuple[TypeOrCallable, Tuple[Any, ...]]: ... -# `Final` and `ReadOnly` types are ignored in tyro. -try: - # Can only import ReadOnly in typing_extensions>=4.9.0, which isn't - # supported by Python 3.7. - from typing_extensions import ReadOnly - - STRIP_TYPES = {Final, ReadOnly} -except ImportError: - STRIP_TYPES = { - Final, - } +@overload +def unwrap_annotated( + typ: TypeOrCallable, + search_type: None = None, +) -> TypeOrCallable: ... def unwrap_annotated( typ: TypeOrCallable, - search_type: Union[TypeForm[MetadataType], object, None] = None, + search_type: Union[TypeForm[MetadataType], Literal["all"], object, None] = None, ) -> Union[Tuple[TypeOrCallable, Tuple[MetadataType, ...]], TypeOrCallable]: """Helper for parsing typing.Annotated types. @@ -297,7 +302,7 @@ def unwrap_annotated( """ # `Final` and `ReadOnly` types are ignored in tyro. - while get_origin(typ) in STRIP_TYPES: + while get_origin(typ) in STRIP_WRAPPER_TYPES: typ = get_args(typ)[0] # Don't search for any annotations. @@ -308,14 +313,18 @@ def unwrap_annotated( return get_args(typ)[0] # Check for __tyro_markers__ from @configure. - if search_type is Any: - targets = getattr(typ, "__tyro_markers__", tuple()) + if hasattr(typ, "__tyro_markers__"): + if search_type == "all": + targets = getattr(typ, "__tyro_markers__") + else: + targets = tuple( + x + for x in getattr(typ, "__tyro_markers__") + if isinstance(x, search_type) # type: ignore + ) else: - targets = tuple( - x - for x in getattr(typ, "__tyro_markers__", tuple()) - if isinstance(x, search_type) # type: ignore - ) + targets = () + assert isinstance(targets, tuple) if not hasattr(typ, "__metadata__"): return typ, targets # type: ignore @@ -327,15 +336,16 @@ def unwrap_annotated( targets += tuple( x for x in targets + args[1:] - if search_type is Any or isinstance(x, search_type) # type: ignore + if search_type == "all" or isinstance(x, search_type) # type: ignore ) # Check for __tyro_markers__ in unwrapped type. - targets += tuple( - x - for x in getattr(args[0], "__tyro_markers__", tuple()) - if search_type is Any or isinstance(x, search_type) # type: ignore - ) + if hasattr(args[0], "__tyro_markers__"): + targets += tuple( + x + for x in getattr(args[0], "__tyro_markers__") + if search_type == "all" or isinstance(x, search_type) # type: ignore + ) return args[0], targets # type: ignore From 720be8b5e63b7a57c7044afe32bb6776a020d9ec Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 11 Jun 2024 17:31:18 -0700 Subject: [PATCH 408/491] 0.8.5 --- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a5a82dc66..bf453bb5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.4" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.5" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index 5f94556b7..dd34b56a9 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.4" +__version__ = "0.8.5" From 958f5fbe539fb67ab50fbba4d6b8ad33b87384aa Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 25 Jun 2024 03:19:11 +0900 Subject: [PATCH 409/491] Improve error messages for custom constructors + tests (cc #145) - We now apply custom constructors from `tyro.conf.subcommand()` even when used outside of a subcommand. This is undefined behavior either way, but it makes error messages less cryptic. - We add assertions to avoid situations where `tyro.conf.subcommand()` is applied outside of a Union type. - Added tests. --- src/tyro/_calling.py | 1 + src/tyro/_fields.py | 13 +++------- src/tyro/_resolver.py | 25 +++++++++++++++++++- src/tyro/extras/_base_configs.py | 2 ++ src/tyro/extras/_subcommand_cli_from_dict.py | 2 ++ tests/test_conf.py | 15 ++++++++++++ tests/test_nested.py | 10 ++++++++ 7 files changed, 57 insertions(+), 11 deletions(-) diff --git a/src/tyro/_calling.py b/src/tyro/_calling.py index 30f51becf..b703f0b77 100644 --- a/src/tyro/_calling.py +++ b/src/tyro/_calling.py @@ -220,6 +220,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: # sometimes produces types like `Tuple[T1, T2, ...]`, where we actually want just # `tuple`. unwrapped_f = f + unwrapped_f = _resolver.swap_type_using_confstruct(unwrapped_f) unwrapped_f = _resolver.unwrap_origin_strip_extras(unwrapped_f) unwrapped_f = _resolver.unwrap_newtype_and_narrow_subtypes( unwrapped_f, default_instance diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 11728c337..7e8fea504 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -243,16 +243,6 @@ def is_nested_type( TODO: we should come up with a better name than 'nested type', which is a little bit misleading.""" - # Need to swap types. - _, argconfs = _resolver.unwrap_annotated( - typ, search_type=conf._confstruct._ArgConfiguration - ) - _, subcommand_confs = _resolver.unwrap_annotated( - typ, search_type=conf._confstruct._SubcommandConfiguration - ) - for union_conf in argconfs + subcommand_confs: - if union_conf.constructor_factory is not None: - typ = union_conf.constructor_factory() return not isinstance( _try_field_list_from_callable(typ, default_instance), UnsupportedNestedTypeMessage, @@ -376,6 +366,9 @@ def _try_field_list_from_callable( f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: + # Apply constructor_factory override for type. + f = _resolver.swap_type_using_confstruct(f) + # Check for default instances in subcommand configs. This is needed for # is_nested_type() when arguments are not valid without a default, and this # default is specified in the subcommand config. diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 25e91d66b..09de425b5 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -35,7 +35,7 @@ get_type_hints, ) -from . import _fields, _unsafe_cache +from . import _fields, _unsafe_cache, conf from ._typing import TypeForm TypeOrCallable = TypeVar("TypeOrCallable", TypeForm[Any], Callable) @@ -236,6 +236,29 @@ def unwrap_newtype_and_narrow_subtypes( return typ +def swap_type_using_confstruct(typ: TypeOrCallable) -> TypeOrCallable: + """Swap types using the `constructor_factory` attribute from + `tyro.conf.arg` and `tyro.conf.subcommand`. Runtime annotations are + kept, but the type is swapped.""" + # Need to swap types. + _, annotations = unwrap_annotated(typ, search_type="all") + for anno in reversed(annotations): + if ( + isinstance( + anno, + ( + conf._confstruct._ArgConfiguration, + conf._confstruct._SubcommandConfiguration, + ), + ) + and anno.constructor_factory is not None + ): + return Annotated.__class_getitem__( # type: ignore + (anno.constructor_factory(),) + annotations + ) + return typ + + def narrow_collection_types( typ: TypeOrCallable, default_instance: Any ) -> TypeOrCallable: diff --git a/src/tyro/extras/_base_configs.py b/src/tyro/extras/_base_configs.py index 083bb2c99..117a944a3 100644 --- a/src/tyro/extras/_base_configs.py +++ b/src/tyro/extras/_base_configs.py @@ -67,6 +67,8 @@ def subcommand_type_from_defaults( Returns: A subcommand type, which can be passed to :func:`tyro.cli`. """ + # We need to form a union type, which requires at least two elements. + assert len(defaults) >= 2, "At least two subcommands are required." return Union.__getitem__( # type: ignore tuple( Annotated.__class_getitem__( # type: ignore diff --git a/src/tyro/extras/_subcommand_cli_from_dict.py b/src/tyro/extras/_subcommand_cli_from_dict.py index b2f769630..6710be40f 100644 --- a/src/tyro/extras/_subcommand_cli_from_dict.py +++ b/src/tyro/extras/_subcommand_cli_from_dict.py @@ -87,6 +87,8 @@ def subcommand_cli_from_dict( when parsing happens. We default helptext to hyphens to follow the GNU style guide. https://www.gnu.org/software/libc/manual/html_node/Argument-Syntax.html """ + # We need to form a union type, which requires at least two elements. + assert len(subcommands) >= 2, "At least two subcommands are required." return cli( Union.__getitem__( # type: ignore tuple( diff --git a/tests/test_conf.py b/tests/test_conf.py index 4cdfa4afb..dcacef4c1 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1181,6 +1181,21 @@ class Config: assert "'b'" not in error +def test_custom_constructor_9() -> None: + def commit(branch: str) -> int: + """Commit""" + print(f"commit {branch=}") + return 3 + + assert ( + tyro.cli( + Annotated[Any, tyro.conf.arg(constructor=commit)], + args="--branch 5".split(" "), + ) + == 3 + ) + + def test_alias() -> None: """Arguments with aliases.""" diff --git a/tests/test_nested.py b/tests/test_nested.py index b9ef69327..0264057df 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -1138,6 +1138,16 @@ def commit(message: str, all: bool = False) -> Tuple[str, bool]: """Make a commit.""" return (message, all) + # If we only get one, we unfortunately can't form subcommands. This is + # because unions in Python require at least 2 types. + with pytest.raises(AssertionError): + tyro.extras.subcommand_cli_from_dict( + { + "commit": commit, + }, + args="--message hello --all".split(" "), + ) + assert ( tyro.extras.subcommand_cli_from_dict( { From f1c5342190e9e4252c6c8fca44ae8a14860d5412 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 25 Jun 2024 03:26:53 +0900 Subject: [PATCH 410/491] Fix pyright error from pydantic 2.7 update --- src/tyro/_fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 7e8fea504..2d3343ee4 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -1118,12 +1118,12 @@ def _get_dataclass_field_default( if TYPE_CHECKING: import pydantic as pydantic - import pydantic.v1 as pydantic_v1 + import pydantic.v1.fields as pydantic_v1_fields def _get_pydantic_v1_field_default( name: str, - field: pydantic_v1.fields.ModelField, + field: pydantic_v1_fields.ModelField, parent_default_instance: DefaultInstance, ) -> Tuple[Any, bool]: """Helper for getting the default instance for a Pydantic field.""" From c75bea78b595be9c2d15163e60648a014f770845 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 25 Jun 2024 03:47:25 +0900 Subject: [PATCH 411/491] Fix Python 3.7 tests --- tests/test_conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_conf.py b/tests/test_conf.py index dcacef4c1..37290d519 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1184,7 +1184,7 @@ class Config: def test_custom_constructor_9() -> None: def commit(branch: str) -> int: """Commit""" - print(f"commit {branch=}") + print(f"commit branch={branch}") return 3 assert ( From b2bea5c5fe05a60f43c816718bfa3943f47658ce Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 25 Jun 2024 03:47:56 +0900 Subject: [PATCH 412/491] Improve error messages for unsupported type annotations --- src/tyro/_arguments.py | 19 ++++++++++++++----- tests/test_errors.py | 13 +++++++++++-- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index 9a1ff1ea5..2779131c2 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -310,11 +310,20 @@ def _rule_recursive_instantiator_from_type( ) except _instantiators.UnsupportedTypeAnnotationError as e: if arg.field.default in _fields.MISSING_SINGLETONS: - raise _instantiators.UnsupportedTypeAnnotationError( - "Unsupported type annotation for the field" - f" {_strings.make_field_name([arg.extern_prefix, arg.field.intern_name])}. To" - " suppress this error, assign the field a default value." - ) from e + field_name = _strings.make_field_name( + [arg.extern_prefix, arg.field.intern_name] + ) + if field_name != "": + raise _instantiators.UnsupportedTypeAnnotationError( + f"Unsupported type annotation for the field {field_name}; " + f"{e.args[0]} " + "To suppress this error, assign the field either a default value or a different type." + ) from e + else: + # If the field name is empty, it means we're raising an error + # for the direct input to `tyro.cli()`. We don't need to write + # out which specific field we're complaining about. + raise e else: # For fields with a default, we'll get by even if there's no instantiator # available. diff --git a/tests/test_errors.py b/tests/test_errors.py index 4b522ed35..b45e515a6 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -31,16 +31,25 @@ def test_ambiguous_collection_2() -> None: def main(x: Tuple[List[str], List[str]]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(tyro.UnsupportedTypeAnnotationError) as e: tyro.cli(main, args=["--help"]) + assert "Unsupported type annotation for the field x" in e.value.args[0] def test_ambiguous_collection_3() -> None: def main(x: List[Union[Tuple[int, int], Tuple[int, int, int]]]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(tyro.UnsupportedTypeAnnotationError) as e: tyro.cli(main, args=["--help"]) + assert "Unsupported type annotation for the field x" in e.value.args[0] + + +def test_ambiguous_collection_4() -> None: + X = List[Union[Tuple[int, int], Tuple[int, int, int]]] + with pytest.raises(tyro.UnsupportedTypeAnnotationError) as e: + tyro.cli(X, args=["--help"]) + assert "Unsupported type annotation for the field" not in e.value.args[0] # Must be global. From 202ef2b4a48107e1e3db216aa9393b37d846379a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 25 Jun 2024 03:51:49 +0900 Subject: [PATCH 413/491] Type ignore for mypy --- tests/test_conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_conf.py b/tests/test_conf.py index 37290d519..067c51df0 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1188,7 +1188,7 @@ def commit(branch: str) -> int: return 3 assert ( - tyro.cli( + tyro.cli( # type: ignore Annotated[Any, tyro.conf.arg(constructor=commit)], args="--branch 5".split(" "), ) From 532d38f9157041f8ccf2d85850e1a34ecee6a906 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 24 Jun 2024 12:13:56 -0700 Subject: [PATCH 414/491] Whitespace formatting nit for docstrings --- src/tyro/_docstrings.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tyro/_docstrings.py b/src/tyro/_docstrings.py index 841f3b504..a268b5c10 100644 --- a/src/tyro/_docstrings.py +++ b/src/tyro/_docstrings.py @@ -177,7 +177,9 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: continue # Skip `object`, `Callable`, `tuple`, etc. docstring = parse_docstring_from_object(cls_search).get(field_name, None) if docstring is not None: - return _strings.remove_single_line_breaks(docstring) + return _strings.dedent( + _strings.remove_single_line_breaks(docstring) + ).strip() # If docstring_parser failed, let's try looking for comments. tokenization = get_class_tokenization_with_field(cls, field_name) From 3f51b015cff8100fa1031f8fecd1aa96924f7b80 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 1 Jul 2024 12:38:16 +0900 Subject: [PATCH 415/491] Update README.md --- README.md | 72 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 5e1d8ec55..c6948d13e 100644 --- a/README.md +++ b/README.md @@ -38,30 +38,54 @@
-tyro is a tool for generating command-line -interfaces and configuration objects in Python. - -Our core API, `tyro.cli()`, - -- **Generates CLI interfaces** from Python type signatures. -- **Populates helptext automatically** from defaults, annotations, and - docstrings. -- **Understands nesting** of `dataclasses`, `pydantic`, and `attrs` structures. -- **Prioritizes static analysis** for type checking and autocompletion with - tools like - [Pylance](https://marketplace.visualstudio.com/items?itemName=ms-python.vscode-pylance), - [Pyright](https://github.com/microsoft/pyright), and - [mypy](https://github.com/python/mypy). - -For advanced users, it also supports: - -- **Subcommands**, as well as choosing between and overriding values in - configuration objects. -- **Shell completion** for `bash`, `zsh`, and `tcsh`. -- **Fine-grained configuration** via [PEP - 593](https://peps.python.org/pep-0593/) annotations (`tyro.conf.*`). - -For examples and the API reference, see our +tyro.cli() generates command-line interfaces via +Python type introspection. We can define configurable scripts using functions: + +```python +"""A command-line interface defined using a function signature. + +Usage: python script_name.py --foo INT [--bar STR] +""" + +import tyro + +def main( + foo: int, + bar: str = "default", +) -> None: + ... # Main body of a script. + +if __name__ == "__main__": + # Generate a CLI and call `main` with its two arguments: `foo` and `bar`. + tyro.cli(main) +``` + +Or instantiate configuration objects defined using tools like `dataclasses`, `pydantic`, and `attrs`: + +```python +"""A command-line interface defined using a class signature. + +Usage: python script_name.py --foo INT [--bar STR] +""" + +from dataclasses import dataclass +import tyro + +@dataclass +class Config: + foo: int + bar: str = "default" + +if __name__ == "__main__": + # Generate a CLI and instantiate `Config` with its two arguments: `foo` and `bar`. + config = tyro.cli(Config) + + # Rest of script. + assert isinstance(config, Config) # Should pass. +``` + +Other features include helptext generation, nested structures, shell +completion, and subcommands. For examples and the API reference, see our [documentation](https://brentyi.github.io/tyro). ### In the wild From 958bf0bd5697dfce1b676742fcf06e91c0c22b97 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 28 Jul 2024 19:46:09 -0700 Subject: [PATCH 416/491] Improve docs + docstrings (related: #148, #149, #150) - Added a new example for dataclass defaults. - Added a warning for `__post_init__`. - Added usage details for where `tyro.conf.arg()` is permitted. - Generally improved formatting. --- .../examples/01_basics/01_functions.rst | 6 +- .../examples/01_basics/02_dataclasses.rst | 5 +- .../01_basics/03_dataclasses_defaults.rst | 79 +++++++++++++++++++ ...{03_collections.rst => 04_collections.rst} | 19 +++-- docs/source/examples/01_basics/05_flags.rst | 2 - .../source/examples/01_basics/06_literals.rst | 18 +---- docs/source/examples/01_basics/07_unions.rst | 5 +- .../01_basics/{04_enums.rst => 08_enums.rst} | 17 ++-- .../source/examples/02_nesting/01_nesting.rst | 2 - .../examples/02_nesting/02_subcommands.rst | 2 - .../02_nesting/03_multiple_subcommands.rst | 2 - .../02_nesting/04_nesting_in_containers.rst | 2 - .../02_nesting/05_subcommands_func.rst | 39 +-------- .../03_config_systems/01_base_configs.rst | 4 +- .../03_config_systems/02_overriding_yaml.rst | 7 +- .../04_additional/01_positional_args.rst | 2 - .../04_additional/02_dictionaries.rst | 10 +-- .../examples/04_additional/03_tuples.rst | 5 +- .../examples/04_additional/04_classes.rst | 6 +- .../examples/04_additional/05_generics.rst | 2 - .../04_additional/06_generics_py312.rst | 5 +- .../source/examples/04_additional/07_conf.rst | 2 - .../examples/04_additional/08_pydantic.rst | 4 +- .../examples/04_additional/09_attrs.rst | 4 +- .../source/examples/04_additional/10_flax.rst | 6 +- .../04_additional/11_custom_constructors.rst | 29 ++++++- .../examples/04_additional/12_aliases.rst | 2 - .../examples/04_additional/12_counters.rst | 2 - .../04_additional/13_type_statement.rst | 4 +- .../14_suppress_console_outputs.rst | 17 ++-- docs/source/index.md | 2 +- docs/update_example_docs.py | 4 +- examples/01_basics/01_functions.py | 4 +- examples/01_basics/02_dataclasses.py | 3 +- examples/01_basics/03_dataclasses_defaults.py | 53 +++++++++++++ .../{03_collections.py => 04_collections.py} | 5 +- examples/01_basics/06_literals.py | 16 +--- examples/01_basics/07_unions.py | 3 +- .../01_basics/{04_enums.py => 08_enums.py} | 3 +- examples/02_nesting/05_subcommands_func.py | 36 +-------- .../03_config_systems/02_overriding_yaml.py | 5 +- examples/04_additional/02_dictionaries.py | 6 +- examples/04_additional/03_tuples.py | 3 +- examples/04_additional/04_classes.py | 4 +- examples/04_additional/06_generics_py312.py | 5 +- examples/04_additional/08_pydantic.py | 4 +- examples/04_additional/09_attrs.py | 4 +- examples/04_additional/10_flax.py | 4 +- .../04_additional/11_custom_constructors.py | 32 +++++++- examples/04_additional/13_type_statement.py | 2 +- .../14_suppress_console_outputs.py | 20 ++--- src/tyro/conf/_confstruct.py | 8 ++ 52 files changed, 305 insertions(+), 230 deletions(-) create mode 100644 docs/source/examples/01_basics/03_dataclasses_defaults.rst rename docs/source/examples/01_basics/{03_collections.rst => 04_collections.rst} (69%) rename docs/source/examples/01_basics/{04_enums.rst => 08_enums.rst} (68%) create mode 100644 examples/01_basics/03_dataclasses_defaults.py rename examples/01_basics/{03_collections.py => 04_collections.py} (84%) rename examples/01_basics/{04_enums.py => 08_enums.py} (88%) diff --git a/docs/source/examples/01_basics/01_functions.rst b/docs/source/examples/01_basics/01_functions.rst index edf75b904..e5729e93b 100644 --- a/docs/source/examples/01_basics/01_functions.rst +++ b/docs/source/examples/01_basics/01_functions.rst @@ -4,10 +4,8 @@ Functions ========================================== - -In the simplest case, ``tyro.cli()`` can be used to run a function with arguments -populated from the CLI. - +In the simplest case, :func:`tyro.cli()` can be used to run a function with +arguments populated from the CLI. .. code-block:: python diff --git a/docs/source/examples/01_basics/02_dataclasses.rst b/docs/source/examples/01_basics/02_dataclasses.rst index 79b64aa35..01f7acb78 100644 --- a/docs/source/examples/01_basics/02_dataclasses.rst +++ b/docs/source/examples/01_basics/02_dataclasses.rst @@ -4,10 +4,7 @@ Dataclasses ========================================== - -Common pattern: use ``tyro.cli()`` to instantiate a dataclass. The outputted instance -can be used as a typed alternative for an argparse namespace. - +Common pattern: use :func:`tyro.cli()` to instantiate a dataclass. .. code-block:: python diff --git a/docs/source/examples/01_basics/03_dataclasses_defaults.rst b/docs/source/examples/01_basics/03_dataclasses_defaults.rst new file mode 100644 index 000000000..51dba3a56 --- /dev/null +++ b/docs/source/examples/01_basics/03_dataclasses_defaults.rst @@ -0,0 +1,79 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Dataclasses + Defaults +========================================== + +The :code:`default=` argument can be used to override default values in dataclass +types. + + +.. warning:: + We advise against mutation of configuration objects from a dataclass's + :code:`__post_init__` method [#f1]_. In the example below, + :code:`__post_init__` would be called for both the :code:`Args()` object + provided as a default value and as an output from :func:`tyro.cli()`. + Instead, we show below one example of how derived fields can be defined + immutably. + + .. [#f1] Official Python docs for ``__post_init__`` can be found `here `_. + + +.. code-block:: python + :linenos: + + + import dataclasses + + import tyro + + + @dataclasses.dataclass + class Args: + """Description. + This should show up in the helptext!""" + + field1: str + """A string field.""" + + field2: int = 3 + """A numeric field, with a default value.""" + + @property + def derived_field(self) -> str: + return ", ".join([self.field1] * self.field2) + + + if __name__ == "__main__": + args = tyro.cli( + Args, + default=Args( + field1="default string", + field2=tyro.MISSING, + ), + ) + print(args.derived_field) + +------------ + +.. raw:: html + + python 01_basics/03_dataclasses_defaults.py --help + +.. program-output:: python ../../examples/01_basics/03_dataclasses_defaults.py --help + +------------ + +.. raw:: html + + python 01_basics/03_dataclasses_defaults.py --field2 3 + +.. program-output:: python ../../examples/01_basics/03_dataclasses_defaults.py --field2 3 + +------------ + +.. raw:: html + + python 01_basics/03_dataclasses_defaults.py --field1 hello --field2 5 + +.. program-output:: python ../../examples/01_basics/03_dataclasses_defaults.py --field1 hello --field2 5 diff --git a/docs/source/examples/01_basics/03_collections.rst b/docs/source/examples/01_basics/04_collections.rst similarity index 69% rename from docs/source/examples/01_basics/03_collections.rst rename to docs/source/examples/01_basics/04_collections.rst index f2ea82004..bd9cf55cd 100644 --- a/docs/source/examples/01_basics/03_collections.rst +++ b/docs/source/examples/01_basics/04_collections.rst @@ -4,11 +4,10 @@ Multi-value Arguments ========================================== - Arguments of both fixed and variable lengths can be annotated with standard -Python collection types. For Python 3.7 and 3.8, we can use either -``from __future__ import annotations`` or\ ``typing.List[T]``\ , ``typing.Tuple[T1, T2]``\ , etc. - +Python collection types. For Python 3.7 and 3.8, we can use either :code:`from +__future__ import annotations` to support :code:`list[T]` and :code:`tuple[T]`, +or the older API :code:`typing.List[T]` and :code:`typing.Tuple[T1, T2]`. .. code-block:: python @@ -41,22 +40,22 @@ Python collection types. For Python 3.7 and 3.8, we can use either .. raw:: html - python 01_basics/03_collections.py --help + python 01_basics/04_collections.py --help -.. program-output:: python ../../examples/01_basics/03_collections.py --help +.. program-output:: python ../../examples/01_basics/04_collections.py --help ------------ .. raw:: html - python 01_basics/03_collections.py --dataset-sources ./data --image-dimensions 16 16 + python 01_basics/04_collections.py --dataset-sources ./data --image-dimensions 16 16 -.. program-output:: python ../../examples/01_basics/03_collections.py --dataset-sources ./data --image-dimensions 16 16 +.. program-output:: python ../../examples/01_basics/04_collections.py --dataset-sources ./data --image-dimensions 16 16 ------------ .. raw:: html - python 01_basics/03_collections.py --dataset-sources ./data + python 01_basics/04_collections.py --dataset-sources ./data -.. program-output:: python ../../examples/01_basics/03_collections.py --dataset-sources ./data +.. program-output:: python ../../examples/01_basics/04_collections.py --dataset-sources ./data diff --git a/docs/source/examples/01_basics/05_flags.rst b/docs/source/examples/01_basics/05_flags.rst index 47ada767d..433e6af72 100644 --- a/docs/source/examples/01_basics/05_flags.rst +++ b/docs/source/examples/01_basics/05_flags.rst @@ -4,14 +4,12 @@ Booleans and Flags ========================================== - Booleans can either be expected to be explicitly passed in, or, if given a default value, automatically converted to flags. To turn off conversion, see :class:`tyro.conf.FlagConversionOff`. - .. code-block:: python :linenos: diff --git a/docs/source/examples/01_basics/06_literals.rst b/docs/source/examples/01_basics/06_literals.rst index eaa8beedf..746e7ec61 100644 --- a/docs/source/examples/01_basics/06_literals.rst +++ b/docs/source/examples/01_basics/06_literals.rst @@ -4,9 +4,7 @@ Choices ========================================== - -``typing.Literal[]`` can be used to restrict inputs to a fixed set of literal choices. - +:code:`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices. .. code-block:: python @@ -14,29 +12,19 @@ Choices import dataclasses - import enum from typing import Literal import tyro - class Color(enum.Enum): - RED = enum.auto() - GREEN = enum.auto() - BLUE = enum.auto() - - @dataclasses.dataclass(frozen=True) class Args: # We can use Literal[] to restrict the set of allowable inputs, for example, over # a set of strings. strings: Literal["red", "green"] = "red" - # Enums also work. - enums: Literal[Color.RED, Color.GREEN] = Color.RED - - # Or mix them with other types! - mixed: Literal[Color.RED, Color.GREEN, "blue"] = "blue" + # Integers also work. (as well as booleans, enums, etc) + numbers: Literal[0, 1, 2] = 0 if __name__ == "__main__": diff --git a/docs/source/examples/01_basics/07_unions.rst b/docs/source/examples/01_basics/07_unions.rst index c188f15a8..8ce43f511 100644 --- a/docs/source/examples/01_basics/07_unions.rst +++ b/docs/source/examples/01_basics/07_unions.rst @@ -4,9 +4,8 @@ Unions ========================================== - -``X | Y`` or ``typing.Union[X, Y]`` can be used to expand inputs to multiple types. - +:code:`X | Y` or :code:`typing.Union[X, Y]` can be used to expand inputs to +multiple types. .. code-block:: python diff --git a/docs/source/examples/01_basics/04_enums.rst b/docs/source/examples/01_basics/08_enums.rst similarity index 68% rename from docs/source/examples/01_basics/04_enums.rst rename to docs/source/examples/01_basics/08_enums.rst index 7b8296b62..08f142963 100644 --- a/docs/source/examples/01_basics/04_enums.rst +++ b/docs/source/examples/01_basics/08_enums.rst @@ -4,9 +4,8 @@ Enums ========================================== - -We can generate argument parsers from more advanced type annotations, like enums. - +In addition to literals, enums can also be used to provide a fixed set of +choices. .. code-block:: python @@ -42,22 +41,22 @@ We can generate argument parsers from more advanced type annotations, like enums .. raw:: html - python 01_basics/04_enums.py --help + python 01_basics/08_enums.py --help -.. program-output:: python ../../examples/01_basics/04_enums.py --help +.. program-output:: python ../../examples/01_basics/08_enums.py --help ------------ .. raw:: html - python 01_basics/04_enums.py --optimizer-type SGD + python 01_basics/08_enums.py --optimizer-type SGD -.. program-output:: python ../../examples/01_basics/04_enums.py --optimizer-type SGD +.. program-output:: python ../../examples/01_basics/08_enums.py --optimizer-type SGD ------------ .. raw:: html - python 01_basics/04_enums.py --optimizer-type ADAM --learning-rate 3e-4 + python 01_basics/08_enums.py --optimizer-type ADAM --learning-rate 3e-4 -.. program-output:: python ../../examples/01_basics/04_enums.py --optimizer-type ADAM --learning-rate 3e-4 +.. program-output:: python ../../examples/01_basics/08_enums.py --optimizer-type ADAM --learning-rate 3e-4 diff --git a/docs/source/examples/02_nesting/01_nesting.rst b/docs/source/examples/02_nesting/01_nesting.rst index 363e4d9e2..d56858eeb 100644 --- a/docs/source/examples/02_nesting/01_nesting.rst +++ b/docs/source/examples/02_nesting/01_nesting.rst @@ -4,12 +4,10 @@ Hierarchical Configs ========================================== - Structures (typically dataclasses) can be nested to build hierarchical configuration objects. This helps with modularity and grouping in larger projects. - .. code-block:: python :linenos: diff --git a/docs/source/examples/02_nesting/02_subcommands.rst b/docs/source/examples/02_nesting/02_subcommands.rst index 09a03f0ed..1992a3bbc 100644 --- a/docs/source/examples/02_nesting/02_subcommands.rst +++ b/docs/source/examples/02_nesting/02_subcommands.rst @@ -4,14 +4,12 @@ Subcommands ========================================== - Unions over nested types (classes or dataclasses) are populated using subcommands. For configuring subcommands beyond what can be expressed with type annotations, see :func:`tyro.conf.subcommand()`. - .. code-block:: python :linenos: diff --git a/docs/source/examples/02_nesting/03_multiple_subcommands.rst b/docs/source/examples/02_nesting/03_multiple_subcommands.rst index 9a0a7406a..bb62f7a0b 100644 --- a/docs/source/examples/02_nesting/03_multiple_subcommands.rst +++ b/docs/source/examples/02_nesting/03_multiple_subcommands.rst @@ -4,11 +4,9 @@ Sequenced Subcommands ========================================== - Multiple unions over nested types are populated using a series of subcommands. - .. code-block:: python :linenos: diff --git a/docs/source/examples/02_nesting/04_nesting_in_containers.rst b/docs/source/examples/02_nesting/04_nesting_in_containers.rst index bb9690ba1..d9d5a812f 100644 --- a/docs/source/examples/02_nesting/04_nesting_in_containers.rst +++ b/docs/source/examples/02_nesting/04_nesting_in_containers.rst @@ -4,14 +4,12 @@ Nesting in Containers ========================================== - Structures can be nested inside of standard containers. Note that lengths must be inferable, either via a fixed-length tuple annotation or by parsing default values. - .. code-block:: python :linenos: diff --git a/docs/source/examples/02_nesting/05_subcommands_func.rst b/docs/source/examples/02_nesting/05_subcommands_func.rst index 3eddf238e..141c9a53d 100644 --- a/docs/source/examples/02_nesting/05_subcommands_func.rst +++ b/docs/source/examples/02_nesting/05_subcommands_func.rst @@ -4,41 +4,10 @@ Subcommands from Functions ========================================== - -:func:`tyro.extras.subcommand_cli_from_dict()` provides a shorthand that generates a -subcommand CLI from a dictionary. - -For an input like: - -.. code-block:: python - - tyro.extras.subcommand_cli_from_dict( - { - "checkout": checkout, - "commit": commit, - } - ) - -This is internally accomplished by generating and calling: - -.. code-block:: python - - from typing import Annotated, Any, Union - import tyro - - tyro.cli( - Union[ - Annotated[ - Any, - tyro.conf.subcommand(name="checkout", constructor=checkout), - ], - Annotated[ - Any, - tyro.conf.subcommand(name="commit", constructor=commit), - ], - ] - ) - +We provide a shorthand for generating a subcommand CLI from a dictionary. This +is a thin wrapper around :func:`tyro.cli()`'s more verbose, type-based API. If +more generality is needed, the internal working are explained in the docs for +:func:`tyro.extras.subcommand_cli_from_dict()`. .. code-block:: python diff --git a/docs/source/examples/03_config_systems/01_base_configs.rst b/docs/source/examples/03_config_systems/01_base_configs.rst index 3cb08479a..d2250b353 100644 --- a/docs/source/examples/03_config_systems/01_base_configs.rst +++ b/docs/source/examples/03_config_systems/01_base_configs.rst @@ -4,13 +4,11 @@ Base Configurations ========================================== - -We can integrate ``tyro.cli()`` into common configuration patterns: here, we select +We can integrate `tyro.cli()` into common configuration patterns: here, we select one of multiple possible base configurations, create a subcommand for each one, and then use the CLI to either override (existing) or fill in (missing) values. - .. code-block:: python :linenos: diff --git a/docs/source/examples/03_config_systems/02_overriding_yaml.rst b/docs/source/examples/03_config_systems/02_overriding_yaml.rst index 1ce864bd2..8262bed13 100644 --- a/docs/source/examples/03_config_systems/02_overriding_yaml.rst +++ b/docs/source/examples/03_config_systems/02_overriding_yaml.rst @@ -4,10 +4,9 @@ Overriding YAML Configs ========================================== - -If you have a library of existing YAML files that you want to use, ``tyro`` can be used to -override values in them. - +If you have a library of existing YAML files that you want to use, `tyro` can +be used to override values in them. This example uses a dictionary; we +generally recommend dataset configs for new projects. .. code-block:: python diff --git a/docs/source/examples/04_additional/01_positional_args.rst b/docs/source/examples/04_additional/01_positional_args.rst index 5c0182a08..b547f0218 100644 --- a/docs/source/examples/04_additional/01_positional_args.rst +++ b/docs/source/examples/04_additional/01_positional_args.rst @@ -4,13 +4,11 @@ Positional Arguments ========================================== - Positional-only arguments in functions are converted to positional CLI arguments. For more general positional arguments, see :class:`tyro.conf.Positional`. - .. code-block:: python :linenos: diff --git a/docs/source/examples/04_additional/02_dictionaries.rst b/docs/source/examples/04_additional/02_dictionaries.rst index 37162ccc9..7d28077e6 100644 --- a/docs/source/examples/04_additional/02_dictionaries.rst +++ b/docs/source/examples/04_additional/02_dictionaries.rst @@ -4,13 +4,11 @@ Dictionaries and TypedDict ========================================== +Dictionary inputs can be specified using either a standard `Dict[K, V]` +annotation, or a :code:`TypedDict` subclass. -Dictionary inputs can be specified using either a standard ``Dict[K, V]`` -annotation, or a ``TypedDict`` subclass. - -For configuring ``TypedDict``\ , we also support ``total={True/False}``\ , -``typing.Required``\ , and ``typing.NotRequired``. - +For configuring :code:`TypedDict`, we also support :code:`total={True/False}`, +:code:`typing.Required`, and :code:`typing.NotRequired`. See the `Python docs `_ for all :code:`TypedDict` features. .. code-block:: python diff --git a/docs/source/examples/04_additional/03_tuples.rst b/docs/source/examples/04_additional/03_tuples.rst index c355783a8..cb979eda9 100644 --- a/docs/source/examples/04_additional/03_tuples.rst +++ b/docs/source/examples/04_additional/03_tuples.rst @@ -4,9 +4,8 @@ Tuples ========================================== - -Example using ``tyro.cli()`` to instantiate tuple types. - +Example using :func:`tyro.cli()` to instantiate tuple types. :code:`tuple`, +:code:`typing.Tuple`, and :code:`NamedTuple` are all supported. .. code-block:: python diff --git a/docs/source/examples/04_additional/04_classes.rst b/docs/source/examples/04_additional/04_classes.rst index 526ae71b1..bf6f64526 100644 --- a/docs/source/examples/04_additional/04_classes.rst +++ b/docs/source/examples/04_additional/04_classes.rst @@ -4,10 +4,8 @@ Instantiating Classes ========================================== - -In addition to functions and dataclasses, we can also generate CLIs from (the -constructors of) standard Python classes. - +In addition to functions and dataclasses, we can also generate CLIs from the +constructors of standard Python classes. .. code-block:: python diff --git a/docs/source/examples/04_additional/05_generics.rst b/docs/source/examples/04_additional/05_generics.rst index 175dddef6..6578dcd93 100644 --- a/docs/source/examples/04_additional/05_generics.rst +++ b/docs/source/examples/04_additional/05_generics.rst @@ -4,11 +4,9 @@ Generic Types ========================================== - Example of parsing for generic dataclasses. - .. code-block:: python :linenos: diff --git a/docs/source/examples/04_additional/06_generics_py312.rst b/docs/source/examples/04_additional/06_generics_py312.rst index dd214d8e0..44dbb9d8e 100644 --- a/docs/source/examples/04_additional/06_generics_py312.rst +++ b/docs/source/examples/04_additional/06_generics_py312.rst @@ -4,10 +4,11 @@ Generic Types (Python 3.12+ syntax) ========================================== - Example of parsing for generic dataclasses using syntax introduced in Python -3.12. Note: this is not compatible with ``from __future__ import annotations``. +3.12 (`PEP 695 `_). +.. warning:: + If used in conjunction with :code:`from __future__ import annotations`, the updated type parameter syntax requires Python 3.12.4 or newer. For technical details, see `this CPython PR `_. .. code-block:: python diff --git a/docs/source/examples/04_additional/07_conf.rst b/docs/source/examples/04_additional/07_conf.rst index babc7a146..2faacdf48 100644 --- a/docs/source/examples/04_additional/07_conf.rst +++ b/docs/source/examples/04_additional/07_conf.rst @@ -4,14 +4,12 @@ Configuration via typing.Annotated[] ========================================== - The :mod:`tyro.conf` module contains utilities that can be used to configure command-line interfaces beyond what is expressible via static type annotations. Features here are supported, but generally unnecessary and should be used sparingly. - .. code-block:: python :linenos: diff --git a/docs/source/examples/04_additional/08_pydantic.rst b/docs/source/examples/04_additional/08_pydantic.rst index 3c74d8e7b..9f529d173 100644 --- a/docs/source/examples/04_additional/08_pydantic.rst +++ b/docs/source/examples/04_additional/08_pydantic.rst @@ -4,12 +4,10 @@ Pydantic Integration ========================================== - -In addition to standard dataclasses, ``tyro`` also supports +In addition to standard dataclasses, :func:`tyro.cli()` also supports `Pydantic `_ models. - .. code-block:: python :linenos: diff --git a/docs/source/examples/04_additional/09_attrs.rst b/docs/source/examples/04_additional/09_attrs.rst index ab4b2de9f..e52a189f5 100644 --- a/docs/source/examples/04_additional/09_attrs.rst +++ b/docs/source/examples/04_additional/09_attrs.rst @@ -4,12 +4,10 @@ Attrs Integration ========================================== - -In addition to standard dataclasses, ``tyro`` also supports +In addition to standard dataclasses, :func:`tyro.cli()` also supports `attrs `_ classes. - .. code-block:: python :linenos: diff --git a/docs/source/examples/04_additional/10_flax.rst b/docs/source/examples/04_additional/10_flax.rst index 8269f51c9..aec226d00 100644 --- a/docs/source/examples/04_additional/10_flax.rst +++ b/docs/source/examples/04_additional/10_flax.rst @@ -4,10 +4,8 @@ JAX/Flax Integration ========================================== - -If you use `flax.linen `_\ , modules can be instantiated -directly from ``tyro.cli``. - +If you use `flax.linen `_, modules can be instantiated +directly from :func:`tyro.cli()`. .. code-block:: python diff --git a/docs/source/examples/04_additional/11_custom_constructors.rst b/docs/source/examples/04_additional/11_custom_constructors.rst index 9c650ea83..3d91faa9a 100644 --- a/docs/source/examples/04_additional/11_custom_constructors.rst +++ b/docs/source/examples/04_additional/11_custom_constructors.rst @@ -4,10 +4,35 @@ Custom Constructors ========================================== +For additional flexibility, :func:`tyro.conf.arg()` accepts a +:code:`constructor` argument, which makes it easier to load complex objects. -For additional flexibility, :func:`tyro.conf.arg()` accepts a ``constructor`` argument, -which makes it easier to load complex objects. +.. warning:: + Custom constructors are permitted wherever :func:`tyro.conf.arg()` is. This + annotation corresponds to a single argument, so we expect that it is placed + placed at the root of the field corresponding to that argument. For + example: + .. code-block:: python + + # Great! + x: Annotated[int, tyro.conf.arg(custom_constructor=...)] + + However, nesting :func:`tyro.conf.arg()` within other types is generally + not supported. For example: + + .. code-block:: python + + # Not supported. + x: tuple[Annotated[int, tyro.conf.arg(custom_constructor=...)], ...] + + This example can be rewritten with a custom constructor that directly + returns a tuple of integers: + + .. code-block:: python + + # Workaround for above. + x: Annotated[tuple[int, ...], tyro.conf.arg(custom_constructor=...)] .. code-block:: python diff --git a/docs/source/examples/04_additional/12_aliases.rst b/docs/source/examples/04_additional/12_aliases.rst index f6368e5c3..1a8c5aa32 100644 --- a/docs/source/examples/04_additional/12_aliases.rst +++ b/docs/source/examples/04_additional/12_aliases.rst @@ -4,11 +4,9 @@ Argument Aliases ========================================== - :func:`tyro.conf.arg()` can be used to attach aliases to arguments. - .. code-block:: python :linenos: diff --git a/docs/source/examples/04_additional/12_counters.rst b/docs/source/examples/04_additional/12_counters.rst index d0bedf571..a05decc34 100644 --- a/docs/source/examples/04_additional/12_counters.rst +++ b/docs/source/examples/04_additional/12_counters.rst @@ -4,11 +4,9 @@ Counters ========================================== - Repeatable 'counter' arguments can be specified via :data:`tyro.conf.UseCounterAction`. - .. code-block:: python :linenos: diff --git a/docs/source/examples/04_additional/13_type_statement.rst b/docs/source/examples/04_additional/13_type_statement.rst index 7406e0825..e568a1102 100644 --- a/docs/source/examples/04_additional/13_type_statement.rst +++ b/docs/source/examples/04_additional/13_type_statement.rst @@ -4,9 +4,7 @@ Type Aliases (Python 3.12+) ========================================== - -In Python 3.12, the ``type`` statement is introduced to create type aliases. - +In Python 3.12, the :code:`type` statement is introduced to create type aliases. .. code-block:: python diff --git a/docs/source/examples/04_additional/14_suppress_console_outputs.rst b/docs/source/examples/04_additional/14_suppress_console_outputs.rst index 0ec626b5d..e6056ea69 100644 --- a/docs/source/examples/04_additional/14_suppress_console_outputs.rst +++ b/docs/source/examples/04_additional/14_suppress_console_outputs.rst @@ -4,24 +4,23 @@ Cleaner Console Outputs for Scripts with Multiple Workers ========================================== - -The ``console_outputs=`` argument can be set to ``False`` to suppress helptext and +The :code:`console_outputs=` argument can be set to :code:`False` to suppress helptext and error message printing. This is useful in PyTorch for distributed training scripts, where you only want to print the helptext from the main process: -.. code-block:: python - # Hugging Face Accelerate. - args = tyro.cli(Args, console_outputs=accelerator.is_main_process) +.. code-block:: python - # PyTorch DDP. - args = tyro.cli(Args, console_outputs=(rank == 0)) + # HuggingFace Accelerate. + args = tyro.cli(Args, console_outputs=accelerator.is_main_process) - # PyTorch Lightning. - args = tyro.cli(Args, console_outputs=trainer.is_global_zero) + # PyTorch DDP. + args = tyro.cli(Args, console_outputs=(rank == 0)) + # PyTorch Lightning. + args = tyro.cli(Args, console_outputs=trainer.is_global_zero) .. code-block:: python diff --git a/docs/source/index.md b/docs/source/index.md index 8610b441b..05d7e25db 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -5,7 +5,7 @@ :code:`tyro` is a tool for generating command-line interfaces and configuration objects in Python. -Our core API, `tyro.cli()`, +Our core API, :func:`tyro.cli()`, - **Generates CLI interfaces** from a comprehensive set of Python type constructs. diff --git a/docs/update_example_docs.py b/docs/update_example_docs.py index 43ae770e7..475b9dbe2 100644 --- a/docs/update_example_docs.py +++ b/docs/update_example_docs.py @@ -8,8 +8,6 @@ import shutil from typing import Iterable -import m2r2 - import tyro @@ -120,7 +118,7 @@ def main( f"{ex.title}", "==========================================", "", - m2r2.convert(ex.description), + ex.description, "", "", ".. code-block:: python", diff --git a/examples/01_basics/01_functions.py b/examples/01_basics/01_functions.py index 52c7640e8..d9fe17413 100644 --- a/examples/01_basics/01_functions.py +++ b/examples/01_basics/01_functions.py @@ -1,7 +1,7 @@ """Functions -In the simplest case, `tyro.cli()` can be used to run a function with arguments -populated from the CLI. +In the simplest case, :func:`tyro.cli()` can be used to run a function with +arguments populated from the CLI. Usage: `python ./01_functions.py --help` diff --git a/examples/01_basics/02_dataclasses.py b/examples/01_basics/02_dataclasses.py index 30424fead..09e94bbfc 100644 --- a/examples/01_basics/02_dataclasses.py +++ b/examples/01_basics/02_dataclasses.py @@ -1,7 +1,6 @@ """Dataclasses -Common pattern: use `tyro.cli()` to instantiate a dataclass. The outputted instance -can be used as a typed alternative for an argparse namespace. +Common pattern: use :func:`tyro.cli()` to instantiate a dataclass. Usage: `python ./02_dataclasses.py --help` diff --git a/examples/01_basics/03_dataclasses_defaults.py b/examples/01_basics/03_dataclasses_defaults.py new file mode 100644 index 000000000..8b62676af --- /dev/null +++ b/examples/01_basics/03_dataclasses_defaults.py @@ -0,0 +1,53 @@ +"""Dataclasses + Defaults + +The :code:`default=` argument can be used to override default values in dataclass +types. + + +.. warning:: + We advise against mutation of configuration objects from a dataclass's + :code:`__post_init__` method [#f1]_. In the example below, + :code:`__post_init__` would be called for both the :code:`Args()` object + provided as a default value and as an output from :func:`tyro.cli()`. + Instead, we show below one example of how derived fields can be defined + immutably. + + .. [#f1] Official Python docs for ``__post_init__`` can be found `here `_. + + +Usage: +`python ./03_dataclasses_defaults.py --help` +`python ./03_dataclasses_defaults.py --field2 3` +`python ./03_dataclasses_defaults.py --field1 hello --field2 5` +""" + +import dataclasses + +import tyro + + +@dataclasses.dataclass +class Args: + """Description. + This should show up in the helptext!""" + + field1: str + """A string field.""" + + field2: int = 3 + """A numeric field, with a default value.""" + + @property + def derived_field(self) -> str: + return ", ".join([self.field1] * self.field2) + + +if __name__ == "__main__": + args = tyro.cli( + Args, + default=Args( + field1="default string", + field2=tyro.MISSING, + ), + ) + print(args.derived_field) diff --git a/examples/01_basics/03_collections.py b/examples/01_basics/04_collections.py similarity index 84% rename from examples/01_basics/03_collections.py rename to examples/01_basics/04_collections.py index c82dd7dd7..a3f3b836e 100644 --- a/examples/01_basics/03_collections.py +++ b/examples/01_basics/04_collections.py @@ -1,8 +1,9 @@ """Multi-value Arguments Arguments of both fixed and variable lengths can be annotated with standard -Python collection types. For Python 3.7 and 3.8, we can use either -`from __future__ import annotations` or`typing.List[T]`, `typing.Tuple[T1, T2]`, etc. +Python collection types. For Python 3.7 and 3.8, we can use either :code:`from +__future__ import annotations` to support :code:`list[T]` and :code:`tuple[T]`, +or the older API :code:`typing.List[T]` and :code:`typing.Tuple[T1, T2]`. Usage: `python ./03_collections.py --help` diff --git a/examples/01_basics/06_literals.py b/examples/01_basics/06_literals.py index 22e38e987..5f3017d85 100644 --- a/examples/01_basics/06_literals.py +++ b/examples/01_basics/06_literals.py @@ -1,35 +1,25 @@ """Choices -`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices. +:code:`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices. Usage: `python ./06_literals.py --help` """ import dataclasses -import enum from typing import Literal import tyro -class Color(enum.Enum): - RED = enum.auto() - GREEN = enum.auto() - BLUE = enum.auto() - - @dataclasses.dataclass(frozen=True) class Args: # We can use Literal[] to restrict the set of allowable inputs, for example, over # a set of strings. strings: Literal["red", "green"] = "red" - # Enums also work. - enums: Literal[Color.RED, Color.GREEN] = Color.RED - - # Or mix them with other types! - mixed: Literal[Color.RED, Color.GREEN, "blue"] = "blue" + # Integers also work. (as well as booleans, enums, etc) + numbers: Literal[0, 1, 2] = 0 if __name__ == "__main__": diff --git a/examples/01_basics/07_unions.py b/examples/01_basics/07_unions.py index ca024cbbc..f99ffb04d 100644 --- a/examples/01_basics/07_unions.py +++ b/examples/01_basics/07_unions.py @@ -1,6 +1,7 @@ """Unions -`X | Y` or `typing.Union[X, Y]` can be used to expand inputs to multiple types. +:code:`X | Y` or :code:`typing.Union[X, Y]` can be used to expand inputs to +multiple types. Usage: `python ./07_unions.py --help` diff --git a/examples/01_basics/04_enums.py b/examples/01_basics/08_enums.py similarity index 88% rename from examples/01_basics/04_enums.py rename to examples/01_basics/08_enums.py index d8679d1d7..0fa47eed4 100644 --- a/examples/01_basics/04_enums.py +++ b/examples/01_basics/08_enums.py @@ -1,6 +1,7 @@ """Enums -We can generate argument parsers from more advanced type annotations, like enums. +In addition to literals, enums can also be used to provide a fixed set of +choices. Usage: `python ./04_enums.py --help` diff --git a/examples/02_nesting/05_subcommands_func.py b/examples/02_nesting/05_subcommands_func.py index 0cf9309b7..ca2f202f9 100644 --- a/examples/02_nesting/05_subcommands_func.py +++ b/examples/02_nesting/05_subcommands_func.py @@ -1,38 +1,10 @@ """Subcommands from Functions -:func:`tyro.extras.subcommand_cli_from_dict()` provides a shorthand that generates a -subcommand CLI from a dictionary. +We provide a shorthand for generating a subcommand CLI from a dictionary. This +is a thin wrapper around :func:`tyro.cli()`'s more verbose, type-based API. If +more generality is needed, the internal working are explained in the docs for +:func:`tyro.extras.subcommand_cli_from_dict()`. -For an input like: - -```python -tyro.extras.subcommand_cli_from_dict( - { - "checkout": checkout, - "commit": commit, - } -) -``` - -This is internally accomplished by generating and calling: - -```python -from typing import Annotated, Any, Union -import tyro - -tyro.cli( - Union[ - Annotated[ - Any, - tyro.conf.subcommand(name="checkout", constructor=checkout), - ], - Annotated[ - Any, - tyro.conf.subcommand(name="commit", constructor=commit), - ], - ] -) -``` Usage: `python ./05_subcommands_func.py --help` diff --git a/examples/03_config_systems/02_overriding_yaml.py b/examples/03_config_systems/02_overriding_yaml.py index a1e83646c..db52832d1 100644 --- a/examples/03_config_systems/02_overriding_yaml.py +++ b/examples/03_config_systems/02_overriding_yaml.py @@ -1,7 +1,8 @@ """Overriding YAML Configs -If you have a library of existing YAML files that you want to use, `tyro` can be used to -override values in them. +If you have a library of existing YAML files that you want to use, `tyro` can +be used to override values in them. This example uses a dictionary; we +generally recommend dataset configs for new projects. Usage: `python ./02_overriding_yaml.py --help` diff --git a/examples/04_additional/02_dictionaries.py b/examples/04_additional/02_dictionaries.py index e7d449c3c..9b0bac6f0 100644 --- a/examples/04_additional/02_dictionaries.py +++ b/examples/04_additional/02_dictionaries.py @@ -1,10 +1,10 @@ """Dictionaries and TypedDict Dictionary inputs can be specified using either a standard `Dict[K, V]` -annotation, or a `TypedDict` subclass. +annotation, or a :code:`TypedDict` subclass. -For configuring `TypedDict`, we also support `total={True/False}`, -`typing.Required`, and `typing.NotRequired`. +For configuring :code:`TypedDict`, we also support :code:`total={True/False}`, +:code:`typing.Required`, and :code:`typing.NotRequired`. See the `Python docs `_ for all :code:`TypedDict` features. Usage: `python ./02_dictionaries.py --help` diff --git a/examples/04_additional/03_tuples.py b/examples/04_additional/03_tuples.py index 5c0865785..ef40280c8 100644 --- a/examples/04_additional/03_tuples.py +++ b/examples/04_additional/03_tuples.py @@ -1,6 +1,7 @@ """Tuples -Example using `tyro.cli()` to instantiate tuple types. +Example using :func:`tyro.cli()` to instantiate tuple types. :code:`tuple`, +:code:`typing.Tuple`, and :code:`NamedTuple` are all supported. Usage: `python ./03_tuples.py --help` diff --git a/examples/04_additional/04_classes.py b/examples/04_additional/04_classes.py index 9780649a7..5dbae4639 100644 --- a/examples/04_additional/04_classes.py +++ b/examples/04_additional/04_classes.py @@ -1,7 +1,7 @@ """Instantiating Classes -In addition to functions and dataclasses, we can also generate CLIs from (the -constructors of) standard Python classes. +In addition to functions and dataclasses, we can also generate CLIs from the +constructors of standard Python classes. Usage: `python ./04_classes.py --help` diff --git a/examples/04_additional/06_generics_py312.py b/examples/04_additional/06_generics_py312.py index f68624a6a..2fd4b4580 100644 --- a/examples/04_additional/06_generics_py312.py +++ b/examples/04_additional/06_generics_py312.py @@ -4,7 +4,10 @@ """Generic Types (Python 3.12+ syntax) Example of parsing for generic dataclasses using syntax introduced in Python -3.12. Note: this is not compatible with `from __future__ import annotations`. +3.12 (`PEP 695 `_). + +.. warning:: + If used in conjunction with :code:`from __future__ import annotations`, the updated type parameter syntax requires Python 3.12.4 or newer. For technical details, see `this CPython PR `_. Usage: `python ./05_generics.py --help` diff --git a/examples/04_additional/08_pydantic.py b/examples/04_additional/08_pydantic.py index 92407a7a4..e9c5ee457 100644 --- a/examples/04_additional/08_pydantic.py +++ b/examples/04_additional/08_pydantic.py @@ -1,7 +1,7 @@ """Pydantic Integration -In addition to standard dataclasses, `tyro` also supports -[Pydantic](https://github.com/pydantic/pydantic) models. +In addition to standard dataclasses, :func:`tyro.cli()` also supports +`Pydantic `_ models. Usage: `python ./08_pydantic.py --help` diff --git a/examples/04_additional/09_attrs.py b/examples/04_additional/09_attrs.py index 0adca8a48..effe864e9 100644 --- a/examples/04_additional/09_attrs.py +++ b/examples/04_additional/09_attrs.py @@ -1,7 +1,7 @@ """Attrs Integration -In addition to standard dataclasses, `tyro` also supports -[attrs](https://www.attrs.org/) classes. +In addition to standard dataclasses, :func:`tyro.cli()` also supports +`attrs `_ classes. Usage: `python ./09_attrs.py --help` diff --git a/examples/04_additional/10_flax.py b/examples/04_additional/10_flax.py index 6707964ee..ecc5c50fc 100644 --- a/examples/04_additional/10_flax.py +++ b/examples/04_additional/10_flax.py @@ -1,7 +1,7 @@ """JAX/Flax Integration -If you use [flax.linen](https://github.com/google/flax), modules can be instantiated -directly from `tyro.cli`. +If you use `flax.linen `_, modules can be instantiated +directly from :func:`tyro.cli()`. Usage: `python ./07_flax.py --help` diff --git a/examples/04_additional/11_custom_constructors.py b/examples/04_additional/11_custom_constructors.py index 993133b08..0bbcebb2d 100644 --- a/examples/04_additional/11_custom_constructors.py +++ b/examples/04_additional/11_custom_constructors.py @@ -1,7 +1,35 @@ """Custom Constructors -For additional flexibility, :func:`tyro.conf.arg()` accepts a `constructor` argument, -which makes it easier to load complex objects. +For additional flexibility, :func:`tyro.conf.arg()` accepts a +:code:`constructor` argument, which makes it easier to load complex objects. + +.. warning:: + Custom constructors are permitted wherever :func:`tyro.conf.arg()` is. This + annotation corresponds to a single argument, so we expect that it is placed + placed at the root of the field corresponding to that argument. For + example: + + .. code-block:: python + + # Great! + x: Annotated[int, tyro.conf.arg(custom_constructor=...)] + + However, nesting :func:`tyro.conf.arg()` within other types is generally + not supported. For example: + + .. code-block:: python + + # Not supported. + x: tuple[Annotated[int, tyro.conf.arg(custom_constructor=...)], ...] + + This example can be rewritten with a custom constructor that directly + returns a tuple of integers: + + .. code-block:: python + + # Workaround for above. + x: Annotated[tuple[int, ...], tyro.conf.arg(custom_constructor=...)] + Usage: `python ./10_custom_constructors.py --help` diff --git a/examples/04_additional/13_type_statement.py b/examples/04_additional/13_type_statement.py index a892bf735..023d5c6f6 100644 --- a/examples/04_additional/13_type_statement.py +++ b/examples/04_additional/13_type_statement.py @@ -3,7 +3,7 @@ # PEP 695 isn't yet supported in mypy. (April 4, 2024) """Type Aliases (Python 3.12+) -In Python 3.12, the `type` statement is introduced to create type aliases. +In Python 3.12, the :code:`type` statement is introduced to create type aliases. Usage: `python ./13_type_statement.py --help` diff --git a/examples/04_additional/14_suppress_console_outputs.py b/examples/04_additional/14_suppress_console_outputs.py index a977102cc..c8b48840a 100644 --- a/examples/04_additional/14_suppress_console_outputs.py +++ b/examples/04_additional/14_suppress_console_outputs.py @@ -1,21 +1,23 @@ """Cleaner Console Outputs for Scripts with Multiple Workers -The `console_outputs=` argument can be set to `False` to suppress helptext and +The :code:`console_outputs=` argument can be set to :code:`False` to suppress helptext and error message printing. This is useful in PyTorch for distributed training scripts, where you only want to print the helptext from the main process: -```python -# Hugging Face Accelerate. -args = tyro.cli(Args, console_outputs=accelerator.is_main_process) -# PyTorch DDP. -args = tyro.cli(Args, console_outputs=(rank == 0)) +.. code-block:: python + + # HuggingFace Accelerate. + args = tyro.cli(Args, console_outputs=accelerator.is_main_process) + + # PyTorch DDP. + args = tyro.cli(Args, console_outputs=(rank == 0)) + + # PyTorch Lightning. + args = tyro.cli(Args, console_outputs=trainer.is_global_zero) -# PyTorch Lightning. -args = tyro.cli(Args, console_outputs=trainer.is_global_zero) -``` Usage: `python ./14_suppress_console_outputs.py --help` diff --git a/src/tyro/conf/_confstruct.py b/src/tyro/conf/_confstruct.py index c7c27c295..d4a87b544 100644 --- a/src/tyro/conf/_confstruct.py +++ b/src/tyro/conf/_confstruct.py @@ -158,10 +158,18 @@ def arg( ) -> Any: """Returns a metadata object for fine-grained argument configuration with `typing.Annotated`. Should typically not be required. + + We support using `arg()` at the root of arguments. For example: ```python x: Annotated[int, tyro.conf.arg(...)] ``` + Nesting `arg()` within other types is generally not supported: + ```python + # Not supported. + x: tuple[Annotated[int, tyro.conf.arg(...)], ...] + ``` + Arguments: name: A new name for the argument in the CLI. metavar: Argument name in usage messages. The type is used by default. From dc5152334d051d153798947c2352a247a4739458 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 28 Jul 2024 20:17:26 -0700 Subject: [PATCH 417/491] Fix errors from pyright update --- src/tyro/_docstrings.py | 18 +++----- src/tyro/_fields.py | 1 + src/tyro/extras/_serialization.py | 1 + tests/test_boolean_optional.py | 8 +++- tests/test_conf.py | 28 +++++++++--- .../test_boolean_optional_generated.py | 8 +++- .../test_conf_generated.py | 43 ++++++++++++++++--- .../test_errors_generated.py | 13 +++++- .../test_nested_generated.py | 10 +++++ 9 files changed, 99 insertions(+), 31 deletions(-) diff --git a/src/tyro/_docstrings.py b/src/tyro/_docstrings.py index a268b5c10..9c49eef4d 100644 --- a/src/tyro/_docstrings.py +++ b/src/tyro/_docstrings.py @@ -331,14 +331,10 @@ def get_callable_description(f: Callable) -> str: return "" parsed_docstring = docstring_parser.parse(docstring) - return "\n".join( - list( - filter( - lambda x: x is not None, # type: ignore - [ - parsed_docstring.short_description, - parsed_docstring.long_description, - ], - ) - ) - ) + + parts: List[str] = [] + if parsed_docstring.short_description is not None: + parts.append(parsed_docstring.short_description) + if parsed_docstring.long_description is not None: + parts.append(parsed_docstring.long_description) + return "\n".join(parts) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 2d3343ee4..78b418ca7 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -587,6 +587,7 @@ def _field_list_from_dataclass( if helptext is None: helptext = _docstrings.get_field_docstring(cls, dc_field.name) + assert not isinstance(dc_field.type, str) field_list.append( FieldDefinition.make( name=dc_field.name, diff --git a/src/tyro/extras/_serialization.py b/src/tyro/extras/_serialization.py index b478f9e3c..bfb5a28d2 100644 --- a/src/tyro/extras/_serialization.py +++ b/src/tyro/extras/_serialization.py @@ -65,6 +65,7 @@ def handle_type(typ: Type[Any]) -> Set[Type[Any]]: # Handle fields. for field in _resolver.resolved_fields(cls): # type: ignore + assert not isinstance(field.type, str) contained_special_types |= handle_type(field.type) # Handle subclasses. diff --git a/tests/test_boolean_optional.py b/tests/test_boolean_optional.py index e3358bcf2..ba48bc9f3 100644 --- a/tests/test_boolean_optional.py +++ b/tests/test_boolean_optional.py @@ -28,10 +28,12 @@ class A: default=A(False), ) == A(False) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 assert tyro.cli( tyro.conf.FlagConversionOff[A], args=["--x", "True"], - default=A(False), + default=A(False), # type: ignore ) == A(True) @@ -60,8 +62,10 @@ class A: default=A(True), ) == A(True) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 assert tyro.cli( tyro.conf.FlagConversionOff[A], args=["--x", "True"], - default=A(False), + default=A(False), # type: ignore ) == A(True) diff --git a/tests/test_conf.py b/tests/test_conf.py index 067c51df0..3a610cb7a 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -149,10 +149,12 @@ class DefaultInstanceSubparser: DefaultInstanceSubparser, args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], ) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 == tyro.cli( tyro.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "--bc.y", "5"], - default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), # type: ignore ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) ) @@ -166,10 +168,12 @@ class DefaultInstanceSubparser: tyro.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], ) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 == tyro.cli( tyro.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--bc.y", "8"], - default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), # type: ignore ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) ) @@ -442,10 +446,12 @@ class A: default=A(False), ) == A(True) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 assert tyro.cli( tyro.conf.FlagConversionOff[A], args=["--x", "True"], - default=A(False), + default=A(False), # type: ignore ) == A(True) @@ -463,10 +469,12 @@ class A: ) == A(True) with pytest.raises(SystemExit): + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 assert tyro.cli( tyro.conf.FlagConversionOff[A], args=["--x", "True"], - default=A(False), + default=A(False), # type: ignore ) == A(True) @@ -483,11 +491,13 @@ class A: default=A(False), ) == A(True) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 with pytest.raises(SystemExit): assert tyro.cli( tyro.conf.Fixed[tyro.conf.FlagConversionOff[A]], args=["--x", "True"], - default=A(False), + default=A(False), # type: ignore ) == A(True) @@ -711,6 +721,8 @@ class DefaultInstanceSubparser: "--no-flag", ], ) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 == tyro.cli( tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[ @@ -720,7 +732,7 @@ class DefaultInstanceSubparser: "--y", "5", ], - default=DefaultInstanceSubparser( + default=DefaultInstanceSubparser( # type: ignore x=1, bc=DefaultInstanceHTTPServer(y=3, flag=False) ), ) @@ -737,6 +749,8 @@ class DefaultInstanceSubparser: "8", ], ) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 == tyro.cli( tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[ @@ -746,7 +760,7 @@ class DefaultInstanceSubparser: "--y", "8", ], - default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), # type: ignore ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) ) diff --git a/tests/test_py311_generated/test_boolean_optional_generated.py b/tests/test_py311_generated/test_boolean_optional_generated.py index e3358bcf2..ba48bc9f3 100644 --- a/tests/test_py311_generated/test_boolean_optional_generated.py +++ b/tests/test_py311_generated/test_boolean_optional_generated.py @@ -28,10 +28,12 @@ class A: default=A(False), ) == A(False) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 assert tyro.cli( tyro.conf.FlagConversionOff[A], args=["--x", "True"], - default=A(False), + default=A(False), # type: ignore ) == A(True) @@ -60,8 +62,10 @@ class A: default=A(True), ) == A(True) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 assert tyro.cli( tyro.conf.FlagConversionOff[A], args=["--x", "True"], - default=A(False), + default=A(False), # type: ignore ) == A(True) diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index bb737fb07..dd26d8bdc 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -148,10 +148,12 @@ class DefaultInstanceSubparser: DefaultInstanceSubparser, args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "5"], ) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 == tyro.cli( tyro.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "--bc.y", "5"], - default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=3)), # type: ignore ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=5)) ) @@ -165,10 +167,12 @@ class DefaultInstanceSubparser: tyro.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--x", "1", "bc:default-instance-http-server", "--bc.y", "8"], ) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 == tyro.cli( tyro.conf.AvoidSubcommands[DefaultInstanceSubparser], args=["--bc.y", "8"], - default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), # type: ignore ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) ) @@ -439,10 +443,12 @@ class A: default=A(False), ) == A(True) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 assert tyro.cli( tyro.conf.FlagConversionOff[A], args=["--x", "True"], - default=A(False), + default=A(False), # type: ignore ) == A(True) @@ -460,10 +466,12 @@ class A: ) == A(True) with pytest.raises(SystemExit): + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 assert tyro.cli( tyro.conf.FlagConversionOff[A], args=["--x", "True"], - default=A(False), + default=A(False), # type: ignore ) == A(True) @@ -480,11 +488,13 @@ class A: default=A(False), ) == A(True) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 with pytest.raises(SystemExit): assert tyro.cli( tyro.conf.Fixed[tyro.conf.FlagConversionOff[A]], args=["--x", "True"], - default=A(False), + default=A(False), # type: ignore ) == A(True) @@ -708,6 +718,8 @@ class DefaultInstanceSubparser: "--no-flag", ], ) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 == tyro.cli( tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[ @@ -717,7 +729,7 @@ class DefaultInstanceSubparser: "--y", "5", ], - default=DefaultInstanceSubparser( + default=DefaultInstanceSubparser( # type: ignore x=1, bc=DefaultInstanceHTTPServer(y=3, flag=False) ), ) @@ -734,6 +746,8 @@ class DefaultInstanceSubparser: "8", ], ) + # Type ignore can be removed once TypeForm lands. + # https://discuss.python.org/t/typeform-spelling-for-a-type-annotation-object-at-runtime/51435 == tyro.cli( tyro.conf.ConsolidateSubcommandArgs[DefaultInstanceSubparser], args=[ @@ -743,7 +757,7 @@ class DefaultInstanceSubparser: "--y", "8", ], - default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), + default=DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=7)), # type: ignore ) == DefaultInstanceSubparser(x=1, bc=DefaultInstanceHTTPServer(y=8)) ) @@ -1178,6 +1192,21 @@ class Config: assert "'b'" not in error +def test_custom_constructor_9() -> None: + def commit(branch: str) -> int: + """Commit""" + print(f"commit branch={branch}") + return 3 + + assert ( + tyro.cli( # type: ignore + Annotated[Any, tyro.conf.arg(constructor=commit)], + args="--branch 5".split(" "), + ) + == 3 + ) + + def test_alias() -> None: """Arguments with aliases.""" diff --git a/tests/test_py311_generated/test_errors_generated.py b/tests/test_py311_generated/test_errors_generated.py index 08facc3ca..bb67c2d78 100644 --- a/tests/test_py311_generated/test_errors_generated.py +++ b/tests/test_py311_generated/test_errors_generated.py @@ -30,16 +30,25 @@ def test_ambiguous_collection_2() -> None: def main(x: Tuple[List[str], List[str]]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(tyro.UnsupportedTypeAnnotationError) as e: tyro.cli(main, args=["--help"]) + assert "Unsupported type annotation for the field x" in e.value.args[0] def test_ambiguous_collection_3() -> None: def main(x: List[Tuple[int, int] | Tuple[int, int, int]]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(tyro.UnsupportedTypeAnnotationError) as e: tyro.cli(main, args=["--help"]) + assert "Unsupported type annotation for the field x" in e.value.args[0] + + +def test_ambiguous_collection_4() -> None: + X = List[Tuple[int, int] | Tuple[int, int, int]] + with pytest.raises(tyro.UnsupportedTypeAnnotationError) as e: + tyro.cli(X, args=["--help"]) + assert "Unsupported type annotation for the field" not in e.value.args[0] # Must be global. diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index dca4d0841..5fb5b31be 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -1148,6 +1148,16 @@ def commit(message: str, all: bool = False) -> Tuple[str, bool]: """Make a commit.""" return (message, all) + # If we only get one, we unfortunately can't form subcommands. This is + # because unions in Python require at least 2 types. + with pytest.raises(AssertionError): + tyro.extras.subcommand_cli_from_dict( + { + "commit": commit, + }, + args="--message hello --all".split(" "), + ) + assert ( tyro.extras.subcommand_cli_from_dict( { From 084eee75db5d39b9bc4210ebba4155e4d0e566bc Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 28 Jul 2024 20:22:45 -0700 Subject: [PATCH 418/491] Docs wording nit --- .../examples/01_basics/03_dataclasses_defaults.rst | 9 +++++---- examples/01_basics/03_dataclasses_defaults.py | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/source/examples/01_basics/03_dataclasses_defaults.rst b/docs/source/examples/01_basics/03_dataclasses_defaults.rst index 51dba3a56..ba87d9fac 100644 --- a/docs/source/examples/01_basics/03_dataclasses_defaults.rst +++ b/docs/source/examples/01_basics/03_dataclasses_defaults.rst @@ -11,10 +11,11 @@ types. .. warning:: We advise against mutation of configuration objects from a dataclass's :code:`__post_init__` method [#f1]_. In the example below, - :code:`__post_init__` would be called for both the :code:`Args()` object - provided as a default value and as an output from :func:`tyro.cli()`. - Instead, we show below one example of how derived fields can be defined - immutably. + :code:`__post_init__` would be called twice: once for the :code:`Args()` + object provided as a default value and another time for the :code:`Args()` + objected instantiated by :func:`tyro.cli()`. This can cause confusing + behavior! Instead, we show below one example of how derived fields can be + defined immutably. .. [#f1] Official Python docs for ``__post_init__`` can be found `here `_. diff --git a/examples/01_basics/03_dataclasses_defaults.py b/examples/01_basics/03_dataclasses_defaults.py index 8b62676af..2a5670383 100644 --- a/examples/01_basics/03_dataclasses_defaults.py +++ b/examples/01_basics/03_dataclasses_defaults.py @@ -7,10 +7,11 @@ .. warning:: We advise against mutation of configuration objects from a dataclass's :code:`__post_init__` method [#f1]_. In the example below, - :code:`__post_init__` would be called for both the :code:`Args()` object - provided as a default value and as an output from :func:`tyro.cli()`. - Instead, we show below one example of how derived fields can be defined - immutably. + :code:`__post_init__` would be called twice: once for the :code:`Args()` + object provided as a default value and another time for the :code:`Args()` + objected instantiated by :func:`tyro.cli()`. This can cause confusing + behavior! Instead, we show below one example of how derived fields can be + defined immutably. .. [#f1] Official Python docs for ``__post_init__`` can be found `here `_. From e22690844de1695ef67fdb5f00e44242ccfe4c89 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 30 Jul 2024 22:03:55 -0700 Subject: [PATCH 419/491] Remove unnecessary error message assert --- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- src/tyro/_docstrings.py | 21 +++++++++++++-------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bf453bb5a..05dd00f8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.5" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.6" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index dd34b56a9..df3545c57 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.5" +__version__ = "0.8.6" diff --git a/src/tyro/_docstrings.py b/src/tyro/_docstrings.py index 9c49eef4d..102d42f50 100644 --- a/src/tyro/_docstrings.py +++ b/src/tyro/_docstrings.py @@ -126,14 +126,19 @@ def get_class_tokenization_with_field( try: tokenization = _ClassTokenization.make(search_cls) # type: ignore - except OSError as e: - assert ( - # Dynamic dataclasses will result in an OSError -- this is fine, we just assume - # there's no docstring. - "could not find class definition" in e.args[0] - # Pydantic. - or "source code not available" in e.args[0] - ) + except OSError: + # OSError is raised when we can't read the source code. This is + # fine, we just assume there's no docstring. We can uncomment the + # assert below for debugging. + # + # assert ( + # # Dynamic dataclasses. + # "could not find class definition" in e.args[0] + # # Pydantic. + # or "source code not available" in e.args[0] + # # Third error that can be raised by inspect.py. + # or "could not get source code" in e.args[0] + # ) return None except TypeError as e: # pragma: no cover # Notebooks cause “___ is a built-in class” TypeError. From 51c685904a498a3241a4703c48e1cf8adff31f18 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 6 Aug 2024 23:33:23 -0700 Subject: [PATCH 420/491] Recursively apply prefix_name for `tyro.conf.arg()` Related to this comment from #148: https://github.com/brentyi/tyro/issues/148#issuecomment-2268426974, but doesn't actually impact the example in the comment. --- src/tyro/_parsers.py | 4 +++- tests/test_conf.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 8d341f55d..39d469f9f 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -350,7 +350,9 @@ def handle_field( ), extern_prefix=_strings.make_field_name( [extern_prefix, field.extern_name] - ), + ) + if field.argconf.prefix_name in (True, None) + else field.extern_name, subcommand_prefix=subcommand_prefix, support_single_arg_types=False, ) diff --git a/tests/test_conf.py b/tests/test_conf.py index 3a610cb7a..219ecfdc6 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1210,6 +1210,44 @@ def commit(branch: str) -> int: ) +def test_custom_constructor_10() -> None: + def commit(branch: str) -> int: + """Commit""" + print(f"commit branch={branch}") + return 3 + + def inner(x: Annotated[Any, tyro.conf.arg(constructor=commit)]) -> None: + return x + + def inner_no_prefix( + x: Annotated[Any, tyro.conf.arg(constructor=commit, prefix_name=False)], + ) -> None: + return x + + def outer(x: Annotated[Any, tyro.conf.arg(constructor=inner)]) -> None: + return x + + def outer_no_prefix( + x: Annotated[Any, tyro.conf.arg(constructor=inner_no_prefix)], + ) -> None: + return x + + assert ( + tyro.cli( + outer, + args="--x.x.branch 5".split(" "), + ) + == 3 + ) + assert ( + tyro.cli( + outer_no_prefix, + args="--x.branch 5".split(" "), + ) + == 3 + ) + + def test_alias() -> None: """Arguments with aliases.""" From bceff43dfbee76f58734905f3aa66580216bf381 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 7 Aug 2024 13:09:07 -0700 Subject: [PATCH 421/491] Add test for `torch.inference_mode()` decorator --- tests/test_dcargs.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index c07f1778c..c0a6c640e 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -512,6 +512,14 @@ def main(device: torch.device) -> torch.device: assert tyro.cli(main, args=["--device", "cpu"]) == torch.device("cpu") +def test_supports_inference_mode_decorator() -> None: + @torch.inference_mode() + def main(x: int, device: str) -> Tuple[int, str]: + return x, device + + assert tyro.cli(main, args="--x 3 --device cuda".split(" ")) == (3, "cuda") + + def test_torch_device_2() -> None: assert tyro.cli(torch.device, args=["cpu"]) == torch.device("cpu") From a4809a281bd090d37f02353d190a2cdaaf3e2f10 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 8 Aug 2024 15:51:35 -0700 Subject: [PATCH 422/491] Fix type narrowing edge case for sequences with zero-length default values --- src/tyro/_fields.py | 5 ++++- tests/test_collections.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 78b418ca7..6ec1555ef 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -845,7 +845,10 @@ def _field_list_from_nontuple_sequence_checked( contained_type: Any if len(get_args(f)) == 0: # A raw collection type (list, tuple, etc) was passed in, but narrowing also failed. - assert default_instance in MISSING_SINGLETONS, f"{default_instance} {f}" + assert ( + default_instance in MISSING_SINGLETONS + or len(cast(typing.Sequence, default_instance)) == 0 + ), f"Default instance {default_instance} for type {f} was unexpected!" return UnsupportedNestedTypeMessage( f"Sequence type {f} needs either an explicit type or a" " default to infer from." diff --git a/tests/test_collections.py b/tests/test_collections.py index 57a693419..006fae64e 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -447,6 +447,13 @@ def main(x: list = [0, 1, 2, "hello"]) -> Any: assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", 5] +def test_list_narrowing_empty() -> None: + def main(x: list = []) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", "5"] + + def test_set_narrowing() -> None: def main(x: set = {0, 1, 2, "hello"}) -> Any: return x @@ -454,6 +461,13 @@ def main(x: set = {0, 1, 2, "hello"}) -> Any: assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", 5} +def test_set_narrowing_empty() -> None: + def main(x: set = set()) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", "5"} + + def test_tuple_narrowing() -> None: def main(x: tuple = (0, 1, 2, "hello")) -> Any: return x @@ -461,6 +475,13 @@ def main(x: tuple = (0, 1, 2, "hello")) -> Any: assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == (0, 1, 2, "3") +def test_tuple_narrowing_empty() -> None: + def main(x: tuple = ()) -> Any: + return x + + assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == ("0", "1", "2", "3") + + def test_tuple_narrowing_empty_default() -> None: def main(x: tuple = ()) -> Any: return x From 0d79d52e72549a66114e3ad0c71c7c362357a339 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 8 Aug 2024 16:20:52 -0700 Subject: [PATCH 423/491] Improve type narrowing support: Any within sequences, collections.abc.Sequence --- src/tyro/_calling.py | 1 - src/tyro/_fields.py | 5 ++- src/tyro/_resolver.py | 30 ++++++++++--- tests/test_collections.py | 95 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 9 deletions(-) diff --git a/src/tyro/_calling.py b/src/tyro/_calling.py index b703f0b77..0fc5ece6e 100644 --- a/src/tyro/_calling.py +++ b/src/tyro/_calling.py @@ -226,7 +226,6 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: unwrapped_f, default_instance ) unwrapped_f = _resolver.unwrap_origin_strip_extras(unwrapped_f) - unwrapped_f = list if unwrapped_f is Sequence else unwrapped_f # type: ignore if unwrapped_f in (tuple, list, set): if len(positional_args) > 0: diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 6ec1555ef..6bbdd4109 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -42,6 +42,7 @@ is_typeddict, ) +from . import conf # Avoid circular import. from . import ( _docstrings, _instantiators, @@ -49,7 +50,6 @@ _singleton, _strings, _unsafe_cache, - conf, # Avoid circular import. ) from ._typing import TypeForm from .conf import _confstruct, _markers @@ -428,10 +428,11 @@ def _try_field_list_from_callable( dict, ): return _field_list_from_dict(f, default_instance) - elif f_origin in (list, set, typing.Sequence) or cls in ( + elif f_origin in (list, set, typing.Sequence, collections.abc.Sequence) or cls in ( list, set, typing.Sequence, + collections.abc.Sequence, ): return _field_list_from_nontuple_sequence_checked(f, default_instance) diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 09de425b5..ee16c9f9e 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -15,6 +15,7 @@ FrozenSet, List, Optional, + Sequence, Set, Tuple, TypeVar, @@ -263,15 +264,26 @@ def narrow_collection_types( typ: TypeOrCallable, default_instance: Any ) -> TypeOrCallable: """TypeForm narrowing for containers. Infers types of container contents.""" - if typ is list and isinstance(default_instance, list): + args = get_args(typ) + origin = get_origin(typ) + if args == (Any,) or (origin is tuple and args == (Any, Ellipsis)): + typ = origin # type: ignore + + if typ in (list, Sequence, collections.abc.Sequence) and isinstance( + default_instance, list + ): if len(default_instance) == 0: return typ typ = List.__getitem__(Union.__getitem__(tuple(map(type, default_instance)))) # type: ignore - elif typ is set and isinstance(default_instance, set): + elif typ in (set, Sequence, collections.abc.Sequence) and isinstance( + default_instance, set + ): if len(default_instance) == 0: return typ typ = Set.__getitem__(Union.__getitem__(tuple(map(type, default_instance)))) # type: ignore - elif typ is tuple and isinstance(default_instance, tuple): + elif typ in (tuple, Sequence, collections.abc.Sequence) and isinstance( + default_instance, tuple + ): if len(default_instance) == 0: return typ typ = Tuple.__getitem__(tuple(map(type, default_instance))) # type: ignore @@ -406,9 +418,15 @@ def apply_type_from_typevar( if isinstance(typ, new) or origin is new: # type: ignore typ = old.__getitem__(args) # type: ignore - return typ.copy_with( # type: ignore - tuple(apply_type_from_typevar(x, type_from_typevar) for x in args) - ) + new_args = tuple(apply_type_from_typevar(x, type_from_typevar) for x in args) + + # Standard generic aliases have a `copy_with()`! + if hasattr(typ, "copy_with"): + return typ.copy_with(new_args) # type: ignore + else: + # `collections` types, like collections.abc.Sequence. + assert hasattr(origin, "__class_getitem__") + return origin.__class_getitem__(new_args) # type: ignore return typ diff --git a/tests/test_collections.py b/tests/test_collections.py index 006fae64e..dedf1f5f1 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -155,6 +155,59 @@ class A: tyro.cli(A, args=[]) +def test_sequences_narrow() -> None: + @dataclasses.dataclass + class A: + x: Sequence = dataclasses.field(default_factory=lambda: [0]) + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=[]) == A(x=[0]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + + +def test_sequences_narrow_any() -> None: + @dataclasses.dataclass + class A: + x: Sequence[Any] = dataclasses.field(default_factory=lambda: [0]) + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=[]) == A(x=[0]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + + +def test_abc_sequences() -> None: + @dataclasses.dataclass + class A: + x: collections.abc.Sequence[int] + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_abc_sequences_narrow() -> None: + @dataclasses.dataclass + class A: + x: collections.abc.Sequence = dataclasses.field(default_factory=lambda: [0]) + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=[]) == A(x=[0]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + + +def test_abc_sequences_narrow_any() -> None: + @dataclasses.dataclass + class A: + x: collections.abc.Sequence[Any] = dataclasses.field( + default_factory=lambda: [0] + ) + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=[]) == A(x=[0]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + + def test_lists() -> None: @dataclasses.dataclass class A: @@ -447,6 +500,13 @@ def main(x: list = [0, 1, 2, "hello"]) -> Any: assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", 5] +def test_list_narrowing_any() -> None: + def main(x: list[Any] = [0, 1, 2, "hello"]) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", 5] + + def test_list_narrowing_empty() -> None: def main(x: list = []) -> Any: return x @@ -454,6 +514,13 @@ def main(x: list = []) -> Any: assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", "5"] +def test_list_narrowing_empty_any() -> None: + def main(x: list[Any] = []) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", "5"] + + def test_set_narrowing() -> None: def main(x: set = {0, 1, 2, "hello"}) -> Any: return x @@ -461,6 +528,13 @@ def main(x: set = {0, 1, 2, "hello"}) -> Any: assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", 5} +def test_set_narrowing_any() -> None: + def main(x: set[Any] = {0, 1, 2, "hello"}) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", 5} + + def test_set_narrowing_empty() -> None: def main(x: set = set()) -> Any: return x @@ -468,6 +542,13 @@ def main(x: set = set()) -> Any: assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", "5"} +def test_set_narrowing_any_empty() -> None: + def main(x: set[Any] = set()) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", "5"} + + def test_tuple_narrowing() -> None: def main(x: tuple = (0, 1, 2, "hello")) -> Any: return x @@ -475,6 +556,13 @@ def main(x: tuple = (0, 1, 2, "hello")) -> Any: assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == (0, 1, 2, "3") +def test_tuple_narrowing_any() -> None: + def main(x: tuple[Any, ...] = (0, 1, 2, "hello")) -> Any: + return x + + assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == (0, 1, 2, "3") + + def test_tuple_narrowing_empty() -> None: def main(x: tuple = ()) -> Any: return x @@ -482,6 +570,13 @@ def main(x: tuple = ()) -> Any: assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == ("0", "1", "2", "3") +def test_tuple_narrowing_empty_any() -> None: + def main(x: tuple[Any, ...] = ()) -> Any: + return x + + assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == ("0", "1", "2", "3") + + def test_tuple_narrowing_empty_default() -> None: def main(x: tuple = ()) -> Any: return x From af310445375cf0f616a2eb5aa00868f2182a7c77 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 8 Aug 2024 16:21:27 -0700 Subject: [PATCH 424/491] ruff --- src/tyro/_calling.py | 2 +- src/tyro/_fields.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tyro/_calling.py b/src/tyro/_calling.py index 0fc5ece6e..04d406eef 100644 --- a/src/tyro/_calling.py +++ b/src/tyro/_calling.py @@ -6,7 +6,7 @@ import dataclasses import itertools from functools import partial -from typing import Any, Callable, Dict, List, Sequence, Set, Tuple, TypeVar, Union +from typing import Any, Callable, Dict, List, Set, Tuple, TypeVar, Union from typing_extensions import get_args diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 6bbdd4109..cf481ecb5 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -42,7 +42,6 @@ is_typeddict, ) -from . import conf # Avoid circular import. from . import ( _docstrings, _instantiators, @@ -50,6 +49,7 @@ _singleton, _strings, _unsafe_cache, + conf, # Avoid circular import. ) from ._typing import TypeForm from .conf import _confstruct, _markers From faddd37e53601184f4aff0081ccbc4947c04a169 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 8 Aug 2024 16:24:50 -0700 Subject: [PATCH 425/491] Fix tests for Python<3.10 --- tests/test_collections.py | 52 +++++++++++++----------- tests/test_generics_and_serialization.py | 25 +++++++----- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/tests/test_collections.py b/tests/test_collections.py index dedf1f5f1..2f84ffefb 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -1,8 +1,10 @@ import collections +import collections.abc import contextlib import dataclasses import enum import io +import sys from typing import ( Any, Deque, @@ -175,15 +177,17 @@ class A: assert tyro.cli(A, args=["--x"]) == A(x=[]) -def test_abc_sequences() -> None: - @dataclasses.dataclass - class A: - x: collections.abc.Sequence[int] +if sys.version_info >= (3, 9): - assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) - assert tyro.cli(A, args=["--x"]) == A(x=[]) - with pytest.raises(SystemExit): - tyro.cli(A, args=[]) + def test_abc_sequences() -> None: + @dataclasses.dataclass + class A: + x: collections.abc.Sequence[int] + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) def test_abc_sequences_narrow() -> None: @@ -196,16 +200,18 @@ class A: assert tyro.cli(A, args=["--x"]) == A(x=[]) -def test_abc_sequences_narrow_any() -> None: - @dataclasses.dataclass - class A: - x: collections.abc.Sequence[Any] = dataclasses.field( - default_factory=lambda: [0] - ) +if sys.version_info >= (3, 9): - assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) - assert tyro.cli(A, args=[]) == A(x=[0]) - assert tyro.cli(A, args=["--x"]) == A(x=[]) + def test_abc_sequences_narrow_any() -> None: + @dataclasses.dataclass + class A: + x: collections.abc.Sequence[Any] = dataclasses.field( + default_factory=lambda: [0] + ) + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=[]) == A(x=[0]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) def test_lists() -> None: @@ -501,7 +507,7 @@ def main(x: list = [0, 1, 2, "hello"]) -> Any: def test_list_narrowing_any() -> None: - def main(x: list[Any] = [0, 1, 2, "hello"]) -> Any: + def main(x: List[Any] = [0, 1, 2, "hello"]) -> Any: return x assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", 5] @@ -515,7 +521,7 @@ def main(x: list = []) -> Any: def test_list_narrowing_empty_any() -> None: - def main(x: list[Any] = []) -> Any: + def main(x: List[Any] = []) -> Any: return x assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", "5"] @@ -529,7 +535,7 @@ def main(x: set = {0, 1, 2, "hello"}) -> Any: def test_set_narrowing_any() -> None: - def main(x: set[Any] = {0, 1, 2, "hello"}) -> Any: + def main(x: Set[Any] = {0, 1, 2, "hello"}) -> Any: return x assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", 5} @@ -543,7 +549,7 @@ def main(x: set = set()) -> Any: def test_set_narrowing_any_empty() -> None: - def main(x: set[Any] = set()) -> Any: + def main(x: Set[Any] = set()) -> Any: return x assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", "5"} @@ -557,7 +563,7 @@ def main(x: tuple = (0, 1, 2, "hello")) -> Any: def test_tuple_narrowing_any() -> None: - def main(x: tuple[Any, ...] = (0, 1, 2, "hello")) -> Any: + def main(x: Tuple[Any, ...] = (0, 1, 2, "hello")) -> Any: return x assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == (0, 1, 2, "3") @@ -571,7 +577,7 @@ def main(x: tuple = ()) -> Any: def test_tuple_narrowing_empty_any() -> None: - def main(x: tuple[Any, ...] = ()) -> Any: + def main(x: Tuple[Any, ...] = ()) -> Any: return x assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == ("0", "1", "2", "3") diff --git a/tests/test_generics_and_serialization.py b/tests/test_generics_and_serialization.py index 89d3d48ad..467a73424 100644 --- a/tests/test_generics_and_serialization.py +++ b/tests/test_generics_and_serialization.py @@ -438,20 +438,23 @@ class Wrapper: assert wrapper1 == tyro.extras.from_yaml(Wrapper, tyro.extras.to_yaml(wrapper1)) -def test_superclass() -> None: - # https://github.com/brentyi/tyro/issues/7 +@dataclasses.dataclass +class TypeA: + data: int - @dataclasses.dataclass - class TypeA: - data: int - @dataclasses.dataclass - class TypeASubclass(TypeA): - pass +@dataclasses.dataclass +class TypeASubclass(TypeA): + pass - @dataclasses.dataclass - class Wrapper: - subclass: TypeA + +@dataclasses.dataclass +class Wrapper: + subclass: TypeA + + +def test_superclass() -> None: + # https://github.com/brentyi/tyro/issues/7 wrapper1 = Wrapper(TypeASubclass(3)) # Create Wrapper object. assert wrapper1 == tyro.extras.from_yaml(Wrapper, tyro.extras.to_yaml(wrapper1)) From 29d7d202a48c62804e444012300e1c87e0eee4d8 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 8 Aug 2024 16:31:34 -0700 Subject: [PATCH 426/491] Update autogenerated Python>=3.11 tests --- tests/test_py311_generated/_generate.py | 1 - .../test_collections_generated.py | 122 ++++++++++++++++++ .../test_conf_generated.py | 38 ++++++ .../test_dcargs_generated.py | 8 ++ ...st_generics_and_serialization_generated.py | 25 ++-- 5 files changed, 182 insertions(+), 12 deletions(-) diff --git a/tests/test_py311_generated/_generate.py b/tests/test_py311_generated/_generate.py index 77c93968a..6ff72a438 100644 --- a/tests/test_py311_generated/_generate.py +++ b/tests/test_py311_generated/_generate.py @@ -49,7 +49,6 @@ def generate_from_path(test_path: pathlib.Path) -> None: ) out_path.write_text(content) - subprocess.run(["isort", "--profile=black", str(out_path)], check=True) subprocess.run(["ruff", "format", str(out_path)], check=True) subprocess.run(["ruff", "check", "--fix", str(out_path)], check=True) diff --git a/tests/test_py311_generated/test_collections_generated.py b/tests/test_py311_generated/test_collections_generated.py index ff03ac8df..5b6e3d811 100644 --- a/tests/test_py311_generated/test_collections_generated.py +++ b/tests/test_py311_generated/test_collections_generated.py @@ -1,8 +1,10 @@ import collections +import collections.abc import contextlib import dataclasses import enum import io +import sys from typing import ( Any, Deque, @@ -154,6 +156,63 @@ class A: tyro.cli(A, args=[]) +def test_sequences_narrow() -> None: + @dataclasses.dataclass + class A: + x: Sequence = dataclasses.field(default_factory=lambda: [0]) + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=[]) == A(x=[0]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + + +def test_sequences_narrow_any() -> None: + @dataclasses.dataclass + class A: + x: Sequence[Any] = dataclasses.field(default_factory=lambda: [0]) + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=[]) == A(x=[0]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + + +if sys.version_info >= (3, 9): + + def test_abc_sequences() -> None: + @dataclasses.dataclass + class A: + x: collections.abc.Sequence[int] + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + with pytest.raises(SystemExit): + tyro.cli(A, args=[]) + + +def test_abc_sequences_narrow() -> None: + @dataclasses.dataclass + class A: + x: collections.abc.Sequence = dataclasses.field(default_factory=lambda: [0]) + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=[]) == A(x=[0]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + + +if sys.version_info >= (3, 9): + + def test_abc_sequences_narrow_any() -> None: + @dataclasses.dataclass + class A: + x: collections.abc.Sequence[Any] = dataclasses.field( + default_factory=lambda: [0] + ) + + assert tyro.cli(A, args=["--x", "1", "2", "3"]) == A(x=[1, 2, 3]) + assert tyro.cli(A, args=[]) == A(x=[0]) + assert tyro.cli(A, args=["--x"]) == A(x=[]) + + def test_lists() -> None: @dataclasses.dataclass class A: @@ -446,6 +505,27 @@ def main(x: list = [0, 1, 2, "hello"]) -> Any: assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", 5] +def test_list_narrowing_any() -> None: + def main(x: List[Any] = [0, 1, 2, "hello"]) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", 5] + + +def test_list_narrowing_empty() -> None: + def main(x: list = []) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", "5"] + + +def test_list_narrowing_empty_any() -> None: + def main(x: List[Any] = []) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == ["hi", "there", "5"] + + def test_set_narrowing() -> None: def main(x: set = {0, 1, 2, "hello"}) -> Any: return x @@ -453,6 +533,27 @@ def main(x: set = {0, 1, 2, "hello"}) -> Any: assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", 5} +def test_set_narrowing_any() -> None: + def main(x: Set[Any] = {0, 1, 2, "hello"}) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", 5} + + +def test_set_narrowing_empty() -> None: + def main(x: set = set()) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", "5"} + + +def test_set_narrowing_any_empty() -> None: + def main(x: Set[Any] = set()) -> Any: + return x + + assert tyro.cli(main, args="--x hi there 5".split(" ")) == {"hi", "there", "5"} + + def test_tuple_narrowing() -> None: def main(x: tuple = (0, 1, 2, "hello")) -> Any: return x @@ -460,6 +561,27 @@ def main(x: tuple = (0, 1, 2, "hello")) -> Any: assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == (0, 1, 2, "3") +def test_tuple_narrowing_any() -> None: + def main(x: Tuple[Any, ...] = (0, 1, 2, "hello")) -> Any: + return x + + assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == (0, 1, 2, "3") + + +def test_tuple_narrowing_empty() -> None: + def main(x: tuple = ()) -> Any: + return x + + assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == ("0", "1", "2", "3") + + +def test_tuple_narrowing_empty_any() -> None: + def main(x: Tuple[Any, ...] = ()) -> Any: + return x + + assert tyro.cli(main, args="--x 0 1 2 3".split(" ")) == ("0", "1", "2", "3") + + def test_tuple_narrowing_empty_default() -> None: def main(x: tuple = ()) -> Any: return x diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index dd26d8bdc..7f688f526 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -1207,6 +1207,44 @@ def commit(branch: str) -> int: ) +def test_custom_constructor_10() -> None: + def commit(branch: str) -> int: + """Commit""" + print(f"commit branch={branch}") + return 3 + + def inner(x: Annotated[Any, tyro.conf.arg(constructor=commit)]) -> None: + return x + + def inner_no_prefix( + x: Annotated[Any, tyro.conf.arg(constructor=commit, prefix_name=False)], + ) -> None: + return x + + def outer(x: Annotated[Any, tyro.conf.arg(constructor=inner)]) -> None: + return x + + def outer_no_prefix( + x: Annotated[Any, tyro.conf.arg(constructor=inner_no_prefix)], + ) -> None: + return x + + assert ( + tyro.cli( + outer, + args="--x.x.branch 5".split(" "), + ) + == 3 + ) + assert ( + tyro.cli( + outer_no_prefix, + args="--x.branch 5".split(" "), + ) + == 3 + ) + + def test_alias() -> None: """Arguments with aliases.""" diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py index 25b1b7c13..fadcf18ed 100644 --- a/tests/test_py311_generated/test_dcargs_generated.py +++ b/tests/test_py311_generated/test_dcargs_generated.py @@ -514,6 +514,14 @@ def main(device: torch.device) -> torch.device: assert tyro.cli(main, args=["--device", "cpu"]) == torch.device("cpu") +def test_supports_inference_mode_decorator() -> None: + @torch.inference_mode() + def main(x: int, device: str) -> Tuple[int, str]: + return x, device + + assert tyro.cli(main, args="--x 3 --device cuda".split(" ")) == (3, "cuda") + + def test_torch_device_2() -> None: assert tyro.cli(torch.device, args=["cpu"]) == torch.device("cpu") diff --git a/tests/test_py311_generated/test_generics_and_serialization_generated.py b/tests/test_py311_generated/test_generics_and_serialization_generated.py index 93eea5277..d832a3f31 100644 --- a/tests/test_py311_generated/test_generics_and_serialization_generated.py +++ b/tests/test_py311_generated/test_generics_and_serialization_generated.py @@ -436,20 +436,23 @@ class Wrapper: assert wrapper1 == tyro.extras.from_yaml(Wrapper, tyro.extras.to_yaml(wrapper1)) -def test_superclass() -> None: - # https://github.com/brentyi/tyro/issues/7 +@dataclasses.dataclass +class TypeA: + data: int - @dataclasses.dataclass - class TypeA: - data: int - @dataclasses.dataclass - class TypeASubclass(TypeA): - pass +@dataclasses.dataclass +class TypeASubclass(TypeA): + pass - @dataclasses.dataclass - class Wrapper: - subclass: TypeA + +@dataclasses.dataclass +class Wrapper: + subclass: TypeA + + +def test_superclass() -> None: + # https://github.com/brentyi/tyro/issues/7 wrapper1 = Wrapper(TypeASubclass(3)) # Create Wrapper object. assert wrapper1 == tyro.extras.from_yaml(Wrapper, tyro.extras.to_yaml(wrapper1)) From 668b072c68b36c07c75ad92f7aca0b82fe1cd9f9 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 14 Aug 2024 00:35:34 -0700 Subject: [PATCH 427/491] Improve error messages when custom constructors are combined with default values --- src/tyro/_calling.py | 21 ++++++++++++++++----- tests/test_conf.py | 24 +++++++++++++++--------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/tyro/_calling.py b/src/tyro/_calling.py index 04d406eef..16508e86a 100644 --- a/src/tyro/_calling.py +++ b/src/tyro/_calling.py @@ -206,11 +206,22 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: message = "either all arguments must be provided or none of them." if len(kwargs) > 0: - missing_kwargs = [ - k for k, v in kwargs.items() if v in _fields.MISSING_SINGLETONS - ] - if len(missing_kwargs): - message += f" We're missing arguments {missing_kwargs}." + missing_args: List[str] = [] + for k, v in kwargs.items(): + if v not in _fields.MISSING_SINGLETONS: + break + + # Argument is missing. + found = False + for arg in arg_from_prefixed_field_name.values(): + if arg.field.call_argname == k: + missing_args.append(arg.lowered.name_or_flag) + found = True + break + assert found, "This is likely a bug in tyro." + + if len(missing_args): + message += f" We're missing arguments {missing_args}." raise InstantiationError( message, field_name_prefix, diff --git a/tests/test_conf.py b/tests/test_conf.py index 219ecfdc6..910fde5e9 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1102,8 +1102,12 @@ class Config: def test_custom_constructor_6() -> None: - def make_float(a: tyro.conf.Positional[float], b: float, c: float = 3) -> float: - return a * b * c + def make_float( + a: tyro.conf.Positional[float], + b2: Annotated[float, tyro.conf.arg(name="b")], + c: float = 3, + ) -> float: + return a * b2 * c @dataclasses.dataclass class Config: @@ -1122,7 +1126,9 @@ class Config: with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Config, args="--x.c 5".split(" ")) error = target.getvalue() - assert "We're missing" in error + assert "either all arguments must be provided" in error + assert "or none of them" in error + assert "We're missing arguments" in error def test_custom_constructor_7() -> None: @@ -1157,8 +1163,8 @@ class Config: tyro.cli(Config, args="--x.struct.c 5".split(" ")) error = target.getvalue() assert "We're missing arguments" in error - assert "'b'" in error - assert "'a'" in error # The 5 is parsed into `a`. + assert "'--x.struct.b'" in error + assert "'--x.struct.a'" in error # The 5 is parsed into `a`. def test_custom_constructor_8() -> None: @@ -1191,8 +1197,8 @@ class Config: tyro.cli(Config, args="--x.struct.b 5".split(" ")) error = target.getvalue() assert "We're missing arguments" in error - assert "'a'" in error - assert "'b'" not in error + assert "'x.struct.a'" in error + assert "'--x.struct.b'" not in error def test_custom_constructor_9() -> None: @@ -1288,8 +1294,8 @@ class Config: tyro.cli(Config, args="--x.struct.b 5".split(" ")) error = target.getvalue() assert "We're missing arguments" in error - assert "'a'" in error - assert "'b'" not in error + assert "'--x.struct.a'" in error + assert "'--x.struct.b'" not in error assert "--x.struct.a INT, --all INT, -d INT" in get_helptext_with_checks(Config) From 95e2c2fd5dc1012f44dcda40ab613cfe16497305 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 14 Aug 2024 00:39:10 -0700 Subject: [PATCH 428/491] Improve error message for variable-length sequences over nested types (cc #151) --- src/tyro/_fields.py | 6 +++-- .../test_conf_generated.py | 24 ++++++++++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index cf481ecb5..822a84f0f 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -886,8 +886,10 @@ def _try_field_list_from_sequence_inner( # We use the broader error type to prevent it from being caught by # is_possibly_nested_type(). This is for sure a bad annotation! raise _instantiators.UnsupportedTypeAnnotationError( - "For variable-length sequences over nested types, we need a default value" - " to infer length from." + "tyro currently only supports fixed-length sequences of nested types. For " + "variable-length sequences over nested types, we need a default value to " + "infer length from. You can also consider a custom constructor, see here " + "for an example: https://github.com/brentyi/tyro/issues/151" ) field_list = [] diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index 7f688f526..b58ad8677 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -1099,8 +1099,12 @@ class Config: def test_custom_constructor_6() -> None: - def make_float(a: tyro.conf.Positional[float], b: float, c: float = 3) -> float: - return a * b * c + def make_float( + a: tyro.conf.Positional[float], + b2: Annotated[float, tyro.conf.arg(name="b")], + c: float = 3, + ) -> float: + return a * b2 * c @dataclasses.dataclass class Config: @@ -1119,7 +1123,9 @@ class Config: with pytest.raises(SystemExit), contextlib.redirect_stderr(target): tyro.cli(Config, args="--x.c 5".split(" ")) error = target.getvalue() - assert "We're missing" in error + assert "either all arguments must be provided" in error + assert "or none of them" in error + assert "We're missing arguments" in error def test_custom_constructor_7() -> None: @@ -1154,8 +1160,8 @@ class Config: tyro.cli(Config, args="--x.struct.c 5".split(" ")) error = target.getvalue() assert "We're missing arguments" in error - assert "'b'" in error - assert "'a'" in error # The 5 is parsed into `a`. + assert "'--x.struct.b'" in error + assert "'--x.struct.a'" in error # The 5 is parsed into `a`. def test_custom_constructor_8() -> None: @@ -1188,8 +1194,8 @@ class Config: tyro.cli(Config, args="--x.struct.b 5".split(" ")) error = target.getvalue() assert "We're missing arguments" in error - assert "'a'" in error - assert "'b'" not in error + assert "'x.struct.a'" in error + assert "'--x.struct.b'" not in error def test_custom_constructor_9() -> None: @@ -1285,8 +1291,8 @@ class Config: tyro.cli(Config, args="--x.struct.b 5".split(" ")) error = target.getvalue() assert "We're missing arguments" in error - assert "'a'" in error - assert "'b'" not in error + assert "'--x.struct.a'" in error + assert "'--x.struct.b'" not in error assert "--x.struct.a INT, --all INT, -d INT" in get_helptext_with_checks(Config) From be74b1fec9d6e99205eafd451a24d8182083bdcd Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 14 Aug 2024 02:14:27 -0700 Subject: [PATCH 429/491] Docs typo --- docs/source/examples/03_config_systems/02_overriding_yaml.rst | 4 ++-- examples/03_config_systems/02_overriding_yaml.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/examples/03_config_systems/02_overriding_yaml.rst b/docs/source/examples/03_config_systems/02_overriding_yaml.rst index 8262bed13..e926429cc 100644 --- a/docs/source/examples/03_config_systems/02_overriding_yaml.rst +++ b/docs/source/examples/03_config_systems/02_overriding_yaml.rst @@ -5,8 +5,8 @@ Overriding YAML Configs ========================================== If you have a library of existing YAML files that you want to use, `tyro` can -be used to override values in them. This example uses a dictionary; we -generally recommend dataset configs for new projects. +be used to override values in them. We generally recommend dataclass configs +for new projects. .. code-block:: python diff --git a/examples/03_config_systems/02_overriding_yaml.py b/examples/03_config_systems/02_overriding_yaml.py index db52832d1..5db0cfd86 100644 --- a/examples/03_config_systems/02_overriding_yaml.py +++ b/examples/03_config_systems/02_overriding_yaml.py @@ -1,8 +1,8 @@ """Overriding YAML Configs If you have a library of existing YAML files that you want to use, `tyro` can -be used to override values in them. This example uses a dictionary; we -generally recommend dataset configs for new projects. +be used to override values in them. We generally recommend dataclass configs +for new projects. Usage: `python ./02_overriding_yaml.py --help` From 4775974e773ae54877b32fefefc6138744476870 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 15 Aug 2024 12:09:35 -0700 Subject: [PATCH 430/491] Add `config=` argument to `tyro.cli()`. This will become redundant once `typing.TypeForm`/`typing.TypeExpr` lands[^1] and behavior for `tyro.cli(Annotated[f])` makes it into static checkers, but either way the new API should be more intuitive/familiar for general audiences. [^1]: https://discuss.python.org/t/pep-747-typeexpr-type-hint-for-a-type-expression/55984 --- examples/02_nesting/03_multiple_subcommands.py | 3 +-- src/tyro/_cli.py | 12 ++++++++++++ src/tyro/conf/_markers.py | 7 +++++-- tests/test_conf.py | 14 ++++++++++++++ 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/examples/02_nesting/03_multiple_subcommands.py b/examples/02_nesting/03_multiple_subcommands.py index 4c2561c11..561232b54 100644 --- a/examples/02_nesting/03_multiple_subcommands.py +++ b/examples/02_nesting/03_multiple_subcommands.py @@ -48,7 +48,6 @@ class Sgd: # Train script. -@tyro.conf.configure(tyro.conf.ConsolidateSubcommandArgs) def train( dataset: Mnist | ImageNet = Mnist(), optimizer: Adam | Sgd = Adam(), @@ -67,4 +66,4 @@ def train( if __name__ == "__main__": - tyro.cli(train) + tyro.cli(train, config=(tyro.conf.ConsolidateSubcommandArgs,)) diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index 2982033bd..4173b16f2 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -53,6 +53,7 @@ def cli( return_unknown_args: Literal[False] = False, use_underscores: bool = False, console_outputs: bool = True, + config: Optional[Sequence[conf._markers.Marker]] = None, ) -> OutT: ... @@ -67,6 +68,7 @@ def cli( return_unknown_args: Literal[True], use_underscores: bool = False, console_outputs: bool = True, + config: Optional[Sequence[conf._markers.Marker]] = None, ) -> Tuple[OutT, List[str]]: ... @@ -84,6 +86,7 @@ def cli( return_unknown_args: Literal[False] = False, use_underscores: bool = False, console_outputs: bool = True, + config: Optional[Sequence[conf._markers.Marker]] = None, ) -> OutT: ... @@ -101,6 +104,7 @@ def cli( return_unknown_args: Literal[True], use_underscores: bool = False, console_outputs: bool = True, + config: Optional[Sequence[conf._markers.Marker]] = None, ) -> Tuple[OutT, List[str]]: ... @@ -114,6 +118,7 @@ def cli( return_unknown_args: bool = False, use_underscores: bool = False, console_outputs: bool = True, + config: Optional[Sequence[conf._markers.Marker]] = None, **deprecated_kwargs, ) -> Union[OutT, Tuple[OutT, List[str]]]: """Call or instantiate `f`, with inputs populated from an automatically generated @@ -180,6 +185,10 @@ def cli( supressed. This can be useful for distributed settings, where `tyro.cli()` is called from multiple workers but we only want console outputs from the main one. + config: Sequence of config marker objects, from `tyro.conf`. As an + alternative to using them locally in annotations + (`tyro.conf.FlagConversionOff[bool]`), we can also pass in a sequence of + them here to apply globally. Returns: The output of `f(...)` or an instance `f`. If `f` is a class, the two are @@ -191,6 +200,9 @@ def cli( # memory address conflicts. _unsafe_cache.clear_cache() + if config is not None: + f = conf.configure(*config)(f) + with _strings.delimeter_context("_" if use_underscores else "-"): output = _cli_impl( f, diff --git a/src/tyro/conf/_markers.py b/src/tyro/conf/_markers.py index c9a0f122b..833b5fc3c 100644 --- a/src/tyro/conf/_markers.py +++ b/src/tyro/conf/_markers.py @@ -147,8 +147,11 @@ def __getitem__(self, key): def configure(*markers: Marker) -> Callable[[CallableType], CallableType]: """Decorator for applying configuration options. - Configuration markers are implemented via `typing.Annotated` and straightforward to - apply to types, for example: + Consider using the `config=` argument of `tyro.cli()` instead, which takes the same + config marker objects as inputs. + + Configuration markers are implemented via `typing.Annotated` and straightforward + to apply to types, for example: ```python field: tyro.conf.FlagConversionOff[bool] diff --git a/tests/test_conf.py b/tests/test_conf.py index 910fde5e9..c51e9d4cd 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -453,6 +453,12 @@ class A: args=["--x", "True"], default=A(False), # type: ignore ) == A(True) + assert tyro.cli( + A, + args=["--x", "True"], + default=A(False), + config=(tyro.conf.FlagConversionOff,), + ) == A(True) def test_fixed() -> None: @@ -477,6 +483,14 @@ class A: default=A(False), # type: ignore ) == A(True) + with pytest.raises(SystemExit): + assert tyro.cli( + A, + args=["--x", "True"], + default=A(False), # type: ignore + config=(tyro.conf.FlagConversionOff,), + ) == A(True) + def test_fixed_recursive() -> None: """When an argument is fixed, we shouldn't be able to override it from the CLI.""" From cb76e7b42be5810e67cdfb2904c19aee50b6d8f2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 15 Aug 2024 12:26:50 -0700 Subject: [PATCH 431/491] 0.8.7 --- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 05dd00f8f..aa3294d05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.6" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.7" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index df3545c57..89528c408 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.6" +__version__ = "0.8.7" From 956ab5e954eb641d56b7e7b2de6938aca2029673 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 15 Aug 2024 13:05:07 -0700 Subject: [PATCH 432/491] 0.8.8, add `tyro.extras.overridable_config_cli()` --- .../02_nesting/03_multiple_subcommands.rst | 3 +- .../03_config_systems/01_base_configs.rst | 24 ++++--- examples/03_config_systems/01_base_configs.py | 24 ++++--- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- src/tyro/extras/__init__.py | 1 + src/tyro/extras/_base_configs.py | 67 ++++++++++++++++++- tests/test_base_configs_nested.py | 20 ++++++ 8 files changed, 118 insertions(+), 25 deletions(-) diff --git a/docs/source/examples/02_nesting/03_multiple_subcommands.rst b/docs/source/examples/02_nesting/03_multiple_subcommands.rst index bb62f7a0b..5064916a8 100644 --- a/docs/source/examples/02_nesting/03_multiple_subcommands.rst +++ b/docs/source/examples/02_nesting/03_multiple_subcommands.rst @@ -50,7 +50,6 @@ Multiple unions over nested types are populated using a series of subcommands. # Train script. - @tyro.conf.configure(tyro.conf.ConsolidateSubcommandArgs) def train( dataset: Mnist | ImageNet = Mnist(), optimizer: Adam | Sgd = Adam(), @@ -69,7 +68,7 @@ Multiple unions over nested types are populated using a series of subcommands. if __name__ == "__main__": - tyro.cli(train) + tyro.cli(train, config=(tyro.conf.ConsolidateSubcommandArgs,)) ------------ diff --git a/docs/source/examples/03_config_systems/01_base_configs.rst b/docs/source/examples/03_config_systems/01_base_configs.rst index d2250b353..0ac47f25e 100644 --- a/docs/source/examples/03_config_systems/01_base_configs.rst +++ b/docs/source/examples/03_config_systems/01_base_configs.rst @@ -4,10 +4,13 @@ Base Configurations ========================================== -We can integrate `tyro.cli()` into common configuration patterns: here, we select +We can integrate `tyro` into common configuration patterns: here, we select one of multiple possible base configurations, create a subcommand for each one, and then use the CLI to either override (existing) or fill in (missing) values. +The helper function used here, :func:`tyro.extras.overridable_config_cli()`, is a +lightweight wrapper over :func:`tyro.cli()`. + .. code-block:: python :linenos: @@ -56,9 +59,10 @@ use the CLI to either override (existing) or fill in (missing) values. # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. - Configs = tyro.extras.subcommand_type_from_defaults( - { - "small": ExperimentConfig( + default_configs = { + "small": ( + "Small experiment.", + ExperimentConfig( dataset="mnist", optimizer=AdamOptimizer(), batch_size=2048, @@ -68,7 +72,10 @@ use the CLI to either override (existing) or fill in (missing) values. seed=0, activation=nn.ReLU, ), - "big": ExperimentConfig( + ), + "big": ( + "Big experiment.", + ExperimentConfig( dataset="imagenet-50", optimizer=AdamOptimizer(), batch_size=32, @@ -78,11 +85,10 @@ use the CLI to either override (existing) or fill in (missing) values. seed=0, activation=nn.GELU, ), - } - ) - + ), + } if __name__ == "__main__": - config = tyro.cli(Configs) + config = tyro.extras.overridable_config_cli(default_configs) print(config) ------------ diff --git a/examples/03_config_systems/01_base_configs.py b/examples/03_config_systems/01_base_configs.py index 0d2a59e20..50ee71fa4 100644 --- a/examples/03_config_systems/01_base_configs.py +++ b/examples/03_config_systems/01_base_configs.py @@ -1,9 +1,12 @@ """Base Configurations -We can integrate `tyro.cli()` into common configuration patterns: here, we select +We can integrate `tyro` into common configuration patterns: here, we select one of multiple possible base configurations, create a subcommand for each one, and then use the CLI to either override (existing) or fill in (missing) values. +The helper function used here, :func:`tyro.extras.overridable_config_cli()`, is a +lightweight wrapper over :func:`tyro.cli()`. + Usage: `python ./01_base_configs.py --help` @@ -56,9 +59,10 @@ class ExperimentConfig: # Note that we could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. -Configs = tyro.extras.subcommand_type_from_defaults( - { - "small": ExperimentConfig( +default_configs = { + "small": ( + "Small experiment.", + ExperimentConfig( dataset="mnist", optimizer=AdamOptimizer(), batch_size=2048, @@ -68,7 +72,10 @@ class ExperimentConfig: seed=0, activation=nn.ReLU, ), - "big": ExperimentConfig( + ), + "big": ( + "Big experiment.", + ExperimentConfig( dataset="imagenet-50", optimizer=AdamOptimizer(), batch_size=32, @@ -78,9 +85,8 @@ class ExperimentConfig: seed=0, activation=nn.GELU, ), - } -) - + ), +} if __name__ == "__main__": - config = tyro.cli(Configs) + config = tyro.extras.overridable_config_cli(default_configs) print(config) diff --git a/pyproject.toml b/pyproject.toml index aa3294d05..821dcaf3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.7" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.8" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index 89528c408..e7bafa092 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.7" +__version__ = "0.8.8" diff --git a/src/tyro/extras/__init__.py b/src/tyro/extras/__init__.py index 567ddaf31..0afb6b854 100644 --- a/src/tyro/extras/__init__.py +++ b/src/tyro/extras/__init__.py @@ -4,6 +4,7 @@ from .._argparse_formatter import set_accent_color as set_accent_color from .._cli import get_parser as get_parser +from ._base_configs import overridable_config_cli as overridable_config_cli from ._base_configs import ( subcommand_type_from_defaults as subcommand_type_from_defaults, ) diff --git a/src/tyro/extras/_base_configs.py b/src/tyro/extras/_base_configs.py index 117a944a3..33157e076 100644 --- a/src/tyro/extras/_base_configs.py +++ b/src/tyro/extras/_base_configs.py @@ -1,13 +1,72 @@ -from typing import Mapping, TypeVar, Union +from typing import Mapping, Optional, Sequence, Tuple, TypeVar, Union from typing_extensions import Annotated from .._typing import TypeForm -from ..conf import subcommand T = TypeVar("T") +def overridable_config_cli( + configs: Mapping[str, Tuple[str, T]], + *, + args: Optional[Sequence[str]] = None, +) -> T: + """Helper function for creating a CLI interface that allows us to choose + between default config objects (typically dataclasses) and override values + within it. Turns off subcommand creation for any union types within the + config object. + + This is a lightweight wrapper over :func:`tyro.cli()`, with some default + arguments populated. Also see + :func:`tyro.extras.subcommand_type_from_defaults()`. + + + Example usage: + ```python + import dataclasses + + import tyro + + + @dataclasses.dataclass + class Config: + a: int + b: str + + + default_configs = { + "small": ( + "Small config", + Config(1, "small"), + ), + "big": ( + "Big config", + Config(100, "big"), + ), + } + config = tyro.extras.overridable_config_cli(default_configs) + print(config) + ``` + + Args: + configs: A dictionary of config names mapped to a tuple of + (description, config object). + args: Optional arguments to pass to the CLI. + """ + import tyro + + return tyro.cli( + tyro.extras.subcommand_type_from_defaults( + defaults={k: v[1] for k, v in configs.items()}, + descriptions={k: v[0] for k, v in configs.items()}, + ), + # Don't create subcommands for union types within the config object. + config=(tyro.conf.AvoidSubcommands,), + args=args, + ) + + def subcommand_type_from_defaults( defaults: Mapping[str, T], descriptions: Mapping[str, str] = {}, @@ -67,6 +126,8 @@ def subcommand_type_from_defaults( Returns: A subcommand type, which can be passed to :func:`tyro.cli`. """ + import tyro + # We need to form a union type, which requires at least two elements. assert len(defaults) >= 2, "At least two subcommands are required." return Union.__getitem__( # type: ignore @@ -74,7 +135,7 @@ def subcommand_type_from_defaults( Annotated.__class_getitem__( # type: ignore ( type(v), - subcommand( + tyro.conf.subcommand( k, default=v, description=descriptions.get(k, ""), diff --git a/tests/test_base_configs_nested.py b/tests/test_base_configs_nested.py index 7500df324..f9dca8f03 100644 --- a/tests/test_base_configs_nested.py +++ b/tests/test_base_configs_nested.py @@ -210,3 +210,23 @@ def test_pernicious_override(): ).data_config.test == 0 ) + + +def test_overridable_config_helper(): + assert tyro.extras.overridable_config_cli( + { + "small-data": ( + "Small data", + DataConfig( + test=2221, + ), + ), + "big-data": ( + "Big data", + DataConfig( + test=2, + ), + ), + }, + args=["small-data", "--test", "100"], + ) == DataConfig(100) From 2203ee65b67f2dc80e5be3a3581fb3334d676dfe Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 15 Aug 2024 14:12:47 -0700 Subject: [PATCH 433/491] Minor fix for `tyro.MISSING` docstring --- src/tyro/_fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 822a84f0f..7d2253404 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -201,8 +201,8 @@ class NotRequiredButWeDontKnowTheValueType(_singleton.Singleton): # Note that our "public" missing API will always be the propagating missing sentinel. MISSING: Any = MISSING_PROP -"""Sentinel value to mark fields as missing. Can be used to mark fields passed in as a -`default_instance` for `tyro.cli()` as required.""" +"""Sentinel value to mark fields as missing. Can be used to mark fields passed +in via `default=` for `tyro.cli()` as required.""" MISSING_SINGLETONS = [ From 42b9454590d5043b01a951d43f9e71f22380a261 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 15 Aug 2024 14:59:13 -0700 Subject: [PATCH 434/491] Show which subcommands unrecognized arguments are applied to (cc #152) --- src/tyro/_argparse_formatter.py | 27 +++++++++++++-------------- tests/test_errors.py | 25 +++++++++++++++++-------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index 90c724a1e..e64e9fda0 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -264,7 +264,7 @@ class _ArgumentInfo: # # Our current solution is to manually track unrecognized arguments in _parse_known_args, # and in error() override other errors when unrecognized arguments are present. -global_unrecognized_args: List[str] = [] +global_unrecognized_arg_and_prog: List[Tuple[str, str]] = [] # We inherit from both our local mirror of argparse and the upstream one. @@ -309,8 +309,8 @@ def _parse_known_args(self, arg_strings, namespace): # pragma: no cover # Reset the unused argument list in the root parser. # Subparsers will have spaces in self.prog. if " " not in self.prog: - global global_unrecognized_args - global_unrecognized_args = [] + global global_unrecognized_arg_and_prog + global_unrecognized_arg_and_prog = [] # # replace arg strings that are file references @@ -395,7 +395,9 @@ def consume_optional(start_index): # Manually track unused arguments to assist with error messages # later. if not self._parsing_known_args: - global_unrecognized_args.append(option_string) + global_unrecognized_arg_and_prog.append( + (option_string, self.prog) + ) # extras.append(arg_strings[start_index]) return start_index + 1 @@ -595,20 +597,14 @@ def error(self, message: str) -> NoReturn: """ extra_info: List[RenderableType] = [] - global global_unrecognized_args - if len(global_unrecognized_args) == 0 and message.startswith( - "unrecognized options: " - ): - global_unrecognized_args = message.partition(":")[2].strip().split(" ") - message_title = "Parsing error" - if len(global_unrecognized_args) > 0: + if len(global_unrecognized_arg_and_prog) > 0: message_title = "Unrecognized options" - message = f"Unrecognized options: {' '.join(global_unrecognized_args)}" + message = f"Unrecognized options: {' '.join([arg for arg, _ in global_unrecognized_arg_and_prog])}" unrecognized_arguments = set( arg - for arg in global_unrecognized_args + for arg, _ in global_unrecognized_arg_and_prog # If we pass in `--spell-chekc on`, we only want `spell-chekc` and not # `on`. if arg.startswith("--") @@ -621,7 +617,10 @@ def error(self, message: str) -> NoReturn: ) if has_subcommands and same_exists: - message = f"Unrecognized or misplaced options: {' '.join(global_unrecognized_args)}" + message = "Unrecognized or misplaced options:\n\n" + for arg, prog in global_unrecognized_arg_and_prog: + message += f" {arg} (applied to [green]{prog}[/green])\n" + message += "\nNote that arguments are applied to the directly preceding subcommand, so ordering matters." # Show similar arguments for keyword options. for unrecognized_argument in unrecognized_arguments: diff --git a/tests/test_errors.py b/tests/test_errors.py index b45e515a6..839947968 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -256,27 +256,36 @@ class ClassB: def test_similar_arguments_subcommands_multiple_contains_match() -> None: @dataclasses.dataclass - class RewardConfig: + class RewardConfigA: track: bool trace: int + @dataclasses.dataclass + class RewardConfigB: ... + @dataclasses.dataclass class ClassA: - reward: RewardConfig + reward: RewardConfigA @dataclasses.dataclass class ClassB: - reward: RewardConfig + reward: RewardConfigB target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stderr(target): - tyro.cli(Union[ClassA, ClassB], args="--rd.trac".split(" ")) # type: ignore + tyro.cli( + Union[ClassA, ClassB], + args="class-b --reward.track True --reward.trace 7".split(" "), + ) # type: ignore error = target.getvalue() - assert "Unrecognized option" in error - assert "Perhaps you meant:" in error - assert error.count("--reward.track {True,False}") == 1 - assert error.count("--reward.trace INT") == 1 + assert "Unrecognized or misplaced" in error + assert ( + "(applied to " in error and "class-b)" in error + ) # (applied to {root prog} class-b) + assert "Arguments similar to" in error + assert error.count("--reward.track {True,False}") == 2 + assert error.count("--reward.trace INT") == 2 assert error.count("--help") == 5 # 2 subcommands * 2 arguments + usage hint. From 0631bc238fe0d4d5a94bb19ea1a38fd3208726e2 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 26 Aug 2024 15:47:28 -0700 Subject: [PATCH 435/491] Add workaround for Python<=3.10 `get_type_hints()` bug (#157) * Add workaround for Python<=3.10 `get_type_hints()` bug * tuple => typing.Tuple --- src/tyro/_resolver.py | 37 +++++++++++- tests/test_optional_annotated.py | 57 +++++++++++++++++++ .../test_base_configs_nested_generated.py | 20 +++++++ .../test_conf_generated.py | 14 +++++ .../test_errors_generated.py | 25 +++++--- .../test_optional_annotated_generated.py | 55 ++++++++++++++++++ 6 files changed, 199 insertions(+), 9 deletions(-) create mode 100644 tests/test_optional_annotated.py create mode 100644 tests/test_py311_generated/test_optional_annotated_generated.py diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index ee16c9f9e..5bb8f82df 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -463,13 +463,48 @@ def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCalla return typ +NoneType = type(None) + + def get_type_hints_with_backported_syntax( obj: Callable[..., Any], include_extras: bool = False ) -> Dict[str, Any]: """Same as `typing.get_type_hints()`, but supports new union syntax (X | Y) and generics (list[str]) in older versions of Python.""" try: - return get_type_hints(obj, include_extras=include_extras) + out = get_type_hints(obj, include_extras=include_extras) + + # Workaround for: + # - https://github.com/brentyi/tyro/issues/156 + # - https://github.com/python/cpython/issues/90353 + # + # Which impacts Python 3.10 and earlier. + # + # It may be possible to remove this if this issue is resolved: + # - https://github.com/python/typing_extensions/issues/310 + if sys.version_info < (3, 11): + # If we see Optional[Annotated[T, ...]], we're going to flip to Annotated[Optional[T]]... + # + # It's unlikely but possible for this to have unintended side effects. + for k, v in out.items(): + origin = get_origin(v) + args = get_args(v) + if ( + origin is Union + and len(args) == 2 + and (args[0] is NoneType or args[1] is NoneType) + ): + non_none = args[1] if args[0] is NoneType else args[0] + if get_origin(non_none) is Annotated: + annotated_args = get_args(non_none) + out[k] = Annotated.__class_getitem__( # type: ignore + ( + Union.__getitem__((annotated_args[0], None)), # type: ignore + *annotated_args[1:], + ) + ) + return out + except TypeError as e: # pragma: no cover # Resolve new type syntax using eval_type_backport. if hasattr(obj, "__annotations__"): diff --git a/tests/test_optional_annotated.py b/tests/test_optional_annotated.py new file mode 100644 index 000000000..640c30695 --- /dev/null +++ b/tests/test_optional_annotated.py @@ -0,0 +1,57 @@ +"""Test adapted from @KolinGuo: https://github.com/brentyi/tyro/issues/156""" + +import dataclasses +from typing import Optional, Tuple + +from typing_extensions import Annotated + +import tyro + + +def detect_usb_device( + vendor_id: Annotated[Optional[str], tyro.conf.arg(aliases=["--vid"])] = None, + product_id: Annotated[Optional[str], tyro.conf.arg(aliases=["--pid"])] = "", +) -> Tuple[Optional[str], Optional[str]]: + """ + Detect connected USB device by vendor_id + product_id. + + :param vendor_id: vendor_id of a specific USB device. + :param product_id: product_id of a specific USB device. + """ + return vendor_id, product_id + + +def test_detect_usb_device() -> None: + assert tyro.cli(detect_usb_device, args=["--vid", "0x1234", "--pid", "0x5678"]) == ( + "0x1234", + "0x5678", + ) + assert tyro.cli( + detect_usb_device, args=["--vendor-id", "0x1234", "--product-id", "0x5678"] + ) == ( + "0x1234", + "0x5678", + ) + assert tyro.cli(detect_usb_device, args=[]) == (None, "") + + +@dataclasses.dataclass +class DeviceStruct: + vendor_id: Annotated[Optional[str], tyro.conf.arg(aliases=["--vid"])] = None + product_id: Annotated[Optional[str], tyro.conf.arg(aliases=["--pid"])] = "" + + +def test_detect_usb_device_dataclass() -> None: + assert tyro.cli( + DeviceStruct, args=["--vid", "0x1234", "--pid", "0x5678"] + ) == DeviceStruct( + "0x1234", + "0x5678", + ) + assert tyro.cli( + DeviceStruct, args=["--vendor-id", "0x1234", "--product-id", "0x5678"] + ) == DeviceStruct( + "0x1234", + "0x5678", + ) + assert tyro.cli(DeviceStruct, args=[]) == DeviceStruct(None, "") diff --git a/tests/test_py311_generated/test_base_configs_nested_generated.py b/tests/test_py311_generated/test_base_configs_nested_generated.py index 8134dbe09..9ce369220 100644 --- a/tests/test_py311_generated/test_base_configs_nested_generated.py +++ b/tests/test_py311_generated/test_base_configs_nested_generated.py @@ -209,3 +209,23 @@ def test_pernicious_override(): ).data_config.test == 0 ) + + +def test_overridable_config_helper(): + assert tyro.extras.overridable_config_cli( + { + "small-data": ( + "Small data", + DataConfig( + test=2221, + ), + ), + "big-data": ( + "Big data", + DataConfig( + test=2, + ), + ), + }, + args=["small-data", "--test", "100"], + ) == DataConfig(100) diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index b58ad8677..b038ba298 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -450,6 +450,12 @@ class A: args=["--x", "True"], default=A(False), # type: ignore ) == A(True) + assert tyro.cli( + A, + args=["--x", "True"], + default=A(False), + config=(tyro.conf.FlagConversionOff,), + ) == A(True) def test_fixed() -> None: @@ -474,6 +480,14 @@ class A: default=A(False), # type: ignore ) == A(True) + with pytest.raises(SystemExit): + assert tyro.cli( + A, + args=["--x", "True"], + default=A(False), # type: ignore + config=(tyro.conf.FlagConversionOff,), + ) == A(True) + def test_fixed_recursive() -> None: """When an argument is fixed, we shouldn't be able to override it from the CLI.""" diff --git a/tests/test_py311_generated/test_errors_generated.py b/tests/test_py311_generated/test_errors_generated.py index bb67c2d78..b7a9f5cfc 100644 --- a/tests/test_py311_generated/test_errors_generated.py +++ b/tests/test_py311_generated/test_errors_generated.py @@ -255,27 +255,36 @@ class ClassB: def test_similar_arguments_subcommands_multiple_contains_match() -> None: @dataclasses.dataclass - class RewardConfig: + class RewardConfigA: track: bool trace: int + @dataclasses.dataclass + class RewardConfigB: ... + @dataclasses.dataclass class ClassA: - reward: RewardConfig + reward: RewardConfigA @dataclasses.dataclass class ClassB: - reward: RewardConfig + reward: RewardConfigB target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stderr(target): - tyro.cli(ClassA | ClassB, args="--rd.trac".split(" ")) # type: ignore + tyro.cli( + ClassA | ClassB, + args="class-b --reward.track True --reward.trace 7".split(" "), + ) # type: ignore error = target.getvalue() - assert "Unrecognized option" in error - assert "Perhaps you meant:" in error - assert error.count("--reward.track {True,False}") == 1 - assert error.count("--reward.trace INT") == 1 + assert "Unrecognized or misplaced" in error + assert ( + "(applied to " in error and "class-b)" in error + ) # (applied to {root prog} class-b) + assert "Arguments similar to" in error + assert error.count("--reward.track {True,False}") == 2 + assert error.count("--reward.trace INT") == 2 assert error.count("--help") == 5 # 2 subcommands * 2 arguments + usage hint. diff --git a/tests/test_py311_generated/test_optional_annotated_generated.py b/tests/test_py311_generated/test_optional_annotated_generated.py new file mode 100644 index 000000000..751e612e6 --- /dev/null +++ b/tests/test_py311_generated/test_optional_annotated_generated.py @@ -0,0 +1,55 @@ +"""Test adapted from @KolinGuo: https://github.com/brentyi/tyro/issues/156""" + +import dataclasses +from typing import Annotated, Optional, Tuple + +import tyro + + +def detect_usb_device( + vendor_id: Annotated[Optional[str], tyro.conf.arg(aliases=["--vid"])] = None, + product_id: Annotated[Optional[str], tyro.conf.arg(aliases=["--pid"])] = "", +) -> Tuple[Optional[str], Optional[str]]: + """ + Detect connected USB device by vendor_id + product_id. + + :param vendor_id: vendor_id of a specific USB device. + :param product_id: product_id of a specific USB device. + """ + return vendor_id, product_id + + +def test_detect_usb_device() -> None: + assert tyro.cli(detect_usb_device, args=["--vid", "0x1234", "--pid", "0x5678"]) == ( + "0x1234", + "0x5678", + ) + assert tyro.cli( + detect_usb_device, args=["--vendor-id", "0x1234", "--product-id", "0x5678"] + ) == ( + "0x1234", + "0x5678", + ) + assert tyro.cli(detect_usb_device, args=[]) == (None, "") + + +@dataclasses.dataclass +class DeviceStruct: + vendor_id: Annotated[Optional[str], tyro.conf.arg(aliases=["--vid"])] = None + product_id: Annotated[Optional[str], tyro.conf.arg(aliases=["--pid"])] = "" + + +def test_detect_usb_device_dataclass() -> None: + assert tyro.cli( + DeviceStruct, args=["--vid", "0x1234", "--pid", "0x5678"] + ) == DeviceStruct( + "0x1234", + "0x5678", + ) + assert tyro.cli( + DeviceStruct, args=["--vendor-id", "0x1234", "--product-id", "0x5678"] + ) == DeviceStruct( + "0x1234", + "0x5678", + ) + assert tyro.cli(DeviceStruct, args=[]) == DeviceStruct(None, "") From e6a4cfb1936bc6ecdfdf63fe7e7e56a6d8c8b898 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 26 Aug 2024 15:52:30 -0700 Subject: [PATCH 436/491] Add support for `datetime.{date,time,datetime}` types (#155) * Add support for `datetime.{date,time,datetime}` types * Refine error message, metavar, tests * Cleanup * Test cleanup * ruff / mypy * mypy * mypy --- src/tyro/_calling.py | 2 +- src/tyro/_instantiators.py | 49 +++++++++ tests/test_dcargs.py | 101 ++++++++++++++++++ tests/test_missing_optional_packages.py | 5 +- .../test_dcargs_generated.py | 101 ++++++++++++++++++ ...est_missing_optional_packages_generated.py | 5 +- 6 files changed, 258 insertions(+), 5 deletions(-) diff --git a/src/tyro/_calling.py b/src/tyro/_calling.py index 16508e86a..632b5bc18 100644 --- a/src/tyro/_calling.py +++ b/src/tyro/_calling.py @@ -107,7 +107,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: try: assert arg.lowered.instantiator is not None value = arg.lowered.instantiator(value) - except ValueError as e: + except (ValueError, TypeError) as e: raise InstantiationError( e.args[0], arg, diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index dad0d30b1..a255c3b90 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -33,6 +33,7 @@ import collections.abc import dataclasses +import datetime import enum import inspect import os @@ -224,6 +225,52 @@ def instantiator(strings: List[str]) -> None: " a valid type converter." ) + # Use ISO 8601 standard for dates/times. + if typ in (datetime.datetime, datetime.date, datetime.time): + + def instantiate(args: List[str]) -> Any: + (arg,) = args + try: + # Type ignore is unnecessary for pyright but needed for mypy. + return typ.fromisoformat(arg) # type: ignore + except ValueError: + raise ValueError( + f"`{typ.__name__}.fromisoformat('{arg}')` failed. Dates " + "should be specified in ISO-8601 format: " + "https://en.wikipedia.org/wiki/ISO_8601" + ) + + return ( + instantiate, + InstantiatorMetadata( + nargs=1, + metavar={ + # Actually we take any ISO 8601 string here. So these + # metavars are a bit incomplete. + # datetime.datetime: "YYYY-MM-DD[THH:MM:SS]", + # datetime.date: "YYYY-MM-DD", + # datetime.time: "HH:MM[:SS]", + # + # More complete. + datetime.datetime: "YYYY-MM-DD[THH:MM:[SS[…]]]", + datetime.date: "YYYY-MM-DD", + datetime.time: "HH:MM[:SS[…]]", + # + # Even more complete! + # datetime.datetime: "YYYY-MM-DD[THH:MM[:SS[.fff]][±HH:MM|Z]]", + # datetime.date: "YYYY-MM-DD", + # datetime.time: "HH:MM[:SS[.fff]][±HH:MM|Z]", + # + # Not very informative but (1) precise and (2) succinct. + # datetime.datetime: "ISO-DATETIME", + # datetime.date: "ISO-DATE", + # datetime.time: "ISO-TIME", + }[cast(Any, typ)], # cast is for mypy, pyright works fine + choices=None, + action=None, + ), + ) + # Special case `choices` for some types, as implemented in `instance_from_string()`. auto_choices: Optional[Tuple[str, ...]] = None if typ is bool: @@ -256,6 +303,8 @@ def instantiator_base_case(strings: List[str]) -> Any: elif typ is bytes: return bytes(string, encoding="ascii") # type: ignore else: + # We assume this base type can be called on a string to convert + # from a string, eg int("5"), float("5."), Path("/home"), etc. return typ(string) # type: ignore return instantiator_base_case, InstantiatorMetadata( diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index c0a6c640e..842151cd8 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -1,6 +1,7 @@ import argparse import copy import dataclasses +import datetime import enum import os import pathlib @@ -749,3 +750,103 @@ class A: ) assert a == A(a_b=[7], c_d=[7]) assert unknown_args == ["--e-f", "--e-f", "--g_h", "--g_h"] + + +def test_datetime_parsing() -> None: + assert tyro.cli( + datetime.datetime, args=["2024-08-25T14:30:00"] + ) == datetime.datetime(2024, 8, 25, 14, 30, 0) + assert tyro.cli( + datetime.datetime, args=["2024-08-25T14:30:00.123456"] + ) == datetime.datetime(2024, 8, 25, 14, 30, 0, 123456) + assert tyro.cli( + datetime.datetime, args=["2024-08-25T14:30:00+05:30"] + ) == datetime.datetime( + 2024, 8, 25, 14, 30, tzinfo=datetime.timezone(datetime.timedelta(seconds=19800)) + ) + + +def test_date_parsing() -> None: + assert tyro.cli(datetime.date, args=["2024-08-25"]) == datetime.date(2024, 8, 25) + assert tyro.cli(datetime.date, args=["1999-12-31"]) == datetime.date(1999, 12, 31) + + +def test_time_parsing() -> None: + assert tyro.cli(datetime.time, args=["14:30:00"]) == datetime.time(14, 30, 0) + assert tyro.cli(datetime.time, args=["14:30:00.123456"]) == datetime.time( + 14, 30, 0, 123456 + ) + assert tyro.cli(datetime.time, args=["14:30:00+05:30"]) == datetime.time( + 14, 30, tzinfo=datetime.timezone(datetime.timedelta(seconds=19800)) + ) + + +def test_datetime_parsing_with_main(): + def main(dt: datetime.datetime) -> datetime.datetime: + return dt + + assert tyro.cli(main, args=["--dt", "2024-08-25T14:30:00"]) == datetime.datetime( + 2024, 8, 25, 14, 30, 0 + ) + assert tyro.cli( + main, args=["--dt", "2024-08-25T14:30:00.123456"] + ) == datetime.datetime(2024, 8, 25, 14, 30, 0, 123456) + assert tyro.cli( + main, args=["--dt", "2024-08-25T14:30:00+05:30"] + ) == datetime.datetime( + 2024, 8, 25, 14, 30, tzinfo=datetime.timezone(datetime.timedelta(seconds=19800)) + ) + + +def test_date_parsing_with_main(): + def main(dt: datetime.date) -> datetime.date: + return dt + + assert tyro.cli(main, args=["--dt", "2024-08-25"]) == datetime.date(2024, 8, 25) + assert tyro.cli(main, args=["--dt", "1999-12-31"]) == datetime.date(1999, 12, 31) + + +def test_time_parsing_with_main(): + def main(dt: datetime.time) -> datetime.time: + return dt + + assert tyro.cli(main, args=["--dt", "14:30:00"]) == datetime.time(14, 30, 0) + assert tyro.cli(main, args=["--dt", "14:30:00.123456"]) == datetime.time( + 14, 30, 0, 123456 + ) + assert tyro.cli(main, args=["--dt", "14:30:00+05:30"]) == datetime.time( + 14, 30, tzinfo=datetime.timezone(datetime.timedelta(seconds=19800)) + ) + + +def test_date_parsing_harder_format(): + def main(dt: datetime.date) -> datetime.date: + return dt + + # Wrong separator '/'. + with pytest.raises(SystemExit): + tyro.cli(main, args=["--dt", "2024/08/25"]) + + # Day-Month-Year format instead of Year-Month-Day. + with pytest.raises(SystemExit): + tyro.cli(main, args=["--dt", "25-08-2024"]) + + +def test_time_parsing_harder_format(): + def main(dt: datetime.time) -> datetime.time: + return dt + + # Wrong separator '/'. + with pytest.raises(SystemExit): + tyro.cli(main, args=["--dt", "14/30"]) + + # Missing seconds. + assert tyro.cli(main, args=["--dt", "14:30"]) == datetime.time(14, 30, 0) + + # Invalid minute value. + with pytest.raises(SystemExit): + tyro.cli(main, args=["--dt", "14:60:00"]) + + # Invalid hour value. + with pytest.raises(SystemExit): + tyro.cli(main, args=["--dt", "25:00:00"]) diff --git a/tests/test_missing_optional_packages.py b/tests/test_missing_optional_packages.py index 1e3afcdec..084e6e7f9 100644 --- a/tests/test_missing_optional_packages.py +++ b/tests/test_missing_optional_packages.py @@ -1,12 +1,13 @@ import builtins import dataclasses +from typing import Any import pytest # https://stackoverflow.com/questions/60227582/making-a-python-test-think-an-installed-package-is-not-available @pytest.fixture -def hide_optional_packages(monkeypatch): +def hide_optional_packages(monkeypatch: Any) -> None: import_orig = builtins.__import__ def mocked_import(name, *args, **kwargs): @@ -17,7 +18,7 @@ def mocked_import(name, *args, **kwargs): monkeypatch.setattr(builtins, "__import__", mocked_import) -def test_missing_optional_packages(hide_optional_packages): +def test_missing_optional_packages(hide_optional_packages: Any) -> None: @dataclasses.dataclass class Extra: a: int = 3 diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py index fadcf18ed..fa8047a8a 100644 --- a/tests/test_py311_generated/test_dcargs_generated.py +++ b/tests/test_py311_generated/test_dcargs_generated.py @@ -1,6 +1,7 @@ import argparse import copy import dataclasses +import datetime import enum import os import pathlib @@ -751,3 +752,103 @@ class A: ) assert a == A(a_b=[7], c_d=[7]) assert unknown_args == ["--e-f", "--e-f", "--g_h", "--g_h"] + + +def test_datetime_parsing() -> None: + assert tyro.cli( + datetime.datetime, args=["2024-08-25T14:30:00"] + ) == datetime.datetime(2024, 8, 25, 14, 30, 0) + assert tyro.cli( + datetime.datetime, args=["2024-08-25T14:30:00.123456"] + ) == datetime.datetime(2024, 8, 25, 14, 30, 0, 123456) + assert tyro.cli( + datetime.datetime, args=["2024-08-25T14:30:00+05:30"] + ) == datetime.datetime( + 2024, 8, 25, 14, 30, tzinfo=datetime.timezone(datetime.timedelta(seconds=19800)) + ) + + +def test_date_parsing() -> None: + assert tyro.cli(datetime.date, args=["2024-08-25"]) == datetime.date(2024, 8, 25) + assert tyro.cli(datetime.date, args=["1999-12-31"]) == datetime.date(1999, 12, 31) + + +def test_time_parsing() -> None: + assert tyro.cli(datetime.time, args=["14:30:00"]) == datetime.time(14, 30, 0) + assert tyro.cli(datetime.time, args=["14:30:00.123456"]) == datetime.time( + 14, 30, 0, 123456 + ) + assert tyro.cli(datetime.time, args=["14:30:00+05:30"]) == datetime.time( + 14, 30, tzinfo=datetime.timezone(datetime.timedelta(seconds=19800)) + ) + + +def test_datetime_parsing_with_main(): + def main(dt: datetime.datetime) -> datetime.datetime: + return dt + + assert tyro.cli(main, args=["--dt", "2024-08-25T14:30:00"]) == datetime.datetime( + 2024, 8, 25, 14, 30, 0 + ) + assert tyro.cli( + main, args=["--dt", "2024-08-25T14:30:00.123456"] + ) == datetime.datetime(2024, 8, 25, 14, 30, 0, 123456) + assert tyro.cli( + main, args=["--dt", "2024-08-25T14:30:00+05:30"] + ) == datetime.datetime( + 2024, 8, 25, 14, 30, tzinfo=datetime.timezone(datetime.timedelta(seconds=19800)) + ) + + +def test_date_parsing_with_main(): + def main(dt: datetime.date) -> datetime.date: + return dt + + assert tyro.cli(main, args=["--dt", "2024-08-25"]) == datetime.date(2024, 8, 25) + assert tyro.cli(main, args=["--dt", "1999-12-31"]) == datetime.date(1999, 12, 31) + + +def test_time_parsing_with_main(): + def main(dt: datetime.time) -> datetime.time: + return dt + + assert tyro.cli(main, args=["--dt", "14:30:00"]) == datetime.time(14, 30, 0) + assert tyro.cli(main, args=["--dt", "14:30:00.123456"]) == datetime.time( + 14, 30, 0, 123456 + ) + assert tyro.cli(main, args=["--dt", "14:30:00+05:30"]) == datetime.time( + 14, 30, tzinfo=datetime.timezone(datetime.timedelta(seconds=19800)) + ) + + +def test_date_parsing_harder_format(): + def main(dt: datetime.date) -> datetime.date: + return dt + + # Wrong separator '/'. + with pytest.raises(SystemExit): + tyro.cli(main, args=["--dt", "2024/08/25"]) + + # Day-Month-Year format instead of Year-Month-Day. + with pytest.raises(SystemExit): + tyro.cli(main, args=["--dt", "25-08-2024"]) + + +def test_time_parsing_harder_format(): + def main(dt: datetime.time) -> datetime.time: + return dt + + # Wrong separator '/'. + with pytest.raises(SystemExit): + tyro.cli(main, args=["--dt", "14/30"]) + + # Missing seconds. + assert tyro.cli(main, args=["--dt", "14:30"]) == datetime.time(14, 30, 0) + + # Invalid minute value. + with pytest.raises(SystemExit): + tyro.cli(main, args=["--dt", "14:60:00"]) + + # Invalid hour value. + with pytest.raises(SystemExit): + tyro.cli(main, args=["--dt", "25:00:00"]) diff --git a/tests/test_py311_generated/test_missing_optional_packages_generated.py b/tests/test_py311_generated/test_missing_optional_packages_generated.py index 1e3afcdec..084e6e7f9 100644 --- a/tests/test_py311_generated/test_missing_optional_packages_generated.py +++ b/tests/test_py311_generated/test_missing_optional_packages_generated.py @@ -1,12 +1,13 @@ import builtins import dataclasses +from typing import Any import pytest # https://stackoverflow.com/questions/60227582/making-a-python-test-think-an-installed-package-is-not-available @pytest.fixture -def hide_optional_packages(monkeypatch): +def hide_optional_packages(monkeypatch: Any) -> None: import_orig = builtins.__import__ def mocked_import(name, *args, **kwargs): @@ -17,7 +18,7 @@ def mocked_import(name, *args, **kwargs): monkeypatch.setattr(builtins, "__import__", mocked_import) -def test_missing_optional_packages(hide_optional_packages): +def test_missing_optional_packages(hide_optional_packages: Any) -> None: @dataclasses.dataclass class Extra: a: int = 3 From 8648be5c04111992db061741d34fa906090a7ca5 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 26 Aug 2024 15:53:51 -0700 Subject: [PATCH 437/491] Bump version --- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- src/tyro/_argparse_formatter.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 821dcaf3a..5a6e555b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.8" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.9" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index e7bafa092..95ec1686a 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.8" +__version__ = "0.8.9" diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index e64e9fda0..1eb6a13e8 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -620,7 +620,7 @@ def error(self, message: str) -> NoReturn: message = "Unrecognized or misplaced options:\n\n" for arg, prog in global_unrecognized_arg_and_prog: message += f" {arg} (applied to [green]{prog}[/green])\n" - message += "\nNote that arguments are applied to the directly preceding subcommand, so ordering matters." + message += "\nArguments are applied to the directly preceding subcommand, so ordering matters." # Show similar arguments for keyword options. for unrecognized_argument in unrecognized_arguments: From 07f1e95891649c124d28cc1b352e58b3ee5410af Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 26 Aug 2024 19:37:05 -0700 Subject: [PATCH 438/491] Merge union choices when possible This should only impact completion script generation (#158) --- src/tyro/_instantiators.py | 7 ++++++- tests/test_completion.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index a255c3b90..e10bd9807 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -511,6 +511,7 @@ def _instantiator_from_union( # right. instantiators = [] metas = [] + choices: Tuple[str, ...] | None = () nargs: Optional[Union[int, Literal["*"]]] = 1 first = True for t in options: @@ -519,6 +520,10 @@ def _instantiator_from_union( ) instantiators.append(a) metas.append(b) + if b.choices is None: + choices = None + elif choices is not None: + choices = choices + b.choices if t is not NoneType: # Enforce that `nargs` is the same for all child types, except for @@ -566,7 +571,7 @@ def union_instantiator(strings: List[str]) -> Any: return union_instantiator, InstantiatorMetadata( nargs=nargs, metavar=metavar, - choices=None, + choices=None if choices is None else tuple(set(choices)), action=None, ) diff --git a/tests/test_completion.py b/tests/test_completion.py index e1d6a9419..82bff4239 100644 --- a/tests/test_completion.py +++ b/tests/test_completion.py @@ -4,6 +4,7 @@ from typing import Union import pytest +from typing_extensions import Annotated, Literal, Optional import tyro @@ -41,3 +42,35 @@ def test_zsh(): with pytest.raises(SystemExit), contextlib.redirect_stdout(target): tyro.cli(Wrapper, args=["--tyro-print-completion", "zsh"]) assert "# AUTOMATICALLY GENERATED by `shtab`" in target.getvalue() + + +def test_completion_zsh(): + """https://github.com/brentyi/tyro/issues/158""" + + def start_device( + preset: Annotated[ + Optional[Literal["rgb", "depth", "ir"]], tyro.conf.arg(aliases=["-p"]) + ] = None, + frame: Annotated[ + Literal["world", "base"], tyro.conf.arg(aliases=["-f"]) + ] = "world", + ) -> None: + """ + Start device with the given preset. + + :param preset: device preset to use. + :param frame: coordinate frame to use. + """ + print(f"{preset=} {frame=}") + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(start_device, args=["--tyro-print-completion", "bash"]) + + completion_script = target.getvalue() + print(completion_script) + assert "# AUTOMATICALLY GENERATED by `shtab`" in completion_script + assert "preset_choices=(" in completion_script + assert "p_choices=(" in completion_script + assert "frame_choices=(" in completion_script + assert "f_choices=(" in completion_script From 7b32fa3142453bff31f01503fd828e6fe600ff46 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 26 Aug 2024 19:41:51 -0700 Subject: [PATCH 439/491] Fix completion script test for Python 3.7 --- tests/test_completion.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_completion.py b/tests/test_completion.py index 82bff4239..44d9e741e 100644 --- a/tests/test_completion.py +++ b/tests/test_completion.py @@ -61,7 +61,6 @@ def start_device( :param preset: device preset to use. :param frame: coordinate frame to use. """ - print(f"{preset=} {frame=}") target = io.StringIO() with pytest.raises(SystemExit), contextlib.redirect_stdout(target): From 1e68768aea5e965233c53a10bb716c510d587ebc Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 26 Aug 2024 20:23:41 -0700 Subject: [PATCH 440/491] Bump version --- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5a6e555b5..b71d6cd52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.9" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.10" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index 95ec1686a..594d61c7f 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.9" +__version__ = "0.8.10" From 6ab907fde31721aa3129bada39822481f2e44ac7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 29 Aug 2024 07:05:23 -0700 Subject: [PATCH 441/491] `__builtins__` => `builtins` --- src/tyro/_docstrings.py | 6 +++++- src/tyro/_fields.py | 3 ++- src/tyro/_instantiators.py | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/tyro/_docstrings.py b/src/tyro/_docstrings.py index 102d42f50..3d461927b 100644 --- a/src/tyro/_docstrings.py +++ b/src/tyro/_docstrings.py @@ -1,5 +1,6 @@ """Helpers for parsing docstrings. Used for helptext generation.""" +import builtins import collections.abc import dataclasses import functools @@ -284,7 +285,10 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: _callable_description_blocklist: Set[Hashable] = set( filter( lambda x: isinstance(x, Hashable), # type: ignore - itertools.chain(__builtins__.values(), vars(collections.abc).values()), # type: ignore + itertools.chain( + vars(builtins).values(), + vars(collections.abc).values(), + ), ) ) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 7d2253404..931a09827 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -3,6 +3,7 @@ from __future__ import annotations +import builtins import collections import collections.abc import dataclasses @@ -353,7 +354,7 @@ def resolve(field: FieldDefinition) -> FieldDefinition: filter( lambda x: isinstance(x, Hashable), # type: ignore itertools.chain( - __builtins__.values(), # type: ignore + vars(builtins).values(), # type: ignore vars(typing).values(), vars(typing_extensions).values(), vars(collections.abc).values(), diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index e10bd9807..9397bda84 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -31,6 +31,7 @@ ``` """ +import builtins import collections.abc import dataclasses import datetime @@ -106,7 +107,7 @@ class UnsupportedTypeAnnotationError(Exception): _builtin_set: Set[Hashable] = set( filter( lambda x: isinstance(x, Hashable), # type: ignore - __builtins__.values(), # type: ignore + vars(builtins).values(), # type: ignore ) ) From cdb025408f0a0e573df1943bc31ba2dcb8cf83ae Mon Sep 17 00:00:00 2001 From: brentyi Date: Thu, 5 Sep 2024 20:27:54 -0700 Subject: [PATCH 442/491] Minor test, readme updates --- README.md | 12 ++++--- tests/test_dcargs.py | 11 +++++++ .../test_completion_generated.py | 32 +++++++++++++++++++ .../test_dcargs_generated.py | 11 +++++++ 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c6948d13e..ae307195a 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,10 @@
-tyro.cli() generates command-line interfaces via -Python type introspection. We can define configurable scripts using functions: +tyro.cli() is a tool for generating CLI +interfaces. + +We can define configurable scripts using functions: ```python """A command-line interface defined using a function signature. @@ -60,7 +62,7 @@ if __name__ == "__main__": tyro.cli(main) ``` -Or instantiate configuration objects defined using tools like `dataclasses`, `pydantic`, and `attrs`: +Or instantiate config objects defined using tools like `dataclasses`, `pydantic`, and `attrs`: ```python """A command-line interface defined using a class signature. @@ -84,8 +86,8 @@ if __name__ == "__main__": assert isinstance(config, Config) # Should pass. ``` -Other features include helptext generation, nested structures, shell -completion, and subcommands. For examples and the API reference, see our +Other features include helptext generation, nested structures, subcommands, and +shell completion. For examples and the API reference, see our [documentation](https://brentyi.github.io/tyro). ### In the wild diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 842151cd8..da76c273b 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -307,6 +307,17 @@ class A: assert tyro.cli(A, args=["--x", "3"]) +def test_literal_none() -> None: + @dataclasses.dataclass + class A: + x: Literal[0, 1, None, 2] + + assert tyro.cli(A, args=["--x", "1"]) == A(x=1) + assert tyro.cli(A, args=["--x", "None"]) == A(x=None) + with pytest.raises(SystemExit): + assert tyro.cli(A, args=["--x", "3"]) + + Choices = int Choices = tyro.extras.literal_type_from_choices([0, 1, 2]) # type: ignore diff --git a/tests/test_py311_generated/test_completion_generated.py b/tests/test_py311_generated/test_completion_generated.py index 41527d9cd..91f0e7fa9 100644 --- a/tests/test_py311_generated/test_completion_generated.py +++ b/tests/test_py311_generated/test_completion_generated.py @@ -1,6 +1,7 @@ import contextlib import dataclasses import io +from typing import Annotated, Literal, Optional import pytest @@ -40,3 +41,34 @@ def test_zsh(): with pytest.raises(SystemExit), contextlib.redirect_stdout(target): tyro.cli(Wrapper, args=["--tyro-print-completion", "zsh"]) assert "# AUTOMATICALLY GENERATED by `shtab`" in target.getvalue() + + +def test_completion_zsh(): + """https://github.com/brentyi/tyro/issues/158""" + + def start_device( + preset: Annotated[ + Optional[Literal["rgb", "depth", "ir"]], tyro.conf.arg(aliases=["-p"]) + ] = None, + frame: Annotated[ + Literal["world", "base"], tyro.conf.arg(aliases=["-f"]) + ] = "world", + ) -> None: + """ + Start device with the given preset. + + :param preset: device preset to use. + :param frame: coordinate frame to use. + """ + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli(start_device, args=["--tyro-print-completion", "bash"]) + + completion_script = target.getvalue() + print(completion_script) + assert "# AUTOMATICALLY GENERATED by `shtab`" in completion_script + assert "preset_choices=(" in completion_script + assert "p_choices=(" in completion_script + assert "frame_choices=(" in completion_script + assert "f_choices=(" in completion_script diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py index fa8047a8a..73aa9a9ef 100644 --- a/tests/test_py311_generated/test_dcargs_generated.py +++ b/tests/test_py311_generated/test_dcargs_generated.py @@ -309,6 +309,17 @@ class A: assert tyro.cli(A, args=["--x", "3"]) +def test_literal_none() -> None: + @dataclasses.dataclass + class A: + x: Literal[0, 1, None, 2] + + assert tyro.cli(A, args=["--x", "1"]) == A(x=1) + assert tyro.cli(A, args=["--x", "None"]) == A(x=None) + with pytest.raises(SystemExit): + assert tyro.cli(A, args=["--x", "3"]) + + Choices = int Choices = tyro.extras.literal_type_from_choices([0, 1, 2]) # type: ignore From fcadb01918c4ad419ce5edd10057bfdeee5b837a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 6 Sep 2024 23:15:52 -0700 Subject: [PATCH 443/491] Exclude pyright version with `Annotated` regression Signed-off-by: Brent Yi --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b71d6cd52..fb6709c56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dev = [ "omegaconf>=2.2.2", "attrs>=21.4.0", "torch>=1.10.0", - "pyright>=1.1.349", + "pyright>=1.1.349,!=1.1.379", "ruff>=0.1.13", "mypy>=1.4.1", "numpy>=1.20.0", From 072dd5f4bdcb88171a85c98088331ed74e99557f Mon Sep 17 00:00:00 2001 From: brentyi Date: Sat, 14 Sep 2024 01:13:49 -0700 Subject: [PATCH 444/491] Accomodate pyright updates --- src/tyro/_argparse_formatter.py | 2 +- src/tyro/_resolver.py | 2 +- tests/test_conf.py | 4 ++-- tests/test_py311_generated/test_conf_generated.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index 1eb6a13e8..69753e4f0 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -138,7 +138,7 @@ def _recursive_arg_search( option_strings = arg.lowered.action( option_strings, dest="", # dest should not matter. - ).option_strings + ).option_strings # type: ignore arguments.append( _ArgumentInfo( diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 5bb8f82df..46b610ddc 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -287,7 +287,7 @@ def narrow_collection_types( if len(default_instance) == 0: return typ typ = Tuple.__getitem__(tuple(map(type, default_instance))) # type: ignore - return typ + return cast(TypeOrCallable, typ) # `Final` and `ReadOnly` types are ignored in tyro. diff --git a/tests/test_conf.py b/tests/test_conf.py index c51e9d4cd..6b72da9af 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1222,8 +1222,8 @@ def commit(branch: str) -> int: return 3 assert ( - tyro.cli( # type: ignore - Annotated[Any, tyro.conf.arg(constructor=commit)], + tyro.cli( + Annotated[Any, tyro.conf.arg(constructor=commit)], # type: ignore args="--branch 5".split(" "), ) == 3 diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index b038ba298..980fd79b6 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -1219,8 +1219,8 @@ def commit(branch: str) -> int: return 3 assert ( - tyro.cli( # type: ignore - Annotated[Any, tyro.conf.arg(constructor=commit)], + tyro.cli( + Annotated[Any, tyro.conf.arg(constructor=commit)], # type: ignore args="--branch 5".split(" "), ) == 3 From bf3f61a2ef53582852a8f69953e7e2c476859521 Mon Sep 17 00:00:00 2001 From: Kevin Chen <79341521+kevinddchen@users.noreply.github.com> Date: Sat, 14 Sep 2024 01:22:33 -0700 Subject: [PATCH 445/491] Support Enum aliases (#162) * populate enum aliases * add test * autogenerated test * Update test_dcargs.py Signed-off-by: Brent Yi * Update test_dcargs_generated.py Signed-off-by: Brent Yi --------- Signed-off-by: Brent Yi Co-authored-by: Brent Yi --- src/tyro/_instantiators.py | 3 ++- tests/test_dcargs.py | 14 ++++++++++++++ .../test_py311_generated/test_dcargs_generated.py | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index 9397bda84..c5b2f50d8 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -277,7 +277,8 @@ def instantiate(args: List[str]) -> Any: if typ is bool: auto_choices = ("True", "False") elif inspect.isclass(typ) and issubclass(typ, enum.Enum): - auto_choices = tuple(x.name for x in typ) + # Using `.__members__` dict to handle aliases correctly + auto_choices = tuple(typ.__members__.keys()) def instantiator_base_case(strings: List[str]) -> Any: """Given a type and and a string from the command-line, reconstruct an object. Not diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index da76c273b..723733a1d 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -297,6 +297,20 @@ class EnumClassB: assert tyro.cli(EnumClassB, args=[]) == EnumClassB() +def test_enum_alias() -> None: + class Color(enum.Enum): + RED = 1 + ROUGE = 1 + + @dataclasses.dataclass + class A: + color: Color + + assert tyro.cli(A, args=["--color", "RED"]) == A(color=Color.RED) + assert tyro.cli(A, args=["--color", "ROUGE"]) == A(color=Color.ROUGE) + assert tyro.cli(A, args=["--color", "ROUGE"]) == A(color=Color.RED) + + def test_literal() -> None: @dataclasses.dataclass class A: diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py index 73aa9a9ef..b8836572b 100644 --- a/tests/test_py311_generated/test_dcargs_generated.py +++ b/tests/test_py311_generated/test_dcargs_generated.py @@ -299,6 +299,20 @@ class EnumClassB: assert tyro.cli(EnumClassB, args=[]) == EnumClassB() +def test_enum_alias() -> None: + class Color(enum.Enum): + RED = 1 + ROUGE = 1 + + @dataclasses.dataclass + class A: + color: Color + + assert tyro.cli(A, args=["--color", "RED"]) == A(color=Color.RED) + assert tyro.cli(A, args=["--color", "ROUGE"]) == A(color=Color.ROUGE) + assert tyro.cli(A, args=["--color", "ROUGE"]) == A(color=Color.RED) + + def test_literal() -> None: @dataclasses.dataclass class A: From 3b436a73b261095147291e263b7876cab496f755 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 18 Sep 2024 10:10:31 -0700 Subject: [PATCH 446/491] Support overriding behavior hint in argument helptext (#161) * Support overriding behavior hint in argument helptext * Handle empty string for `help_behavior_hint` * Support lambdai nput for `help_behavior_hint=` --- src/tyro/_arguments.py | 35 ++++++++------ src/tyro/_fields.py | 3 +- src/tyro/conf/_confstruct.py | 93 ++++++++++++++++++++---------------- tests/test_conf.py | 41 +++++++++++++++- 4 files changed, 113 insertions(+), 59 deletions(-) diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index 2779131c2..11ab223e5 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -442,7 +442,7 @@ def _rule_generate_helptext( [arg.extern_prefix, arg.field.intern_name] ) - if primary_help is not None and primary_help != "": + if primary_help is not None: help_parts.append(_rich_tag_if_enabled(primary_help, "helptext")) if not lowered.required: @@ -474,26 +474,31 @@ def _rule_generate_helptext( else: default_label = str(default) - # Include default value in helptext. We intentionally don't use the % template - # because the types of all arguments are set to strings, which will cause the - # default to be casted to a string and introduce extra quotation marks. - if lowered.instantiator is None: + # Suffix helptext with some behavior hint, such as the default value of the argument. + help_behavior_hint = arg.field.argconf.help_behavior_hint + if help_behavior_hint is not None: + behavior_hint = ( + help_behavior_hint(default_label) + if callable(help_behavior_hint) + else help_behavior_hint + ) + elif lowered.instantiator is None: # Intentionally not quoted via shlex, since this can't actually be passed # in via the commandline. - default_text = f"(fixed to: {default_label})" + behavior_hint = f"(fixed to: {default_label})" elif lowered.action == "count": # Repeatable argument. - default_text = "(repeatable)" + behavior_hint = "(repeatable)" elif lowered.action == "append" and ( default in _fields.MISSING_SINGLETONS or len(cast(tuple, default)) == 0 ): - default_text = "(repeatable)" + behavior_hint = "(repeatable)" elif lowered.action == "append" and len(cast(tuple, default)) > 0: assert default is not None # Just for type checker. - default_text = f"(repeatable, appends to: {default_label})" + behavior_hint = f"(repeatable, appends to: {default_label})" elif arg.field.default is _fields.EXCLUDE_FROM_CALL: # ^important to use arg.field.default and not the stringified default variable. - default_text = "(unset by default)" + behavior_hint = "(unset by default)" elif ( _markers._OPTIONAL_GROUP in arg.field.markers and default in _fields.MISSING_SINGLETONS @@ -507,20 +512,20 @@ def _rule_generate_helptext( # There are some usage details that aren't communicated right now in the # helptext. For example: all arguments within an optional group without a # default should be passed in or none at all. - default_text = "(optional)" + behavior_hint = "(optional)" elif _markers._OPTIONAL_GROUP in arg.field.markers: # Argument in an optional group, but which also has a default. - default_text = f"(default if used: {default_label})" + behavior_hint = f"(default if used: {default_label})" else: - default_text = f"(default: {default_label})" + behavior_hint = f"(default: {default_label})" - help_parts.append(_rich_tag_if_enabled(default_text, "helptext_default")) + help_parts.append(_rich_tag_if_enabled(behavior_hint, "helptext_default")) else: help_parts.append(_rich_tag_if_enabled("(required)", "helptext_required")) # Note that the percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string - lowered.help = " ".join(help_parts).replace("%", "%%") + lowered.help = " ".join([p for p in help_parts if len(p) > 0]).replace("%", "%%") return diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 931a09827..e83e4a0a6 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -105,6 +105,7 @@ def make( None, None, help=None, + help_behavior_hint=None, aliases=None, prefix_name=True, constructor_factory=None, @@ -288,7 +289,7 @@ def field_list_from_callable( custom_constructor=False, markers={_markers.Positional, _markers._PositionalCall}, argconf=_confstruct._ArgConfiguration( - None, None, None, None, None, None + None, None, None, None, None, None, None ), call_argname="", ) diff --git a/src/tyro/conf/_confstruct.py b/src/tyro/conf/_confstruct.py index d4a87b544..ad4f57b10 100644 --- a/src/tyro/conf/_confstruct.py +++ b/src/tyro/conf/_confstruct.py @@ -1,16 +1,18 @@ +from __future__ import annotations + import dataclasses -from typing import Any, Callable, Optional, Sequence, Tuple, Type, Union, overload +from typing import Any, Callable, Sequence, overload from .._fields import MISSING_NONPROP @dataclasses.dataclass(frozen=True) class _SubcommandConfiguration: - name: Optional[str] + name: str | None default: Any - description: Optional[str] + description: str | None prefix_name: bool - constructor_factory: Optional[Callable[[], Union[Type, Callable]]] + constructor_factory: Callable[[], type | Callable] | None def __hash__(self) -> int: return object.__hash__(self) @@ -18,36 +20,36 @@ def __hash__(self) -> int: @overload def subcommand( - name: Optional[str] = None, + name: str | None = None, *, default: Any = MISSING_NONPROP, - description: Optional[str] = None, + description: str | None = None, prefix_name: bool = True, constructor: None = None, - constructor_factory: Optional[Callable[[], Union[Type, Callable]]] = None, + constructor_factory: Callable[[], type | Callable] | None = None, ) -> Any: ... @overload def subcommand( - name: Optional[str] = None, + name: str | None = None, *, default: Any = MISSING_NONPROP, - description: Optional[str] = None, + description: str | None = None, prefix_name: bool = True, - constructor: Optional[Union[Type, Callable]] = None, + constructor: type | Callable | None = None, constructor_factory: None = None, ) -> Any: ... def subcommand( - name: Optional[str] = None, + name: str | None = None, *, default: Any = MISSING_NONPROP, - description: Optional[str] = None, + description: str | None = None, prefix_name: bool = True, - constructor: Optional[Union[Type, Callable]] = None, - constructor_factory: Optional[Callable[[], Union[Type, Callable]]] = None, + constructor: type | Callable | None = None, + constructor_factory: Callable[[], type | Callable] | None = None, ) -> Any: """Returns a metadata object for configuring subcommands with `typing.Annotated`. Useful for aesthetics. @@ -110,51 +112,53 @@ def subcommand( @dataclasses.dataclass(frozen=True) class _ArgConfiguration: - # These are all optional by default in order to support multiple tyro.conf.arg() - # annotations. A None value means "don't overwrite the current value". - name: Optional[str] - metavar: Optional[str] - help: Optional[str] - aliases: Optional[Tuple[str, ...]] - prefix_name: Optional[bool] - constructor_factory: Optional[Callable[[], Union[Type, Callable]]] + name: str | None + metavar: str | None + help: str | None + help_behavior_hint: str | Callable[[str], str] | None + aliases: tuple[str, ...] | None + prefix_name: bool | None + constructor_factory: Callable[[], type | Callable] | None @overload def arg( *, - name: Optional[str] = None, - metavar: Optional[str] = None, - help: Optional[str] = None, - aliases: Optional[Sequence[str]] = None, - prefix_name: Optional[bool] = None, + name: str | None = None, + metavar: str | None = None, + help: str | None = None, + help_behavior_hint: str | Callable[[str], str] | None = None, + aliases: Sequence[str] | None = None, + prefix_name: bool | None = None, constructor: None = None, - constructor_factory: Optional[Callable[[], Union[Type, Callable]]] = None, + constructor_factory: Callable[[], type | Callable] | None = None, ) -> Any: ... @overload def arg( *, - name: Optional[str] = None, - metavar: Optional[str] = None, - help: Optional[str] = None, - aliases: Optional[Sequence[str]] = None, - prefix_name: Optional[bool] = None, - constructor: Optional[Union[Type, Callable]] = None, + name: str | None = None, + metavar: str | None = None, + help: str | None = None, + help_behavior_hint: str | Callable[[str], str] | None = None, + aliases: Sequence[str] | None = None, + prefix_name: bool | None = None, + constructor: type | Callable | None = None, constructor_factory: None = None, ) -> Any: ... def arg( *, - name: Optional[str] = None, - metavar: Optional[str] = None, - help: Optional[str] = None, - aliases: Optional[Sequence[str]] = None, - prefix_name: Optional[bool] = None, - constructor: Optional[Union[Type, Callable]] = None, - constructor_factory: Optional[Callable[[], Union[Type, Callable]]] = None, + name: str | None = None, + metavar: str | None = None, + help: str | None = None, + help_behavior_hint: str | Callable[[str], str] | None = None, + aliases: Sequence[str] | None = None, + prefix_name: bool | None = None, + constructor: type | Callable | None = None, + constructor_factory: Callable[[], type | Callable] | None = None, ) -> Any: """Returns a metadata object for fine-grained argument configuration with `typing.Annotated`. Should typically not be required. @@ -173,7 +177,11 @@ def arg( Arguments: name: A new name for the argument in the CLI. metavar: Argument name in usage messages. The type is used by default. - help: Helptext for this argument. The docstring is used by default. + help: Override helptext for this argument. The docstring is used by default. + help_behavior_hint: Override highlighted text that follows the helptext. + Typically used for behavior hints like the `(default: XXX)` or + `(optional)`. Can either be a string or a lambda function whose + input is a formatted default value. aliases: Aliases for this argument. All strings in the sequence should start with a hyphen (-). Aliases will _not_ currently be prefixed in a nested structure, and are not supported for positional arguments. @@ -201,6 +209,7 @@ def arg( name=name, metavar=metavar, help=help, + help_behavior_hint=help_behavior_hint, aliases=tuple(aliases) if aliases is not None else None, prefix_name=prefix_name, constructor_factory=constructor_factory diff --git a/tests/test_conf.py b/tests/test_conf.py index 6b72da9af..ee58bb3ba 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -627,7 +627,44 @@ def test_argconf_help() -> None: @dataclasses.dataclass class Struct: a: Annotated[ - int, tyro.conf.arg(name="nice", help="Hello world", metavar="NUMBER") + int, + tyro.conf.arg( + name="nice", + help="Hello world", + help_behavior_hint="(hint)", + metavar="NUMBER", + ), + ] = 5 + b: tyro.conf.Suppress[str] = "7" + + def main(x: Any = Struct()) -> int: + return x.a + + helptext = get_helptext_with_checks(main) + assert "Hello world" in helptext + assert "INT" not in helptext + assert "NUMBER" in helptext + assert "(hint)" in helptext + assert "(default: 5)" not in helptext + assert "--x.a" not in helptext + assert "--x.nice" in helptext + assert "--x.b" not in helptext + + assert tyro.cli(main, args=[]) == 5 + assert tyro.cli(main, args=["--x.nice", "3"]) == 3 + + +def test_argconf_help_behavior_hint_lambda() -> None: + @dataclasses.dataclass + class Struct: + a: Annotated[ + int, + tyro.conf.arg( + name="nice", + help="Hello world", + help_behavior_hint=lambda default: f"(default value: {default})", + metavar="NUMBER", + ), ] = 5 b: tyro.conf.Suppress[str] = "7" @@ -638,6 +675,8 @@ def main(x: Any = Struct()) -> int: assert "Hello world" in helptext assert "INT" not in helptext assert "NUMBER" in helptext + assert "(default value: 5)" in helptext + assert "(default: 5)" not in helptext assert "--x.a" not in helptext assert "--x.nice" in helptext assert "--x.b" not in helptext From 791c9d6f114ce84848d91980a4e5a53e4f2b61dc Mon Sep 17 00:00:00 2001 From: brentyi Date: Wed, 18 Sep 2024 10:10:54 -0700 Subject: [PATCH 447/491] `0.8.11` --- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fb6709c56..98382d197 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.10" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.11" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index 594d61c7f..f876eab09 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.10" +__version__ = "0.8.11" From 01a77d49148f53a3759bd173a6c23293b73d96db Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 20 Sep 2024 10:44:06 -0700 Subject: [PATCH 448/491] Fix inconsistency between `type[T]` and `typing.Type[T]` annotations (#165) * Fix runtime error from type[T] annotation for Python 3.10 * Generated tests * Add docstring * Fixes --- src/tyro/_resolver.py | 4 +- tests/test_new_style_annotations_min_py310.py | 17 ++++++++ .../test_conf_generated.py | 41 ++++++++++++++++++- ...w_style_annotations_min_py310_generated.py | 17 ++++++++ 4 files changed, 77 insertions(+), 2 deletions(-) diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 46b610ddc..b426f16eb 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -18,6 +18,7 @@ Sequence, Set, Tuple, + Type, TypeVar, Union, cast, @@ -409,13 +410,14 @@ def apply_type_from_typevar( dict: Dict, set: Set, frozenset: FrozenSet, + type: Type, } if hasattr(types, "UnionType"): # type: ignore # PEP 604. Requires Python 3.10. shim_table[types.UnionType] = Union # type: ignore for new, old in shim_table.items(): - if isinstance(typ, new) or origin is new: # type: ignore + if origin is new: # type: ignore typ = old.__getitem__(args) # type: ignore new_args = tuple(apply_type_from_typevar(x, type_from_typevar) for x in args) diff --git a/tests/test_new_style_annotations_min_py310.py b/tests/test_new_style_annotations_min_py310.py index dc17e3db4..66c79169c 100644 --- a/tests/test_new_style_annotations_min_py310.py +++ b/tests/test_new_style_annotations_min_py310.py @@ -1,3 +1,4 @@ +import dataclasses from typing import Any, Literal import pytest @@ -62,3 +63,19 @@ def main( ] with pytest.raises(SystemExit): tyro.cli(main, args=["--help"]) + + +def test_type(): + """Test adapted from mirceamironenco: https://github.com/brentyi/tyro/issues/164""" + + class Thing: ... + + class SubThing(Thing): ... + + @dataclasses.dataclass + class Config: + foo: int + barr: type[Thing] = dataclasses.field(default=SubThing) + bar: type[Thing] = dataclasses.field(default=SubThing) + + assert tyro.cli(Config, args=["--foo", "5"]) == Config(5, SubThing, SubThing) diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index 980fd79b6..553a1d09e 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -624,7 +624,44 @@ def test_argconf_help() -> None: @dataclasses.dataclass class Struct: a: Annotated[ - int, tyro.conf.arg(name="nice", help="Hello world", metavar="NUMBER") + int, + tyro.conf.arg( + name="nice", + help="Hello world", + help_behavior_hint="(hint)", + metavar="NUMBER", + ), + ] = 5 + b: tyro.conf.Suppress[str] = "7" + + def main(x: Any = Struct()) -> int: + return x.a + + helptext = get_helptext_with_checks(main) + assert "Hello world" in helptext + assert "INT" not in helptext + assert "NUMBER" in helptext + assert "(hint)" in helptext + assert "(default: 5)" not in helptext + assert "--x.a" not in helptext + assert "--x.nice" in helptext + assert "--x.b" not in helptext + + assert tyro.cli(main, args=[]) == 5 + assert tyro.cli(main, args=["--x.nice", "3"]) == 3 + + +def test_argconf_help_behavior_hint_lambda() -> None: + @dataclasses.dataclass + class Struct: + a: Annotated[ + int, + tyro.conf.arg( + name="nice", + help="Hello world", + help_behavior_hint=lambda default: f"(default value: {default})", + metavar="NUMBER", + ), ] = 5 b: tyro.conf.Suppress[str] = "7" @@ -635,6 +672,8 @@ def main(x: Any = Struct()) -> int: assert "Hello world" in helptext assert "INT" not in helptext assert "NUMBER" in helptext + assert "(default value: 5)" in helptext + assert "(default: 5)" not in helptext assert "--x.a" not in helptext assert "--x.nice" in helptext assert "--x.b" not in helptext diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py index dc17e3db4..66c79169c 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py @@ -1,3 +1,4 @@ +import dataclasses from typing import Any, Literal import pytest @@ -62,3 +63,19 @@ def main( ] with pytest.raises(SystemExit): tyro.cli(main, args=["--help"]) + + +def test_type(): + """Test adapted from mirceamironenco: https://github.com/brentyi/tyro/issues/164""" + + class Thing: ... + + class SubThing(Thing): ... + + @dataclasses.dataclass + class Config: + foo: int + barr: type[Thing] = dataclasses.field(default=SubThing) + bar: type[Thing] = dataclasses.field(default=SubThing) + + assert tyro.cli(Config, args=["--foo", "5"]) == Config(5, SubThing, SubThing) From 5104ffadadff59a37d65d3938db7237523cc6c03 Mon Sep 17 00:00:00 2001 From: brentyi Date: Fri, 20 Sep 2024 15:17:15 -0700 Subject: [PATCH 449/491] Improve consistency between `type` and `Type` when used as annotations (#164) --- src/tyro/_instantiators.py | 9 +++++++-- tests/test_conf.py | 12 ++++++++++++ tests/test_py311_generated/test_conf_generated.py | 12 ++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index c5b2f50d8..ae164ee9a 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -51,6 +51,7 @@ Sequence, Set, Tuple, + Type, TypeVar, Union, cast, @@ -213,7 +214,11 @@ def instantiator(strings: List[str]) -> None: ) # Validate that typ is a `(arg: str) -> T` type converter, as expected by argparse. - if typ in _builtin_set: + if typ in (type, Type): + raise UnsupportedTypeAnnotationError( + "We do not support populating `type` instances from the command-line" + ) + elif typ in _builtin_set: pass elif not callable(typ): raise UnsupportedTypeAnnotationError( @@ -222,7 +227,7 @@ def instantiator(strings: List[str]) -> None: ) elif not is_type_string_converter(typ): raise UnsupportedTypeAnnotationError( - f"Expected {typ} to be an `(arg: str) -> T` type converter, but is not" + f"Expected type to be an `(arg: str) -> T` type converter, but {typ} is not" " a valid type converter." ) diff --git a/tests/test_conf.py b/tests/test_conf.py index ee58bb3ba..1750e8db1 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -515,6 +515,18 @@ class A: ) == A(True) +def test_type_with_no_conf_is_fixed() -> None: + """The `type` type doesn't make sense to parse via the CLI, and should be + fixed. See: https://github.com/brentyi/tyro/issues/164""" + + @dataclasses.dataclass + class A: + x: type = int + + assert tyro.cli(A, args=[]) == A() + assert "fixed" in get_helptext_with_checks(A) + + def test_suppressed_group() -> None: """Reproduction of https://github.com/nerfstudio-project/nerfstudio/issues/882.""" diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index 553a1d09e..04301ed76 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -512,6 +512,18 @@ class A: ) == A(True) +def test_type_with_no_conf_is_fixed() -> None: + """The `type` type doesn't make sense to parse via the CLI, and should be + fixed. See: https://github.com/brentyi/tyro/issues/164""" + + @dataclasses.dataclass + class A: + x: type = int + + assert tyro.cli(A, args=[]) == A() + assert "fixed" in get_helptext_with_checks(A) + + def test_suppressed_group() -> None: """Reproduction of https://github.com/nerfstudio-project/nerfstudio/issues/882.""" From 8f2c05aabcaab647262bbd8ac1e73eb8c005a4c8 Mon Sep 17 00:00:00 2001 From: brentyi Date: Sat, 21 Sep 2024 15:16:24 -0700 Subject: [PATCH 450/491] Fix helptext formatting edge case (#164) --- src/tyro/_arguments.py | 2 +- tests/test_new_style_annotations_min_py310.py | 13 ++++++++++++- ...est_new_style_annotations_min_py310_generated.py | 13 ++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index 11ab223e5..4466b75d0 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -462,7 +462,7 @@ def _rule_generate_helptext( if arg.field.type_or_callable is not json.loads else json.dumps(arg.field.default) ) - elif hasattr(default, "__iter__"): + elif type(default) in (tuple, list, set): # For tuple types, we might have default as (0, 1, 2, 3). # For list types, we might have default as [0, 1, 2, 3]. # For set types, we might have default as {0, 1, 2, 3}. diff --git a/tests/test_new_style_annotations_min_py310.py b/tests/test_new_style_annotations_min_py310.py index 66c79169c..cce8a5c70 100644 --- a/tests/test_new_style_annotations_min_py310.py +++ b/tests/test_new_style_annotations_min_py310.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Any, Literal +from typing import Any, Literal, Type import pytest @@ -79,3 +79,14 @@ class Config: bar: type[Thing] = dataclasses.field(default=SubThing) assert tyro.cli(Config, args=["--foo", "5"]) == Config(5, SubThing, SubThing) + + +def test_type_default_factory(): + """Test adapted from mirceamironenco: https://github.com/brentyi/tyro/issues/164""" + + @dataclasses.dataclass + class Config: + foo: int + bar: type[Type] = dataclasses.field(default_factory=lambda: Type) + + assert tyro.cli(Config, args=["--foo", "5"]) == Config(5) diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py index 66c79169c..cce8a5c70 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py @@ -1,5 +1,5 @@ import dataclasses -from typing import Any, Literal +from typing import Any, Literal, Type import pytest @@ -79,3 +79,14 @@ class Config: bar: type[Thing] = dataclasses.field(default=SubThing) assert tyro.cli(Config, args=["--foo", "5"]) == Config(5, SubThing, SubThing) + + +def test_type_default_factory(): + """Test adapted from mirceamironenco: https://github.com/brentyi/tyro/issues/164""" + + @dataclasses.dataclass + class Config: + foo: int + bar: type[Type] = dataclasses.field(default_factory=lambda: Type) + + assert tyro.cli(Config, args=["--foo", "5"]) == Config(5) From 088aa2a488e6660d67d556e3f5a9bc4f379f7cf8 Mon Sep 17 00:00:00 2001 From: Eric McDonald <221418+emcd@users.noreply.github.com> Date: Thu, 10 Oct 2024 13:48:59 -0700 Subject: [PATCH 451/491] Feature: Markers: SelectionFromEnumValues (#168) * Feature: Markers: SelectFromEnumValues * Add configuration marker: SelectFromEnumValues * Refactor instantiator production logic for types with automatically-populated choices. Encapsulation to keep related variables tracked togther. * Nits: naming, flatten some class structure * Relax enum value string constraint * ruff * Update example to match renamed marker. Signed-off-by: Eric McDonald <221418+emcd@users.noreply.github.com> --------- Signed-off-by: Eric McDonald <221418+emcd@users.noreply.github.com> Co-authored-by: Eric McDonald Co-authored-by: brentyi --- pyproject.toml | 1 + src/tyro/_instantiators.py | 43 +++++++----- src/tyro/conf/__init__.py | 1 + src/tyro/conf/_markers.py | 26 +++++++ tests/test_dcargs.py | 67 +++++++++++++++++++ .../test_dcargs_generated.py | 45 +++++++++++++ 6 files changed, 167 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 98382d197..6191a6110 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,6 +119,7 @@ lint.select = [ lint.ignore = [ "E741", # Ambiguous variable name. (l, O, or I) "E501", # Line too long. + "E731", # Do not assign a lambda expression, use a def. "PLR2004", # Magic value used in comparison. "PLR0915", # Too many statements. "PLR0913", # Too many arguments. diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index ae164ee9a..1b8e6ed98 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -83,6 +83,7 @@ Instantiator = Union[_StandardInstantiator, _AppendNargsInstantiator, _FlagInstantiator] NoneType = type(None) +T = TypeVar("T", bound=enum.Enum) @dataclasses.dataclass @@ -277,13 +278,24 @@ def instantiate(args: List[str]) -> Any: ), ) - # Special case `choices` for some types, as implemented in `instance_from_string()`. - auto_choices: Optional[Tuple[str, ...]] = None + # Special case `choices` for some types. Booleans accept {True,False}, + # enums accept based on members. + choices: Union[None, Tuple[str, ...]] = None + autochoice_instantiate: Union[None, Callable[[str], Any]] = None if typ is bool: - auto_choices = ("True", "False") + choices = ("True", "False") + autochoice_instantiate = lambda x: {"True": True, "False": False}[x] elif inspect.isclass(typ) and issubclass(typ, enum.Enum): - # Using `.__members__` dict to handle aliases correctly - auto_choices = tuple(typ.__members__.keys()) + if _markers.EnumChoicesFromValues in markers: + value_from_str_choice = {str(member.value): member for member in typ} + choices = tuple(value_from_str_choice.keys()) + autochoice_instantiate = lambda x: value_from_str_choice[x] + else: + choices = tuple(typ.__members__.keys()) + autochoice_instantiate = lambda x: typ[x] + + if choices is not None: + metavar = "{" + ",".join(choices) + "}" def instantiator_base_case(strings: List[str]) -> Any: """Given a type and and a string from the command-line, reconstruct an object. Not @@ -303,10 +315,8 @@ def instantiator_base_case(strings: List[str]) -> Any: """ assert len(get_args(typ)) == 0, f"TypeForm {typ} cannot be instantiated." (string,) = strings - if typ is bool: - return {"True": True, "False": False}[string] # type: ignore - elif isinstance(typ, type) and issubclass(typ, enum.Enum): - return typ[string] # type: ignore + if autochoice_instantiate is not None: + return autochoice_instantiate(string) elif typ is bytes: return bytes(string, encoding="ascii") # type: ignore else: @@ -316,12 +326,8 @@ def instantiator_base_case(strings: List[str]) -> Any: return instantiator_base_case, InstantiatorMetadata( nargs=1, - metavar=( - metavar - if auto_choices is None - else "{" + ",".join(map(str, auto_choices)) + "}" - ), - choices=None if auto_choices is None else auto_choices, + metavar=metavar, + choices=choices, action=None, ) @@ -741,7 +747,12 @@ def _instantiator_from_literal( markers: Set[_markers.Marker], ) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: choices = get_args(typ) - str_choices = tuple(x.name if isinstance(x, enum.Enum) else str(x) for x in choices) + str_choices = tuple( + (x.value if _markers.EnumChoicesFromValues in markers else x.name) + if isinstance(x, enum.Enum) + else str(x) + for x in choices + ) return ( # Note that if string is not in str_choices, it will be caught from setting # `choices` below. diff --git a/src/tyro/conf/__init__.py b/src/tyro/conf/__init__.py index f58407773..7ace6c511 100644 --- a/src/tyro/conf/__init__.py +++ b/src/tyro/conf/__init__.py @@ -12,6 +12,7 @@ from ._confstruct import subcommand as subcommand from ._markers import AvoidSubcommands as AvoidSubcommands from ._markers import ConsolidateSubcommandArgs as ConsolidateSubcommandArgs +from ._markers import EnumChoicesFromValues as EnumChoicesFromValues from ._markers import Fixed as Fixed from ._markers import FlagConversionOff as FlagConversionOff from ._markers import OmitArgPrefixes as OmitArgPrefixes diff --git a/src/tyro/conf/_markers.py b/src/tyro/conf/_markers.py index 833b5fc3c..50377aeca 100644 --- a/src/tyro/conf/_markers.py +++ b/src/tyro/conf/_markers.py @@ -127,6 +127,32 @@ UseCounterAction = Annotated[T, None] """Use "counter" actions for integer arguments. Example usage: `verbose: UseCounterAction[int]`.""" +EnumChoicesFromValues = Annotated[T, None] +"""Populate choices from enum values rather than enum names. + +Example: +``` + class OutputFormats(enum.StrEnum): + JSON = enum.auto() + PRETTY = enum.auto() + RICH = enum.auto() + TOML = enum.auto() + + @dataclasses.dataclass + class Args: + display_format: Annotated[ + OutputFormats, tyro.conf.EnumChoicesFromValues + ] = OutputFormats.PRETTY +``` + +The above will result in `json`, `pretty`, `rich`, and `toml` (all lowercase) as choices, +since the auto values for `StrEnum` (Python 3.11+) are lowercase transformations of the +names. Without this marker, the choices would be `JSON`, `PRETTY`, `RICH`, and `TOML`. + +Enum aliases are not relevant when this marker is present. The first entry matching the +chosen value will be selected. +""" + CallableType = TypeVar("CallableType", bound=Callable) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 723733a1d..432140881 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -295,6 +295,8 @@ class EnumClassB: assert tyro.cli(EnumClassA, args=["--color", "RED"]) == EnumClassA(color=Color.RED) assert tyro.cli(EnumClassB, args=[]) == EnumClassB() + with pytest.raises(SystemExit): + assert tyro.cli(EnumClassA, args=["--color", "red"]) def test_enum_alias() -> None: @@ -311,6 +313,48 @@ class A: assert tyro.cli(A, args=["--color", "ROUGE"]) == A(color=Color.RED) +def test_enum_values() -> None: + class Color(enum.Enum): + RED = "red" + GREEN = "green" + BLUE = "blue" + + @dataclasses.dataclass + class EnumClassA: + color: Annotated[Color, tyro.conf.EnumChoicesFromValues] + + @dataclasses.dataclass + class EnumClassB: + color: Annotated[Color, tyro.conf.EnumChoicesFromValues] = Color.GREEN + + assert tyro.cli(EnumClassA, args=["--color", "red"]) == EnumClassA(color=Color.RED) + assert tyro.cli(EnumClassB, args=[]) == EnumClassB() + with pytest.raises(SystemExit): + assert tyro.cli(EnumClassA, args=["--color", "RED"]) + + +def test_enum_values_ints() -> None: + class Color(enum.Enum): + RED = 0 + GREEN = 1 + BLUE = 2 + + @dataclasses.dataclass + class EnumClassA: + color: Annotated[Color, tyro.conf.EnumChoicesFromValues] + + @dataclasses.dataclass + class EnumClassB: + color: Annotated[Color, tyro.conf.EnumChoicesFromValues] = Color.GREEN + + assert tyro.cli(EnumClassA, args=["--color", "0"]) == EnumClassA(color=Color.RED) + assert tyro.cli(EnumClassB, args=[]) == EnumClassB() + with pytest.raises(SystemExit): + assert tyro.cli(EnumClassA, args=["--color", "red"]) + with pytest.raises(SystemExit): + assert tyro.cli(EnumClassA, args=["--color", "RED"]) + + def test_literal() -> None: @dataclasses.dataclass class A: @@ -377,6 +421,29 @@ class A: assert tyro.cli(A, args=["--x", "GREEN"]) == A(x=Color.GREEN) with pytest.raises(SystemExit): assert tyro.cli(A, args=["--x", "BLUE"]) + with pytest.raises(SystemExit): + assert tyro.cli(A, args=["--x", "red"]) + + +def test_literal_enum_values() -> None: + class Color(enum.Enum): + RED = "red" + GREEN = "green" + BLUE = "blue" + + @dataclasses.dataclass + class A: + x: Annotated[ + Literal[Color.RED, Color.GREEN], + tyro.conf.EnumChoicesFromValues, + ] + + assert tyro.cli(A, args=["--x", "red"]) == A(x=Color.RED) + assert tyro.cli(A, args=["--x", "green"]) == A(x=Color.GREEN) + with pytest.raises(SystemExit): + assert tyro.cli(A, args=["--x", "RED"]) + with pytest.raises(SystemExit): + assert tyro.cli(A, args=["--x", "blue"]) def test_optional_literal() -> None: diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py index b8836572b..792c8e982 100644 --- a/tests/test_py311_generated/test_dcargs_generated.py +++ b/tests/test_py311_generated/test_dcargs_generated.py @@ -297,6 +297,8 @@ class EnumClassB: assert tyro.cli(EnumClassA, args=["--color", "RED"]) == EnumClassA(color=Color.RED) assert tyro.cli(EnumClassB, args=[]) == EnumClassB() + with pytest.raises(SystemExit): + assert tyro.cli(EnumClassA, args=["--color", "red"]) def test_enum_alias() -> None: @@ -313,6 +315,26 @@ class A: assert tyro.cli(A, args=["--color", "ROUGE"]) == A(color=Color.RED) +def test_enum_values() -> None: + class Color(enum.StrEnum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + @dataclasses.dataclass + class EnumClassA: + color: Annotated[Color, tyro.conf.EnumChoicesFromValues] + + @dataclasses.dataclass + class EnumClassB: + color: Annotated[Color, tyro.conf.EnumChoicesFromValues] = Color.GREEN + + assert tyro.cli(EnumClassA, args=["--color", "red"]) == EnumClassA(color=Color.RED) + assert tyro.cli(EnumClassB, args=[]) == EnumClassB() + with pytest.raises(SystemExit): + assert tyro.cli(EnumClassA, args=["--color", "RED"]) + + def test_literal() -> None: @dataclasses.dataclass class A: @@ -379,6 +401,29 @@ class A: assert tyro.cli(A, args=["--x", "GREEN"]) == A(x=Color.GREEN) with pytest.raises(SystemExit): assert tyro.cli(A, args=["--x", "BLUE"]) + with pytest.raises(SystemExit): + assert tyro.cli(A, args=["--x", "red"]) + + +def test_literal_enum_values() -> None: + class Color(enum.StrEnum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + @dataclasses.dataclass + class A: + x: Annotated[ + Literal[Color.RED, Color.GREEN], + tyro.conf.EnumChoicesFromValues, + ] + + assert tyro.cli(A, args=["--x", "red"]) == A(x=Color.RED) + assert tyro.cli(A, args=["--x", "green"]) == A(x=Color.GREEN) + with pytest.raises(SystemExit): + assert tyro.cli(A, args=["--x", "RED"]) + with pytest.raises(SystemExit): + assert tyro.cli(A, args=["--x", "blue"]) def test_optional_literal() -> None: From 3b7e37ddbb3a261db9c298cc378cc7f919f3755d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 10 Oct 2024 14:18:59 -0700 Subject: [PATCH 452/491] Fix `--tyro-write-completion` when `use_underscores=True` (#173) * Fix `--tyro-write-completion` when `use_underscores=True` * Revert example update --- src/tyro/_cli.py | 12 +++++++++--- tests/helptext_utils.py | 38 ++++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index 4173b16f2..ff01894cf 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -381,8 +381,14 @@ def _cli_impl( # # Note: --tyro-print-completion is deprecated! --tyro-write-completion is less prone # to errors from accidental logging, print statements, etc. - print_completion = len(args) >= 2 and args[0] == "--tyro-print-completion" - write_completion = len(args) >= 3 and args[0] == "--tyro-write-completion" + print_completion = False + write_completion = False + if len(args) >= 2: + # We replace underscores with hyphens to accomodate for `use_undercores`. + print_completion = args[0].replace("_", "-") == "--tyro-print-completion" + write_completion = ( + len(args) >= 3 and args[0].replace("_", "-") == "--tyro-write-completion" + ) # Note: setting USE_RICH must happen before the parser specification is generated. # TODO: revisit this. Ideally we should be able to eliminate the global state @@ -442,7 +448,7 @@ def _cli_impl( f" {completion_shell}" ) - if write_completion: + if write_completion and completion_target_path != pathlib.Path("-"): assert completion_target_path is not None completion_target_path.write_text( shtab.complete( diff --git a/tests/helptext_utils.py b/tests/helptext_utils.py index 041f5c58b..73dd1fc99 100644 --- a/tests/helptext_utils.py +++ b/tests/helptext_utils.py @@ -44,21 +44,35 @@ def get_helptext_with_checks( unformatted_usage = parser.format_usage() assert tyro._strings.strip_ansi_sequences(unformatted_usage) == unformatted_usage - # Completion scripts; just smoke test for now. - with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): - # `--tyro-print-completion` is deprecated! We should use `--tyro-write-completion` instead. - tyro.cli(f, default=default, args=["--tyro-print-completion", "bash"]) - with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): - # `--tyro-print-completion` is deprecated! We should use `--tyro-write-completion` instead. - tyro.cli(f, default=default, args=["--tyro-print-completion", "zsh"]) - with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): + # Basic checks for completion scripts. + with pytest.raises(SystemExit): tyro.cli( f, default=default, args=["--tyro-write-completion", "bash", os.devnull] ) - with pytest.raises(SystemExit), contextlib.redirect_stdout(open(os.devnull, "w")): - tyro.cli( - f, default=default, args=["--tyro-write-completion", "zsh", os.devnull] - ) + for shell in ["bash", "zsh"]: + for command in ["--tyro-print-completion", "--tyro-write-completion"]: + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + if command == "--tyro-write-completion": + tyro.cli(f, default=default, args=[command, shell, "-"]) + else: + # `--tyro-print-completion` is deprecated! We should use `--tyro-write-completion` instead. + tyro.cli(f, default=default, args=[command, shell]) + output = target.getvalue() + assert "shtab" in output + + # Test with underscores + for shell in ["bash", "zsh"]: + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stdout(target): + tyro.cli( + f, + default=default, + args=["--tyro_write_completion", shell, "-"], + use_underscores=True, + ) + output = target.getvalue() + assert "shtab" in output # Get the actual helptext. target = io.StringIO() From 80b1316c9da525cdee1cd35541a564bec5a06dfc Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 10 Oct 2024 15:20:13 -0700 Subject: [PATCH 453/491] Decorator-based subcommand API (#169) * Decorator-based subcommands * Fix example * Switch to class, tests * Add missing * Coverage * Sync docs * Coverage --- .../15_decorator_subcommands.rst | 85 +++++++++++ .../04_additional/15_decorator_subcommands.py | 37 +++++ src/tyro/extras/__init__.py | 1 + src/tyro/extras/_subcommand_app.py | 140 ++++++++++++++++++ tests/test_decorator_subcommands.py | 72 +++++++++ .../test_decorator_subcommands_generated.py | 59 ++++++++ 6 files changed, 394 insertions(+) create mode 100644 docs/source/examples/04_additional/15_decorator_subcommands.rst create mode 100644 examples/04_additional/15_decorator_subcommands.py create mode 100644 src/tyro/extras/_subcommand_app.py create mode 100644 tests/test_decorator_subcommands.py create mode 100644 tests/test_py311_generated/test_decorator_subcommands_generated.py diff --git a/docs/source/examples/04_additional/15_decorator_subcommands.rst b/docs/source/examples/04_additional/15_decorator_subcommands.rst new file mode 100644 index 000000000..5b8b5f506 --- /dev/null +++ b/docs/source/examples/04_additional/15_decorator_subcommands.rst @@ -0,0 +1,85 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Decorator-based Subcommands +========================================== + +:func:`tyro.extras.app.command()` and :func:`tyro.extras.app.cli()` provide a +decorator-based API for subcommands, which is inspired by `click +`_. + + +.. code-block:: python + :linenos: + + + from tyro.extras import SubcommandApp + + app = SubcommandApp() + + + @app.command + def greet(name: str, loud: bool = False) -> None: + """Greet someone.""" + greeting = f"Hello, {name}!" + if loud: + greeting = greeting.upper() + print(greeting) + + + @app.command(name="addition") + def add(a: int, b: int) -> None: + """Add two numbers.""" + print(f"{a} + {b} = {a + b}") + + + if __name__ == "__main__": + app.cli() + +------------ + +.. raw:: html + + python 04_additional/15_decorator_subcommands.py --help + +.. program-output:: python ../../examples/04_additional/15_decorator_subcommands.py --help + +------------ + +.. raw:: html + + python 04_additional/15_decorator_subcommands.py greet --help + +.. program-output:: python ../../examples/04_additional/15_decorator_subcommands.py greet --help + +------------ + +.. raw:: html + + python 04_additional/15_decorator_subcommands.py greet --name Alice + +.. program-output:: python ../../examples/04_additional/15_decorator_subcommands.py greet --name Alice + +------------ + +.. raw:: html + + python 04_additional/15_decorator_subcommands.py greet --name Bob --loud + +.. program-output:: python ../../examples/04_additional/15_decorator_subcommands.py greet --name Bob --loud + +------------ + +.. raw:: html + + python 04_additional/15_decorator_subcommands.py addition --help + +.. program-output:: python ../../examples/04_additional/15_decorator_subcommands.py addition --help + +------------ + +.. raw:: html + + python 04_additional/15_decorator_subcommands.py addition --a 5 --b 3 + +.. program-output:: python ../../examples/04_additional/15_decorator_subcommands.py addition --a 5 --b 3 diff --git a/examples/04_additional/15_decorator_subcommands.py b/examples/04_additional/15_decorator_subcommands.py new file mode 100644 index 000000000..466b9a780 --- /dev/null +++ b/examples/04_additional/15_decorator_subcommands.py @@ -0,0 +1,37 @@ +"""Decorator-based Subcommands + +:func:`tyro.extras.app.command()` and :func:`tyro.extras.app.cli()` provide a +decorator-based API for subcommands, which is inspired by `click +`_. + +Usage: +`python my_script.py --help` +`python my_script.py greet --help` +`python my_script.py greet --name Alice` +`python my_script.py greet --name Bob --loud` +`python my_script.py addition --help` +`python my_script.py addition --a 5 --b 3` +""" + +from tyro.extras import SubcommandApp + +app = SubcommandApp() + + +@app.command +def greet(name: str, loud: bool = False) -> None: + """Greet someone.""" + greeting = f"Hello, {name}!" + if loud: + greeting = greeting.upper() + print(greeting) + + +@app.command(name="addition") +def add(a: int, b: int) -> None: + """Add two numbers.""" + print(f"{a} + {b} = {a + b}") + + +if __name__ == "__main__": + app.cli() diff --git a/src/tyro/extras/__init__.py b/src/tyro/extras/__init__.py index 0afb6b854..39878d58e 100644 --- a/src/tyro/extras/__init__.py +++ b/src/tyro/extras/__init__.py @@ -11,6 +11,7 @@ from ._choices_type import literal_type_from_choices as literal_type_from_choices from ._serialization import from_yaml as from_yaml from ._serialization import to_yaml as to_yaml +from ._subcommand_app import SubcommandApp as SubcommandApp from ._subcommand_cli_from_dict import ( subcommand_cli_from_dict as subcommand_cli_from_dict, ) diff --git a/src/tyro/extras/_subcommand_app.py b/src/tyro/extras/_subcommand_app.py new file mode 100644 index 000000000..0a8a906f5 --- /dev/null +++ b/src/tyro/extras/_subcommand_app.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +from typing import Any, Callable, Dict, Optional, Sequence, TypeVar, overload + +import tyro + +CallableT = TypeVar("CallableT", bound=Callable) + + +class SubcommandApp: + """This module provides a decorator-based API for subcommands in `tyro`, inspired by click. + + Example: + + ```python + from tyro.extras import SubcommandApp + + app = SubcommandApp() + + @app.command + def greet(name: str, loud: bool = False): + '''Greet someone.''' + greeting = f"Hello, {name}!" + if loud: + greeting = greeting.upper() + print(greeting) + + @app.command(name="addition") + def add(a: int, b: int): + '''Add two numbers.''' + print(f"{a} + {b} = {a + b}") + + if __name__ == "__main__": + app.cli() + ``` + + Usage: + `python my_script.py greet Alice` + `python my_script.py greet Bob --loud` + `python my_script.py addition 5 3` + """ + + def __init__(self) -> None: + self._subcommands: Dict[str, Callable] = {} + + @overload + def command(self, func: CallableT) -> CallableT: ... + + @overload + def command( + self, + func: None = None, + *, + name: str | None = None, + ) -> Callable[[CallableT], CallableT]: ... + + def command( + self, + func: CallableT | None = None, + *, + name: str | None = None, + ) -> CallableT | Callable[[CallableT], CallableT]: + """A decorator to register a function as a subcommand. + + This method is inspired by Click's @cli.command() decorator. + It adds the decorated function to the list of subcommands. + + Args: + func: The function to register as a subcommand. If None, returns a + function to use as a decorator. + name: The name of the subcommand. If None, the name of the function is used. + """ + + def inner(func: CallableT) -> CallableT: + nonlocal name + if name is None: + name = func.__name__ + + self._subcommands[name] = func + return func + + if func is not None: + return inner(func) + else: + return inner + + def cli( + self, + *, + prog: Optional[str] = None, + description: Optional[str] = None, + args: Optional[Sequence[str]] = None, + use_underscores: bool = False, + sort_subcommands: bool = True, + ) -> Any: + """Run the command-line interface. + + This method creates a CLI using tyro, with all subcommands registered using + :func:`command()`. + + Args: + prog: The name of the program printed in helptext. Mirrors argument from + `argparse.ArgumentParser()`. + description: Description text for the parser, displayed when the --help flag is + passed in. If not specified, the class docstring is used. Mirrors argument from + `argparse.ArgumentParser()`. + args: If set, parse arguments from a sequence of strings instead of the + commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. + use_underscores: If True, use underscores as a word delimiter instead of hyphens. + This primarily impacts helptext; underscores and hyphens are treated equivalently + when parsing happens. We default helptext to hyphens to follow the GNU style guide. + https://www.gnu.org/software/libc/manual/html_node/Argument-Syntax.html + sort_subcommands: If True, sort the subcommands alphabetically by name. + """ + assert self._subcommands is not None + + # Sort subcommands by name. + if sort_subcommands: + sorted_subcommands = dict( + sorted(self._subcommands.items(), key=lambda x: x[0]) + ) + else: + sorted_subcommands = self._subcommands + + if len(sorted_subcommands) == 1: + return tyro.cli( + next(iter(sorted_subcommands.values())), + prog=prog, + description=description, + args=args, + use_underscores=use_underscores, + ) + else: + return tyro.extras.subcommand_cli_from_dict( + sorted_subcommands, + prog=prog, + description=description, + args=args, + use_underscores=use_underscores, + ) diff --git a/tests/test_decorator_subcommands.py b/tests/test_decorator_subcommands.py new file mode 100644 index 000000000..422819d15 --- /dev/null +++ b/tests/test_decorator_subcommands.py @@ -0,0 +1,72 @@ +import pytest + +from tyro.extras import SubcommandApp + +app = SubcommandApp() +app_just_one = SubcommandApp() + + +@app_just_one.command +@app.command +def greet(name: str, loud: bool = False) -> None: + """Greet someone.""" + greeting = f"Hello, {name}!" + if loud: + greeting = greeting.upper() + print(greeting) + + +@app.command(name="addition") +def add(a: int, b: int) -> None: + """Add two numbers.""" + print(f"{a} + {b} = {a + b}") + + +def test_app_just_one_cli(capsys): + # Test: `python my_script.py --help` + with pytest.raises(SystemExit): + app_just_one.cli(args=["--help"], sort_subcommands=False) + captured = capsys.readouterr() + assert "usage: " in captured.out + assert "greet" not in captured.out + assert "addition" not in captured.out + assert "--name" in captured.out + + +def test_app_cli(capsys): + # Test: `python my_script.py --help` + with pytest.raises(SystemExit): + app.cli(args=["--help"]) + captured = capsys.readouterr() + assert "usage: " in captured.out + assert "greet" in captured.out + assert "addition" in captured.out + + # Test: `python my_script.py greet --help` + with pytest.raises(SystemExit): + app.cli(args=["greet", "--help"]) + captured = capsys.readouterr() + assert "usage: " in captured.out + assert "Greet someone." in captured.out + + # Test: `python my_script.py greet --name Alice` + app.cli(args=["greet", "--name", "Alice"]) + captured = capsys.readouterr() + assert captured.out.strip() == "Hello, Alice!" + + # Test: `python my_script.py greet --name Bob --loud` + app.cli(args=["greet", "--name", "Bob", "--loud"]) + captured = capsys.readouterr() + assert captured.out.strip() == "HELLO, BOB!" + + # Test: `python my_script.py addition --help` + with pytest.raises(SystemExit): + app.cli(args=["addition", "--help"]) + captured = capsys.readouterr() + assert "usage: " in captured.out + assert "Add two numbers." in captured.out + + # Test: `python my_script.py addition 5 3` + app.cli(args=["addition", "--a", "5", "--b", "3"]) + captured = capsys.readouterr() + assert captured.out.strip() == "5 + 3 = 8" diff --git a/tests/test_py311_generated/test_decorator_subcommands_generated.py b/tests/test_py311_generated/test_decorator_subcommands_generated.py new file mode 100644 index 000000000..8a623bc3a --- /dev/null +++ b/tests/test_py311_generated/test_decorator_subcommands_generated.py @@ -0,0 +1,59 @@ +import pytest + +from tyro.extras import SubcommandApp + +app = SubcommandApp() + + +@app.command +def greet(name: str, loud: bool = False) -> None: + """Greet someone.""" + greeting = f"Hello, {name}!" + if loud: + greeting = greeting.upper() + print(greeting) + + +@app.command(name="addition") +def add(a: int, b: int) -> None: + """Add two numbers.""" + print(f"{a} + {b} = {a + b}") + + +def test_app_cli(capsys): + # Test: `python my_script.py --help` + with pytest.raises(SystemExit): + app.cli(args=["--help"]) + captured = capsys.readouterr() + assert "usage: " in captured.out + assert "greet" in captured.out + assert "addition" in captured.out + + # Test: `python my_script.py greet --help` + with pytest.raises(SystemExit): + app.cli(args=["greet", "--help"]) + captured = capsys.readouterr() + assert "usage: " in captured.out + assert "Greet someone." in captured.out + + # Test: `python my_script.py greet --name Alice` + app.cli(args=["greet", "--name", "Alice"]) + captured = capsys.readouterr() + assert captured.out.strip() == "Hello, Alice!" + + # Test: `python my_script.py greet --name Bob --loud` + app.cli(args=["greet", "--name", "Bob", "--loud"]) + captured = capsys.readouterr() + assert captured.out.strip() == "HELLO, BOB!" + + # Test: `python my_script.py addition --help` + with pytest.raises(SystemExit): + app.cli(args=["addition", "--help"]) + captured = capsys.readouterr() + assert "usage: " in captured.out + assert "Add two numbers." in captured.out + + # Test: `python my_script.py addition 5 3` + app.cli(args=["addition", "--a", "5", "--b", "3"]) + captured = capsys.readouterr() + assert captured.out.strip() == "5 + 3 = 8" From 2df6aa19f8ddfe44b89f93003504bcc51c18aa2d Mon Sep 17 00:00:00 2001 From: brentyi Date: Thu, 10 Oct 2024 17:49:36 -0700 Subject: [PATCH 454/491] Bump version, fix subcommand example docs/comments --- .../examples/04_additional/15_decorator_subcommands.rst | 5 ++--- examples/04_additional/15_decorator_subcommands.py | 5 ++--- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- src/tyro/extras/_subcommand_app.py | 2 +- tests/test_decorator_subcommands.py | 6 +++--- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/source/examples/04_additional/15_decorator_subcommands.rst b/docs/source/examples/04_additional/15_decorator_subcommands.rst index 5b8b5f506..1def51afe 100644 --- a/docs/source/examples/04_additional/15_decorator_subcommands.rst +++ b/docs/source/examples/04_additional/15_decorator_subcommands.rst @@ -4,9 +4,8 @@ Decorator-based Subcommands ========================================== -:func:`tyro.extras.app.command()` and :func:`tyro.extras.app.cli()` provide a -decorator-based API for subcommands, which is inspired by `click -`_. +:func:`tyro.extras.SubcommandApp()` provides a decorator-based API for +subcommands, which is inspired by `click `_. .. code-block:: python diff --git a/examples/04_additional/15_decorator_subcommands.py b/examples/04_additional/15_decorator_subcommands.py index 466b9a780..e9fc77ccb 100644 --- a/examples/04_additional/15_decorator_subcommands.py +++ b/examples/04_additional/15_decorator_subcommands.py @@ -1,8 +1,7 @@ """Decorator-based Subcommands -:func:`tyro.extras.app.command()` and :func:`tyro.extras.app.cli()` provide a -decorator-based API for subcommands, which is inspired by `click -`_. +:func:`tyro.extras.SubcommandApp()` provides a decorator-based API for +subcommands, which is inspired by `click `_. Usage: `python my_script.py --help` diff --git a/pyproject.toml b/pyproject.toml index 6191a6110..71bdffb82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.11" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.12" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index f876eab09..7fd0b1827 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.11" +__version__ = "0.8.12" diff --git a/src/tyro/extras/_subcommand_app.py b/src/tyro/extras/_subcommand_app.py index 0a8a906f5..f5497188c 100644 --- a/src/tyro/extras/_subcommand_app.py +++ b/src/tyro/extras/_subcommand_app.py @@ -91,7 +91,7 @@ def cli( description: Optional[str] = None, args: Optional[Sequence[str]] = None, use_underscores: bool = False, - sort_subcommands: bool = True, + sort_subcommands: bool = False, ) -> Any: """Run the command-line interface. diff --git a/tests/test_decorator_subcommands.py b/tests/test_decorator_subcommands.py index 422819d15..0ed47227c 100644 --- a/tests/test_decorator_subcommands.py +++ b/tests/test_decorator_subcommands.py @@ -25,7 +25,7 @@ def add(a: int, b: int) -> None: def test_app_just_one_cli(capsys): # Test: `python my_script.py --help` with pytest.raises(SystemExit): - app_just_one.cli(args=["--help"], sort_subcommands=False) + app_just_one.cli(args=["--help"]) captured = capsys.readouterr() assert "usage: " in captured.out assert "greet" not in captured.out @@ -44,13 +44,13 @@ def test_app_cli(capsys): # Test: `python my_script.py greet --help` with pytest.raises(SystemExit): - app.cli(args=["greet", "--help"]) + app.cli(args=["greet", "--help"], sort_subcommands=False) captured = capsys.readouterr() assert "usage: " in captured.out assert "Greet someone." in captured.out # Test: `python my_script.py greet --name Alice` - app.cli(args=["greet", "--name", "Alice"]) + app.cli(args=["greet", "--name", "Alice"], sort_subcommands=True) captured = capsys.readouterr() assert captured.out.strip() == "Hello, Alice!" From 4cbf663cb137d6a7c7a3eb75281d1e2b7e5c5d12 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 11 Oct 2024 00:37:29 -0700 Subject: [PATCH 455/491] Support suppressed arguments with duplicate names + light refactor for maintainability (#174) --- src/tyro/_arguments.py | 54 +++++++++++-------- src/tyro/_fields.py | 50 +++++++++-------- src/tyro/_parsers.py | 13 +++-- tests/test_new_style_annotations_min_py310.py | 18 ++++--- .../test_dcargs_generated.py | 38 ++++++++++--- .../test_decorator_subcommands_generated.py | 17 +++++- ...w_style_annotations_min_py310_generated.py | 18 ++++--- .../test_repeat_suppressed_generated.py | 38 +++++++++++++ tests/test_repeat_suppressed.py | 38 +++++++++++++ 9 files changed, 211 insertions(+), 73 deletions(-) create mode 100644 tests/test_py311_generated/test_repeat_suppressed_generated.py create mode 100644 tests/test_repeat_suppressed.py diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index 4466b75d0..40bce6745 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -533,32 +533,44 @@ def _rule_set_name_or_flag_and_dest( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> None: - name_or_flag = _strings.make_field_name( - [arg.extern_prefix, arg.field.extern_name] - if arg.field.argconf.prefix_name - and _markers.OmitArgPrefixes not in arg.field.markers - else [arg.field.extern_name] - ) + if lowered.help is argparse.SUPPRESS: + # Use standard name for suppressed arguments. + # Relevant: https://github.com/brentyi/tyro/issues/170 + name_or_flag = _strings.make_field_name( + [arg.extern_prefix, arg.field.extern_name] + ) + elif ( + arg.field.argconf.prefix_name is False + or _markers.OmitArgPrefixes in arg.field.markers + ): + # Strip prefixes when the argument is suppressed. + # Still need to call make_field_name() because it converts underscores + # to hyphens, etc. + name_or_flag = _strings.make_field_name([arg.field.extern_name]) + elif ( + _markers.OmitSubcommandPrefixes in arg.field.markers + and arg.subcommand_prefix != "" + ): + # Strip subcommand prefixes, but keep following prefixes. Note that + # `extern_prefix` can start with the prefix corresponding to the parent + # subcommand, but end with other prefixes correspondeding to nested + # structures within the subcommand. + name_or_flag = _strings.make_field_name( + [arg.extern_prefix, arg.field.extern_name] + ) + strip_prefix = arg.subcommand_prefix + "." + assert name_or_flag.startswith(strip_prefix), name_or_flag + name_or_flag = name_or_flag[len(strip_prefix) :] + else: + # Standard prefixed name. + name_or_flag = _strings.make_field_name( + [arg.extern_prefix, arg.field.extern_name] + ) # Prefix keyword arguments with --. if not arg.field.is_positional(): name_or_flag = "--" + name_or_flag - # Strip. - if ( - # If OmitArgPrefixes was applied, then the subcommand prefix was already - # stripped. :) - _markers.OmitArgPrefixes not in arg.field.markers - and _markers.Positional not in arg.field.markers - and name_or_flag.startswith("--") - and arg.subcommand_prefix != "" - ): - # This will run even when unused because we want the assert. - strip_prefix = "--" + arg.subcommand_prefix + "." - if _markers.OmitSubcommandPrefixes in arg.field.markers: - assert name_or_flag.startswith(strip_prefix), name_or_flag - name_or_flag = "--" + name_or_flag[len(strip_prefix) :] - lowered.name_or_flag = name_or_flag lowered.dest = _strings.make_field_name([arg.intern_prefix, arg.field.intern_name]) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index e83e4a0a6..d1257ea98 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -6,6 +6,7 @@ import builtins import collections import collections.abc +import contextlib import dataclasses import enum import functools @@ -55,6 +56,8 @@ from ._typing import TypeForm from .conf import _confstruct, _markers +_field_context_markers: List[Tuple[_markers.Marker, ...]] = [] + @dataclasses.dataclass class FieldDefinition: @@ -86,6 +89,15 @@ def __post_init__(self): f"Field {self.intern_name} is missing a default value!" ) + @staticmethod + @contextlib.contextmanager + def marker_context(markers: Tuple[_markers.Marker, ...]): + """Context for setting markers on fields. All fields created within the + context will have the specified markers.""" + _field_context_markers.append(markers) + yield + _field_context_markers.pop() + @staticmethod def make( name: str, @@ -126,7 +138,13 @@ def make( type_or_callable, inferred_markers = _resolver.unwrap_annotated( type_or_callable, _markers._Marker ) - return FieldDefinition( + markers = inferred_markers + markers + + # Include markers set via context manager. + for context_markers in _field_context_markers: + markers += context_markers + + out = FieldDefinition( intern_name=name, extern_name=name if argconf.name is None else argconf.name, type_or_callable=type_or_callable @@ -135,19 +153,14 @@ def make( default=default, is_default_from_default_instance=is_default_from_default_instance, helptext=helptext, - markers=set(inferred_markers).union(markers), + markers=set(markers), custom_constructor=argconf.constructor_factory is not None, argconf=argconf, call_argname=call_argname_override if call_argname_override is not None else name, ) - - def add_markers(self, markers: Tuple[Any, ...]) -> FieldDefinition: - return dataclasses.replace( - self, - markers=self.markers.union(markers), - ) + return out def is_positional(self) -> bool: """Returns True if the argument should be positional in the commandline.""" @@ -271,7 +284,10 @@ def field_list_from_callable( f = _resolver.unwrap_newtype_and_narrow_subtypes(f, default_instance) # Try to generate field list. - field_list = _try_field_list_from_callable(f, default_instance) + # We recursively apply markers. + _, parent_markers = _resolver.unwrap_annotated(f, _markers._Marker) + with FieldDefinition.marker_context(parent_markers): + field_list = _try_field_list_from_callable(f, default_instance) if isinstance(field_list, UnsupportedNestedTypeMessage): if support_single_arg_types: @@ -298,10 +314,6 @@ def field_list_from_callable( else: raise _instantiators.UnsupportedTypeAnnotationError(field_list.message) - # Recursively apply markers. - _, parent_markers = _resolver.unwrap_annotated(f, _markers._Marker) - field_list = list(map(lambda field: field.add_markers(parent_markers), field_list)) - # Try to resolve types in our list of fields. def resolve(field: FieldDefinition) -> FieldDefinition: typ = field.type_or_callable @@ -947,17 +959,13 @@ def _try_field_list_from_general_callable( # Ignore self parameter. params = params[1:] - out = _field_list_from_params(f, cls, params) - if isinstance(out, UnsupportedNestedTypeMessage): - # Return error message. - return out - # If a default is provided: either all or none of the arguments must be passed in. + markers: Tuple[Any, ...] = () if default_instance not in MISSING_SINGLETONS: - for i, field in enumerate(out): - out[i] = field.add_markers((_markers._OPTIONAL_GROUP,)) + markers = (_markers._OPTIONAL_GROUP,) - return out + with FieldDefinition.marker_context(markers): + return _field_list_from_params(f, cls, params) def _field_list_from_params( diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 39d469f9f..c0d711062 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -79,13 +79,12 @@ def from_callable_or_type( consolidate_subcommand_args = _markers.ConsolidateSubcommandArgs in markers # Resolve the type of `f`, generate a field list. - f, type_from_typevar, field_list = _fields.field_list_from_callable( - f=f, - default_instance=default_instance, - support_single_arg_types=support_single_arg_types, - ) - for i in range(len(field_list)): - field_list[i].markers |= set(markers) + with _fields.FieldDefinition.marker_context(markers): + f, type_from_typevar, field_list = _fields.field_list_from_callable( + f=f, + default_instance=default_instance, + support_single_arg_types=support_single_arg_types, + ) # Cycle detection. # diff --git a/tests/test_new_style_annotations_min_py310.py b/tests/test_new_style_annotations_min_py310.py index cce8a5c70..f509d3cbc 100644 --- a/tests/test_new_style_annotations_min_py310.py +++ b/tests/test_new_style_annotations_min_py310.py @@ -1,3 +1,7 @@ +# mypy: ignore-errors +# +# We can remove this ignore after: https://peps.python.org/pep-0747/ + import dataclasses from typing import Any, Literal, Type @@ -6,12 +10,12 @@ import tyro -def test_union_direct(): +def test_union_direct() -> None: assert tyro.cli(int | str, args=["5"]) == 5 assert tyro.cli(int | str, args=["five"]) == "five" -def test_union_basic(): +def test_union_basic() -> None: def main(x: int | str) -> int | str: return x @@ -20,7 +24,7 @@ def main(x: int | str) -> int | str: assert tyro.cli(main, args=["--x", "five"]) == "five" -def test_union_with_list(): +def test_union_with_list() -> None: def main(x: int | str | list[bool]) -> Any: return x @@ -31,7 +35,7 @@ def main(x: int | str | list[bool]) -> Any: assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] -def test_union_literal(): +def test_union_literal() -> None: def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: return x @@ -40,7 +44,7 @@ def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: assert tyro.cli(main, args=["--x", "five"]) == "five" -def test_super_nested(): +def test_super_nested() -> None: def main( x: None | list[ @@ -65,7 +69,7 @@ def main( tyro.cli(main, args=["--help"]) -def test_type(): +def test_type() -> None: """Test adapted from mirceamironenco: https://github.com/brentyi/tyro/issues/164""" class Thing: ... @@ -81,7 +85,7 @@ class Config: assert tyro.cli(Config, args=["--foo", "5"]) == Config(5, SubThing, SubThing) -def test_type_default_factory(): +def test_type_default_factory() -> None: """Test adapted from mirceamironenco: https://github.com/brentyi/tyro/issues/164""" @dataclasses.dataclass diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py index 792c8e982..8952be3e2 100644 --- a/tests/test_py311_generated/test_dcargs_generated.py +++ b/tests/test_py311_generated/test_dcargs_generated.py @@ -316,10 +316,10 @@ class A: def test_enum_values() -> None: - class Color(enum.StrEnum): - RED = enum.auto() - GREEN = enum.auto() - BLUE = enum.auto() + class Color(enum.Enum): + RED = "red" + GREEN = "green" + BLUE = "blue" @dataclasses.dataclass class EnumClassA: @@ -335,6 +335,28 @@ class EnumClassB: assert tyro.cli(EnumClassA, args=["--color", "RED"]) +def test_enum_values_ints() -> None: + class Color(enum.Enum): + RED = 0 + GREEN = 1 + BLUE = 2 + + @dataclasses.dataclass + class EnumClassA: + color: Annotated[Color, tyro.conf.EnumChoicesFromValues] + + @dataclasses.dataclass + class EnumClassB: + color: Annotated[Color, tyro.conf.EnumChoicesFromValues] = Color.GREEN + + assert tyro.cli(EnumClassA, args=["--color", "0"]) == EnumClassA(color=Color.RED) + assert tyro.cli(EnumClassB, args=[]) == EnumClassB() + with pytest.raises(SystemExit): + assert tyro.cli(EnumClassA, args=["--color", "red"]) + with pytest.raises(SystemExit): + assert tyro.cli(EnumClassA, args=["--color", "RED"]) + + def test_literal() -> None: @dataclasses.dataclass class A: @@ -406,10 +428,10 @@ class A: def test_literal_enum_values() -> None: - class Color(enum.StrEnum): - RED = enum.auto() - GREEN = enum.auto() - BLUE = enum.auto() + class Color(enum.Enum): + RED = "red" + GREEN = "green" + BLUE = "blue" @dataclasses.dataclass class A: diff --git a/tests/test_py311_generated/test_decorator_subcommands_generated.py b/tests/test_py311_generated/test_decorator_subcommands_generated.py index 8a623bc3a..0ed47227c 100644 --- a/tests/test_py311_generated/test_decorator_subcommands_generated.py +++ b/tests/test_py311_generated/test_decorator_subcommands_generated.py @@ -3,8 +3,10 @@ from tyro.extras import SubcommandApp app = SubcommandApp() +app_just_one = SubcommandApp() +@app_just_one.command @app.command def greet(name: str, loud: bool = False) -> None: """Greet someone.""" @@ -20,6 +22,17 @@ def add(a: int, b: int) -> None: print(f"{a} + {b} = {a + b}") +def test_app_just_one_cli(capsys): + # Test: `python my_script.py --help` + with pytest.raises(SystemExit): + app_just_one.cli(args=["--help"]) + captured = capsys.readouterr() + assert "usage: " in captured.out + assert "greet" not in captured.out + assert "addition" not in captured.out + assert "--name" in captured.out + + def test_app_cli(capsys): # Test: `python my_script.py --help` with pytest.raises(SystemExit): @@ -31,13 +44,13 @@ def test_app_cli(capsys): # Test: `python my_script.py greet --help` with pytest.raises(SystemExit): - app.cli(args=["greet", "--help"]) + app.cli(args=["greet", "--help"], sort_subcommands=False) captured = capsys.readouterr() assert "usage: " in captured.out assert "Greet someone." in captured.out # Test: `python my_script.py greet --name Alice` - app.cli(args=["greet", "--name", "Alice"]) + app.cli(args=["greet", "--name", "Alice"], sort_subcommands=True) captured = capsys.readouterr() assert captured.out.strip() == "Hello, Alice!" diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py index cce8a5c70..f509d3cbc 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py @@ -1,3 +1,7 @@ +# mypy: ignore-errors +# +# We can remove this ignore after: https://peps.python.org/pep-0747/ + import dataclasses from typing import Any, Literal, Type @@ -6,12 +10,12 @@ import tyro -def test_union_direct(): +def test_union_direct() -> None: assert tyro.cli(int | str, args=["5"]) == 5 assert tyro.cli(int | str, args=["five"]) == "five" -def test_union_basic(): +def test_union_basic() -> None: def main(x: int | str) -> int | str: return x @@ -20,7 +24,7 @@ def main(x: int | str) -> int | str: assert tyro.cli(main, args=["--x", "five"]) == "five" -def test_union_with_list(): +def test_union_with_list() -> None: def main(x: int | str | list[bool]) -> Any: return x @@ -31,7 +35,7 @@ def main(x: int | str | list[bool]) -> Any: assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False] -def test_union_literal(): +def test_union_literal() -> None: def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: return x @@ -40,7 +44,7 @@ def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: assert tyro.cli(main, args=["--x", "five"]) == "five" -def test_super_nested(): +def test_super_nested() -> None: def main( x: None | list[ @@ -65,7 +69,7 @@ def main( tyro.cli(main, args=["--help"]) -def test_type(): +def test_type() -> None: """Test adapted from mirceamironenco: https://github.com/brentyi/tyro/issues/164""" class Thing: ... @@ -81,7 +85,7 @@ class Config: assert tyro.cli(Config, args=["--foo", "5"]) == Config(5, SubThing, SubThing) -def test_type_default_factory(): +def test_type_default_factory() -> None: """Test adapted from mirceamironenco: https://github.com/brentyi/tyro/issues/164""" @dataclasses.dataclass diff --git a/tests/test_py311_generated/test_repeat_suppressed_generated.py b/tests/test_py311_generated/test_repeat_suppressed_generated.py new file mode 100644 index 000000000..e238637f5 --- /dev/null +++ b/tests/test_py311_generated/test_repeat_suppressed_generated.py @@ -0,0 +1,38 @@ +"""Adapted from @mirceamironenco: https://github.com/brentyi/tyro/issues/170""" + +from dataclasses import dataclass + +import tyro + + +class LayerAExample: + def __init__(self, **kwargs): ... + + +@dataclass +class LayerAConfig: + _target: type = LayerAExample + foo: int = 13 + + +class LayerBExample: + def __init__(self, **kwargs): ... + + +@dataclass +class LayerBConfig: + _target: type = LayerBExample + bar: int = 13 + + +@dataclass +class BlockConfig: + layer_a: LayerAConfig + layer_b: LayerBConfig + + +def test_repeat_suppressed() -> None: + assert tyro.cli( + tyro.conf.OmitArgPrefixes[tyro.conf.SuppressFixed[BlockConfig]], + args="--foo 14".split(" "), + ) == BlockConfig(LayerAConfig(foo=14), LayerBConfig()) diff --git a/tests/test_repeat_suppressed.py b/tests/test_repeat_suppressed.py new file mode 100644 index 000000000..e238637f5 --- /dev/null +++ b/tests/test_repeat_suppressed.py @@ -0,0 +1,38 @@ +"""Adapted from @mirceamironenco: https://github.com/brentyi/tyro/issues/170""" + +from dataclasses import dataclass + +import tyro + + +class LayerAExample: + def __init__(self, **kwargs): ... + + +@dataclass +class LayerAConfig: + _target: type = LayerAExample + foo: int = 13 + + +class LayerBExample: + def __init__(self, **kwargs): ... + + +@dataclass +class LayerBConfig: + _target: type = LayerBExample + bar: int = 13 + + +@dataclass +class BlockConfig: + layer_a: LayerAConfig + layer_b: LayerBConfig + + +def test_repeat_suppressed() -> None: + assert tyro.cli( + tyro.conf.OmitArgPrefixes[tyro.conf.SuppressFixed[BlockConfig]], + args="--foo 14".split(" "), + ) == BlockConfig(LayerAConfig(foo=14), LayerBConfig()) From a867851ddaa9c23eb2f82b955bc2a570eb3563fb Mon Sep 17 00:00:00 2001 From: brentyi Date: Fri, 11 Oct 2024 01:10:57 -0700 Subject: [PATCH 456/491] Docs fix, gtag --- docs/requirements.txt | 1 + docs/source/conf.py | 4 ++++ docs/source/examples/04_additional/11_custom_constructors.rst | 4 ++-- examples/04_additional/11_custom_constructors.py | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 7260ab5aa..241369d0c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,3 +5,4 @@ sphinx-autoapi==3.0.0 m2r2==0.3.3.post2 git+https://github.com/brentyi/sphinxcontrib-programoutput.git git+https://github.com/brentyi/ansi.git +sphinxcontrib-googleanalytics==0.4 diff --git a/docs/source/conf.py b/docs/source/conf.py index 53f889d5d..1ac56224a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,6 +54,7 @@ "m2r2", "sphinxcontrib.programoutput", "sphinxcontrib.ansi", + "sphinxcontrib.googleanalytics", ] programoutput_use_ansi = True html_ansi_stylesheet = "black-on-white.css" @@ -212,6 +213,9 @@ # -- Extension configuration -------------------------------------------------- +# -- Google analytics ID ------------------------------------------------------ +googleanalytics_id = "G-624W9VWZWK" + # -- Options for autoapi extension -------------------------------------------- autoapi_dirs = ["../../src/tyro"] autoapi_root = "api" diff --git a/docs/source/examples/04_additional/11_custom_constructors.rst b/docs/source/examples/04_additional/11_custom_constructors.rst index 3d91faa9a..27af528ee 100644 --- a/docs/source/examples/04_additional/11_custom_constructors.rst +++ b/docs/source/examples/04_additional/11_custom_constructors.rst @@ -90,6 +90,6 @@ For additional flexibility, :func:`tyro.conf.arg()` accepts a .. raw:: html - python 04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}`' --dict2.json '{"hello": "world"}' + python 04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' -.. program-output:: python ../../examples/04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}`' --dict2.json '{"hello": "world"}' +.. program-output:: python ../../examples/04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' diff --git a/examples/04_additional/11_custom_constructors.py b/examples/04_additional/11_custom_constructors.py index 0bbcebb2d..2c314b9d2 100644 --- a/examples/04_additional/11_custom_constructors.py +++ b/examples/04_additional/11_custom_constructors.py @@ -34,7 +34,7 @@ Usage: `python ./10_custom_constructors.py --help` `python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}"` -`python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}"` --dict2.json "{\"hello\": \"world\"}"` +`python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}" --dict2.json "{\"hello\": \"world\"}"` """ import json as json_ From 34f3b143baab3acab09d3c2b8c2920d02dcf29be Mon Sep 17 00:00:00 2001 From: Mircea Mironenco Date: Fri, 11 Oct 2024 19:30:14 +0300 Subject: [PATCH 457/491] Allow subcommand_cli_from_dict to specify console_outputs (#175) * Add console_outputs option to subcommand_cli_from_dict * Add console_outputs test for cli_from_dict. * Fix formatting. --- src/tyro/extras/_subcommand_cli_from_dict.py | 8 ++++++++ tests/test_errors.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/tyro/extras/_subcommand_cli_from_dict.py b/src/tyro/extras/_subcommand_cli_from_dict.py index 6710be40f..4c17c7258 100644 --- a/src/tyro/extras/_subcommand_cli_from_dict.py +++ b/src/tyro/extras/_subcommand_cli_from_dict.py @@ -16,6 +16,7 @@ def subcommand_cli_from_dict( description: Optional[str] = None, args: Optional[Sequence[str]] = None, use_underscores: bool = False, + console_outputs: bool = True, ) -> T: ... @@ -29,6 +30,7 @@ def subcommand_cli_from_dict( description: Optional[str] = None, args: Optional[Sequence[str]] = None, use_underscores: bool = False, + console_outputs: bool = True, ) -> Any: ... @@ -39,6 +41,7 @@ def subcommand_cli_from_dict( description: Optional[str] = None, args: Optional[Sequence[str]] = None, use_underscores: bool = False, + console_outputs: bool = True, ) -> Any: """Generate a subcommand CLI from a dictionary of functions. @@ -86,6 +89,10 @@ def subcommand_cli_from_dict( This primarily impacts helptext; underscores and hyphens are treated equivalently when parsing happens. We default helptext to hyphens to follow the GNU style guide. https://www.gnu.org/software/libc/manual/html_node/Argument-Syntax.html + console_outputs: If set to `False`, parsing errors and help messages will be + supressed. This can be useful for distributed settings, where `tyro.cli()` + is called from multiple workers but we only want console outputs from the + main one. """ # We need to form a union type, which requires at least two elements. assert len(subcommands) >= 2, "At least two subcommands are required." @@ -108,4 +115,5 @@ def subcommand_cli_from_dict( description=description, args=args, use_underscores=use_underscores, + console_outputs=console_outputs ) diff --git a/tests/test_errors.py b/tests/test_errors.py index 839947968..3a7abcc7d 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -204,6 +204,25 @@ class Class: assert error == "" +def test_suppress_console_outputs_fromdict() -> None: + def foo(track: bool) -> None: + print(track) + + def bar(track: bool) -> None: + print(track) + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): + tyro.extras.subcommand_cli_from_dict( + {"foo": foo, "bar": bar}, + args="foo --reward.trac".split(" "), + console_outputs=False, + ) + + error = target.getvalue() + assert error == "" + + def test_similar_arguments_subcommands() -> None: @dataclasses.dataclass class RewardConfig: From 8828943f94e1d4776d427f2f7cd7cf2b3d583786 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 11 Oct 2024 10:01:00 -0700 Subject: [PATCH 458/491] More complete wrappers in `tyro.extras` (for `torchrun`, etc) (#176) --- src/tyro/extras/_subcommand_app.py | 9 +++++++++ src/tyro/extras/_subcommand_cli_from_dict.py | 9 ++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/tyro/extras/_subcommand_app.py b/src/tyro/extras/_subcommand_app.py index f5497188c..e0cac9ddc 100644 --- a/src/tyro/extras/_subcommand_app.py +++ b/src/tyro/extras/_subcommand_app.py @@ -91,6 +91,8 @@ def cli( description: Optional[str] = None, args: Optional[Sequence[str]] = None, use_underscores: bool = False, + console_outputs: bool = True, + config: Optional[Sequence[Any]] = None, sort_subcommands: bool = False, ) -> Any: """Run the command-line interface. @@ -110,6 +112,9 @@ def cli( This primarily impacts helptext; underscores and hyphens are treated equivalently when parsing happens. We default helptext to hyphens to follow the GNU style guide. https://www.gnu.org/software/libc/manual/html_node/Argument-Syntax.html + console_outputs: If set to `False`, parsing errors and help messages will be + suppressed. + config: Sequence of config marker objects, from `tyro.conf`. sort_subcommands: If True, sort the subcommands alphabetically by name. """ assert self._subcommands is not None @@ -129,6 +134,8 @@ def cli( description=description, args=args, use_underscores=use_underscores, + console_outputs=console_outputs, + config=config, ) else: return tyro.extras.subcommand_cli_from_dict( @@ -137,4 +144,6 @@ def cli( description=description, args=args, use_underscores=use_underscores, + console_outputs=console_outputs, + config=config, ) diff --git a/src/tyro/extras/_subcommand_cli_from_dict.py b/src/tyro/extras/_subcommand_cli_from_dict.py index 4c17c7258..038d3f635 100644 --- a/src/tyro/extras/_subcommand_cli_from_dict.py +++ b/src/tyro/extras/_subcommand_cli_from_dict.py @@ -2,6 +2,8 @@ from typing_extensions import Annotated +from tyro.conf._markers import Marker + from .._cli import cli from ..conf import subcommand @@ -17,6 +19,7 @@ def subcommand_cli_from_dict( args: Optional[Sequence[str]] = None, use_underscores: bool = False, console_outputs: bool = True, + config: Optional[Sequence[Marker]] = None, ) -> T: ... @@ -31,6 +34,7 @@ def subcommand_cli_from_dict( args: Optional[Sequence[str]] = None, use_underscores: bool = False, console_outputs: bool = True, + config: Optional[Sequence[Marker]] = None, ) -> Any: ... @@ -42,6 +46,7 @@ def subcommand_cli_from_dict( args: Optional[Sequence[str]] = None, use_underscores: bool = False, console_outputs: bool = True, + config: Optional[Sequence[Marker]] = None, ) -> Any: """Generate a subcommand CLI from a dictionary of functions. @@ -93,6 +98,7 @@ def subcommand_cli_from_dict( supressed. This can be useful for distributed settings, where `tyro.cli()` is called from multiple workers but we only want console outputs from the main one. + config: Sequence of config marker objects, from `tyro.conf`. """ # We need to form a union type, which requires at least two elements. assert len(subcommands) >= 2, "At least two subcommands are required." @@ -115,5 +121,6 @@ def subcommand_cli_from_dict( description=description, args=args, use_underscores=use_underscores, - console_outputs=console_outputs + console_outputs=console_outputs, + config=config, ) From 3f055032ce2a744dbd95887d50e421491833194b Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 18 Oct 2024 22:01:03 -0700 Subject: [PATCH 459/491] Fix edge case when combining PEP 695 aliases with `typing.Annotated[]` (#178) * Add failing test * Resolve aliases when unwrapping types with runtime annotations --- src/tyro/_fields.py | 8 ++++---- src/tyro/_instantiators.py | 2 +- src/tyro/_parsers.py | 18 ++++++++++------- src/tyro/_resolver.py | 20 +++++++++++-------- src/tyro/_strings.py | 2 +- src/tyro/_subcommand_matching.py | 4 ++-- src/tyro/extras/_serialization.py | 2 +- tests/test_new_style_annotations_min_py312.py | 12 +++++++++++ .../test_errors_generated.py | 19 ++++++++++++++++++ ...w_style_annotations_min_py312_generated.py | 12 +++++++++++ 10 files changed, 75 insertions(+), 24 deletions(-) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index d1257ea98..dd503b53b 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -110,7 +110,7 @@ def make( markers: Tuple[_markers.Marker, ...] = (), ): # Try to extract argconf overrides from type. - _, argconfs = _resolver.unwrap_annotated( + _, argconfs = _resolver.unwrap_annotated_and_aliases( type_or_callable, _confstruct._ArgConfiguration ) argconf = _confstruct._ArgConfiguration( @@ -135,7 +135,7 @@ def make( if argconf.help is not None: helptext = argconf.help - type_or_callable, inferred_markers = _resolver.unwrap_annotated( + type_or_callable, inferred_markers = _resolver.unwrap_annotated_and_aliases( type_or_callable, _markers._Marker ) markers = inferred_markers + markers @@ -285,7 +285,7 @@ def field_list_from_callable( # Try to generate field list. # We recursively apply markers. - _, parent_markers = _resolver.unwrap_annotated(f, _markers._Marker) + _, parent_markers = _resolver.unwrap_annotated_and_aliases(f, _markers._Marker) with FieldDefinition.marker_context(parent_markers): field_list = _try_field_list_from_callable(f, default_instance) @@ -386,7 +386,7 @@ def _try_field_list_from_callable( # Check for default instances in subcommand configs. This is needed for # is_nested_type() when arguments are not valid without a default, and this # default is specified in the subcommand config. - f, found_subcommand_configs = _resolver.unwrap_annotated( + f, found_subcommand_configs = _resolver.unwrap_annotated_and_aliases( f, conf._confstruct._SubcommandConfiguration ) if len(found_subcommand_configs) > 0: diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index 1b8e6ed98..af1b40b73 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -197,7 +197,7 @@ def instantiator(strings: List[str]) -> None: if maybe_newtype_name is not None: metavar = maybe_newtype_name.upper() - typ = _resolver.unwrap_annotated(typ) + typ = _resolver.unwrap_annotated_and_aliases(typ) # Address container types. If a matching container is found, this will recursively # call instantiator_from_type(). diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index c0d711062..47b461474 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -75,7 +75,7 @@ def from_callable_or_type( """Create a parser definition from a callable or type.""" # Consolidate subcommand types. - markers = _resolver.unwrap_annotated(f, _markers._Marker)[1] + markers = _resolver.unwrap_annotated_and_aliases(f, _markers._Marker)[1] consolidate_subcommand_args = _markers.ConsolidateSubcommandArgs in markers # Resolve the type of `f`, generate a field list. @@ -387,7 +387,7 @@ def from_field( extern_prefix: str, ) -> Optional[SubparsersSpecification]: # Union of classes should create subparsers. - typ = _resolver.unwrap_annotated(field.type_or_callable) + typ = _resolver.unwrap_annotated_and_aliases(field.type_or_callable) if get_origin(typ) is not Union: return None @@ -407,7 +407,7 @@ def from_field( # If specified, swap types using tyro.conf.subcommand(constructor=...). for i, option in enumerate(options): - _, found_subcommand_configs = _resolver.unwrap_annotated( + _, found_subcommand_configs = _resolver.unwrap_annotated_and_aliases( option, _confstruct._SubcommandConfiguration ) if ( @@ -417,7 +417,7 @@ def from_field( options[i] = Annotated.__class_getitem__( # type: ignore ( found_subcommand_configs[0].constructor_factory(), - *_resolver.unwrap_annotated(option, "all")[1], + *_resolver.unwrap_annotated_and_aliases(option, "all")[1], ) ) @@ -443,8 +443,10 @@ def from_field( else extern_prefix, type(None) if option is none_proxy else cast(type, option), ) - option_unwrapped, found_subcommand_configs = _resolver.unwrap_annotated( - option, _confstruct._SubcommandConfiguration + option_unwrapped, found_subcommand_configs = ( + _resolver.unwrap_annotated_and_aliases( + option, _confstruct._SubcommandConfiguration + ) ) if len(found_subcommand_configs) != 0: # Explicitly annotated default. @@ -508,7 +510,9 @@ def from_field( # Strip the subcommand config from the option type. # Relevant: https://github.com/brentyi/tyro/pull/117 - option_origin, annotations = _resolver.unwrap_annotated(option, "all") + option_origin, annotations = _resolver.unwrap_annotated_and_aliases( + option, "all" + ) annotations = tuple( a for a in annotations diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index b426f16eb..25f231359 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -47,7 +47,7 @@ def unwrap_origin_strip_extras(typ: TypeOrCallable) -> TypeOrCallable: """Returns the origin, ignoring typing.Annotated, of typ if it exists. Otherwise, returns typ.""" # TODO: Annotated[] handling should be revisited... - typ = unwrap_annotated(typ) + typ = unwrap_annotated_and_aliases(typ) origin = get_origin(typ) if origin is not None: @@ -72,7 +72,7 @@ def resolve_generic_types( # ^We need this `if` statement for an obscure edge case: when `cls` is a # function with `__tyro_markers__` set, we don't want/need to return # Annotated[func, markers]. - cls, annotations = unwrap_annotated(cls, "all") + cls, annotations = unwrap_annotated_and_aliases(cls, "all") # We'll ignore NewType when getting the origin + args for generics. origin_cls = get_origin(unwrap_newtype_and_aliases(cls)[0]) @@ -218,7 +218,7 @@ def unwrap_newtype_and_narrow_subtypes( # it doesn't really make sense to parse this case. return typ - superclass = unwrap_annotated(typ) + superclass = unwrap_annotated_and_aliases(typ) # For Python 3.10. if get_origin(superclass) is Union: @@ -243,7 +243,7 @@ def swap_type_using_confstruct(typ: TypeOrCallable) -> TypeOrCallable: `tyro.conf.arg` and `tyro.conf.subcommand`. Runtime annotations are kept, but the type is swapped.""" # Need to swap types. - _, annotations = unwrap_annotated(typ, search_type="all") + _, annotations = unwrap_annotated_and_aliases(typ, search_type="all") for anno in reversed(annotations): if ( isinstance( @@ -305,27 +305,27 @@ def narrow_collection_types( @overload -def unwrap_annotated( +def unwrap_annotated_and_aliases( typ: TypeOrCallable, search_type: TypeForm[MetadataType], ) -> Tuple[TypeOrCallable, Tuple[MetadataType, ...]]: ... @overload -def unwrap_annotated( +def unwrap_annotated_and_aliases( typ: TypeOrCallable, search_type: Literal["all"], ) -> Tuple[TypeOrCallable, Tuple[Any, ...]]: ... @overload -def unwrap_annotated( +def unwrap_annotated_and_aliases( typ: TypeOrCallable, search_type: None = None, ) -> TypeOrCallable: ... -def unwrap_annotated( +def unwrap_annotated_and_aliases( typ: TypeOrCallable, search_type: Union[TypeForm[MetadataType], Literal["all"], object, None] = None, ) -> Union[Tuple[TypeOrCallable, Tuple[MetadataType, ...]], TypeOrCallable]: @@ -337,6 +337,10 @@ def unwrap_annotated( - Annotated[int, "1"], int => (int, ()) """ + # Unwrap aliases defined using Python 3.12's `type` syntax. + if isinstance(typ, TypeAliasType): + return unwrap_annotated_and_aliases(typ.__value__, search_type) + # `Final` and `ReadOnly` types are ignored in tyro. while get_origin(typ) in STRIP_WRAPPER_TYPES: typ = get_args(typ)[0] diff --git a/src/tyro/_strings.py b/src/tyro/_strings.py index e2fc7c09f..9a3953e7d 100644 --- a/src/tyro/_strings.py +++ b/src/tyro/_strings.py @@ -80,7 +80,7 @@ def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: from .conf import _confstruct # Prevent circular imports cls, type_from_typevar = _resolver.resolve_generic_types(cls) - cls, found_subcommand_configs = _resolver.unwrap_annotated( + cls, found_subcommand_configs = _resolver.unwrap_annotated_and_aliases( cls, _confstruct._SubcommandConfiguration ) diff --git a/src/tyro/_subcommand_matching.py b/src/tyro/_subcommand_matching.py index e9a45a425..ff21e4817 100644 --- a/src/tyro/_subcommand_matching.py +++ b/src/tyro/_subcommand_matching.py @@ -96,11 +96,11 @@ def _get_type_options(typ: _typing.TypeForm) -> Tuple[_typing.TypeForm, ...]: # Check against supertypes. for self_type in self_types: - self_type = _resolver.unwrap_annotated(self_type) + self_type = _resolver.unwrap_annotated_and_aliases(self_type) self_type, _ = _resolver.unwrap_newtype_and_aliases(self_type) ok = False for super_type in super_types: - super_type = _resolver.unwrap_annotated(super_type) + super_type = _resolver.unwrap_annotated_and_aliases(super_type) self_type, _ = _resolver.unwrap_newtype_and_aliases(self_type) if issubclass(self_type, super_type): ok = True diff --git a/src/tyro/extras/_serialization.py b/src/tyro/extras/_serialization.py index bfb5a28d2..ae77abb40 100644 --- a/src/tyro/extras/_serialization.py +++ b/src/tyro/extras/_serialization.py @@ -35,7 +35,7 @@ def _get_contained_special_types_from_type( else _parent_contained_dataclasses ) - cls = _resolver.unwrap_annotated(cls) + cls = _resolver.unwrap_annotated_and_aliases(cls) cls, type_from_typevar = _resolver.resolve_generic_types(cls) contained_special_types = {cls} diff --git a/tests/test_new_style_annotations_min_py312.py b/tests/test_new_style_annotations_min_py312.py index d04c73f3b..9189e585c 100644 --- a/tests/test_new_style_annotations_min_py312.py +++ b/tests/test_new_style_annotations_min_py312.py @@ -2,6 +2,7 @@ # # PEP 695 isn't yet supported in mypy. (April 4, 2024) from dataclasses import dataclass +from typing import Annotated import tyro @@ -52,3 +53,14 @@ class Container[T]: assert tyro.cli(Container[Y], args="--a.a 1 --a.b 2".split(" ")) == Container( Inner(1, 2) ) + + +type AnnotatedBasic = Annotated[int, tyro.conf.arg(name="basic")] + + +def test_annotated_alias(): + @dataclass(frozen=True) + class Container: + a: AnnotatedBasic + + assert tyro.cli(Container, args="--basic 1".split(" ")) == Container(1) diff --git a/tests/test_py311_generated/test_errors_generated.py b/tests/test_py311_generated/test_errors_generated.py index b7a9f5cfc..cd5f7b866 100644 --- a/tests/test_py311_generated/test_errors_generated.py +++ b/tests/test_py311_generated/test_errors_generated.py @@ -203,6 +203,25 @@ class Class: assert error == "" +def test_suppress_console_outputs_fromdict() -> None: + def foo(track: bool) -> None: + print(track) + + def bar(track: bool) -> None: + print(track) + + target = io.StringIO() + with pytest.raises(SystemExit), contextlib.redirect_stderr(target): + tyro.extras.subcommand_cli_from_dict( + {"foo": foo, "bar": bar}, + args="foo --reward.trac".split(" "), + console_outputs=False, + ) + + error = target.getvalue() + assert error == "" + + def test_similar_arguments_subcommands() -> None: @dataclasses.dataclass class RewardConfig: diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py index d04c73f3b..9189e585c 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py @@ -2,6 +2,7 @@ # # PEP 695 isn't yet supported in mypy. (April 4, 2024) from dataclasses import dataclass +from typing import Annotated import tyro @@ -52,3 +53,14 @@ class Container[T]: assert tyro.cli(Container[Y], args="--a.a 1 --a.b 2".split(" ")) == Container( Inner(1, 2) ) + + +type AnnotatedBasic = Annotated[int, tyro.conf.arg(name="basic")] + + +def test_annotated_alias(): + @dataclass(frozen=True) + class Container: + a: AnnotatedBasic + + assert tyro.cli(Container, args="--basic 1".split(" ")) == Container(1) From 89ff57aff81afb26d9bc588abfeb418db8076815 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Fri, 18 Oct 2024 23:12:23 -0700 Subject: [PATCH 460/491] Fix generic aliases generated by Python 3.12 `type` statements (#179) * Fix generic aliases generated by Python 3.12 `type` statements * Add fix * ruff * ruff + pyright * Fix GenericAlias import --- src/tyro/_fields.py | 3 +++ src/tyro/_resolver.py | 15 +++++++++++++-- tests/test_new_style_annotations_min_py312.py | 13 +++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index dd503b53b..a9e59b143 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -109,6 +109,9 @@ def make( *, markers: Tuple[_markers.Marker, ...] = (), ): + # Resolve generic aliases. + type_or_callable = _resolver.apply_type_from_typevar(type_or_callable, {}) + # Try to extract argconf overrides from type. _, argconfs = _resolver.unwrap_annotated_and_aliases( type_or_callable, _confstruct._ArgConfiguration diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 25f231359..b1b76467e 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -392,6 +392,17 @@ def unwrap_annotated_and_aliases( def apply_type_from_typevar( typ: TypeOrCallable, type_from_typevar: Dict[TypeVar, TypeForm[Any]] ) -> TypeOrCallable: + GenericAlias = getattr(types, "GenericAlias", None) + if ( + GenericAlias is not None + and isinstance(typ, GenericAlias) + and len(getattr(typ, "__type_params__", ())) > 0 + ): + type_from_typevar = type_from_typevar.copy() + for k, v in zip(typ.__type_params__, typ.__args__): # type: ignore + type_from_typevar[k] = v # type: ignore + typ = typ.__value__ # type: ignore + if typ in type_from_typevar: return type_from_typevar[typ] # type: ignore @@ -405,7 +416,7 @@ def apply_type_from_typevar( args = tuple(args[0]) + args[1:] # Convert Python 3.9 and 3.10 types to their typing library equivalents, which - # support `.copy_with()`. + # support `.copy_with()`. This is not really the right place for this logic... if sys.version_info[:2] >= (3, 9): shim_table = { # PEP 585. Requires Python 3.9. @@ -434,7 +445,7 @@ def apply_type_from_typevar( assert hasattr(origin, "__class_getitem__") return origin.__class_getitem__(new_args) # type: ignore - return typ + return typ # type: ignore @_unsafe_cache.unsafe_cache(maxsize=1024) diff --git a/tests/test_new_style_annotations_min_py312.py b/tests/test_new_style_annotations_min_py312.py index 9189e585c..120d5d09b 100644 --- a/tests/test_new_style_annotations_min_py312.py +++ b/tests/test_new_style_annotations_min_py312.py @@ -64,3 +64,16 @@ class Container: a: AnnotatedBasic assert tyro.cli(Container, args="--basic 1".split(" ")) == Container(1) + + +type TT[T] = Annotated[T, tyro.conf.arg(name="", constructor=lambda: True)] + + +def test_pep695_generic_alias() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: TT[bool] + + assert tyro.cli(Config, args=[]) == Config(arg=True) From bd54f7731d1575681edc66e63432ebffd0e6c076 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 19 Oct 2024 12:23:29 -0700 Subject: [PATCH 461/491] (refactor) Context-based resolution for generic types (#180) * Start refactor (broken) * Fixes * Fix generic shim edge case * New alias tests, edge case fixes * Add type ignore * CI fixes * Revert example change * Fix example numbering * Appease mypy * Remove unnecessary type alias unwrap --- .../{12_counters.rst => 13_counters.rst} | 16 +- ...pe_statement.rst => 16_type_statement.rst} | 4 +- .../{12_counters.py => 13_counters.py} | 0 ...type_statement.py => 16_type_statement.py} | 0 src/tyro/_arguments.py | 13 +- src/tyro/_cli.py | 195 ++++++------ src/tyro/_docstrings.py | 2 +- src/tyro/_fields.py | 278 +++++++++-------- src/tyro/_instantiators.py | 39 +-- src/tyro/_parsers.py | 38 +-- src/tyro/_resolver.py | 292 +++++++++++------- src/tyro/_subcommand_matching.py | 2 +- tests/test_new_style_annotations_min_py312.py | 13 + ...w_style_annotations_min_py312_generated.py | 26 ++ ...subcommands_aliases_min_py312_generated.py | 164 ++++++++++ ...uple_with_subcommands_aliases_min_py312.py | 166 ++++++++++ 16 files changed, 830 insertions(+), 418 deletions(-) rename docs/source/examples/04_additional/{12_counters.rst => 13_counters.rst} (70%) rename docs/source/examples/04_additional/{13_type_statement.rst => 16_type_statement.rst} (88%) rename examples/04_additional/{12_counters.py => 13_counters.py} (100%) rename examples/04_additional/{13_type_statement.py => 16_type_statement.py} (100%) create mode 100644 tests/test_py311_generated/test_tuple_with_subcommands_aliases_min_py312_generated.py create mode 100644 tests/test_tuple_with_subcommands_aliases_min_py312.py diff --git a/docs/source/examples/04_additional/12_counters.rst b/docs/source/examples/04_additional/13_counters.rst similarity index 70% rename from docs/source/examples/04_additional/12_counters.rst rename to docs/source/examples/04_additional/13_counters.rst index a05decc34..4068f1375 100644 --- a/docs/source/examples/04_additional/12_counters.rst +++ b/docs/source/examples/04_additional/13_counters.rst @@ -38,30 +38,30 @@ Repeatable 'counter' arguments can be specified via :data:`tyro.conf.UseCounterA .. raw:: html - python 04_additional/12_counters.py --help + python 04_additional/13_counters.py --help -.. program-output:: python ../../examples/04_additional/12_counters.py --help +.. program-output:: python ../../examples/04_additional/13_counters.py --help ------------ .. raw:: html - python 04_additional/12_counters.py --verbosity + python 04_additional/13_counters.py --verbosity -.. program-output:: python ../../examples/04_additional/12_counters.py --verbosity +.. program-output:: python ../../examples/04_additional/13_counters.py --verbosity ------------ .. raw:: html - python 04_additional/12_counters.py --verbosity --verbosity + python 04_additional/13_counters.py --verbosity --verbosity -.. program-output:: python ../../examples/04_additional/12_counters.py --verbosity --verbosity +.. program-output:: python ../../examples/04_additional/13_counters.py --verbosity --verbosity ------------ .. raw:: html - python 04_additional/12_counters.py -vvv + python 04_additional/13_counters.py -vvv -.. program-output:: python ../../examples/04_additional/12_counters.py -vvv +.. program-output:: python ../../examples/04_additional/13_counters.py -vvv diff --git a/docs/source/examples/04_additional/13_type_statement.rst b/docs/source/examples/04_additional/16_type_statement.rst similarity index 88% rename from docs/source/examples/04_additional/13_type_statement.rst rename to docs/source/examples/04_additional/16_type_statement.rst index e568a1102..8fe3d666e 100644 --- a/docs/source/examples/04_additional/13_type_statement.rst +++ b/docs/source/examples/04_additional/16_type_statement.rst @@ -45,6 +45,6 @@ In Python 3.12, the :code:`type` statement is introduced to create type aliases. .. raw:: html - python 04_additional/13_type_statement.py --help + python 04_additional/16_type_statement.py --help -.. program-output:: python ../../examples/04_additional/13_type_statement.py --help +.. program-output:: python ../../examples/04_additional/16_type_statement.py --help diff --git a/examples/04_additional/12_counters.py b/examples/04_additional/13_counters.py similarity index 100% rename from examples/04_additional/12_counters.py rename to examples/04_additional/13_counters.py diff --git a/examples/04_additional/13_type_statement.py b/examples/04_additional/16_type_statement.py similarity index 100% rename from examples/04_additional/13_type_statement.py rename to examples/04_additional/16_type_statement.py diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index 40bce6745..1dc29e986 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -12,7 +12,6 @@ TYPE_CHECKING, Any, Callable, - Dict, Iterable, Mapping, Optional, @@ -27,8 +26,7 @@ import shtab from . import _argparse as argparse -from . import _fields, _instantiators, _resolver, _strings -from ._typing import TypeForm +from . import _fields, _instantiators, _strings from .conf import _markers if TYPE_CHECKING: @@ -108,7 +106,6 @@ class ArgumentDefinition: extern_prefix: str # User-facing prefix. subcommand_prefix: str # Prefix for nesting. field: _fields.FieldDefinition - type_from_typevar: Dict[TypeVar, TypeForm[Any]] def add_argument( self, parser: Union[argparse.ArgumentParser, argparse._ArgumentGroup] @@ -254,12 +251,7 @@ def _rule_handle_boolean_flags( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> None: - if ( - _resolver.apply_type_from_typevar( - arg.field.type_or_callable, arg.type_from_typevar - ) - is not bool - ): + if arg.field.type_or_callable is not bool: return if ( @@ -305,7 +297,6 @@ def _rule_recursive_instantiator_from_type( try: instantiator, metadata = _instantiators.instantiator_from_type( arg.field.type_or_callable, - arg.type_from_typevar, arg.field.markers, ) except _instantiators.UnsupportedTypeAnnotationError as e: diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index ff01894cf..760d78e86 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -20,6 +20,8 @@ import shtab from typing_extensions import Literal +from tyro._resolver import TypeParamResolver + from . import _argparse as argparse from . import ( _argparse_formatter, @@ -316,104 +318,109 @@ def _cli_impl( stacklevel=2, ) - # Internally, we distinguish between two concepts: - # - "default", which is used for individual arguments. - # - "default_instance", which is used for _fields_ (which may be broken down into - # one or many arguments, depending on various factors). - # - # This could be revisited. - default_instance_internal: Union[_fields.NonpropagatingMissingType, OutT] = ( - _fields.MISSING_NONPROP if default is None else default - ) - - # We wrap our type with a dummy dataclass if it can't be treated as a nested type. - # For example: passing in f=int will result in a dataclass with a single field - # typed as int. - if not _fields.is_nested_type(cast(type, f), default_instance_internal): - dummy_field = cast( - dataclasses.Field, - dataclasses.field(), + resolve_context = TypeParamResolver.get_assignment_context(f) + with resolve_context: + f = resolve_context.origin_type + + # Internally, we distinguish between two concepts: + # - "default", which is used for individual arguments. + # - "default_instance", which is used for _fields_ (which may be broken down into + # one or many arguments, depending on various factors). + # + # This could be revisited. + default_instance_internal: Union[_fields.NonpropagatingMissingType, OutT] = ( + _fields.MISSING_NONPROP if default is None else default ) - f = dataclasses.make_dataclass( - cls_name="dummy", - fields=[(_strings.dummy_field_name, cast(type, f), dummy_field)], - frozen=True, - ) - default_instance_internal = f(default_instance_internal) # type: ignore - dummy_wrapped = True - else: - dummy_wrapped = False - - # Read and fix arguments. If the user passes in --field_name instead of - # --field-name, correct for them. - args = list(sys.argv[1:]) if args is None else list(args) - - # Fix arguments. This will modify all option-style arguments replacing - # underscores with hyphens, or vice versa if use_underscores=True. - # If two options are ambiguous, e.g., --a_b and --a-b, raise a runtime error. - modified_args: Dict[str, str] = {} - for index, arg in enumerate(args): - if not arg.startswith("--"): - continue - - if "=" in arg: - arg, _, val = arg.partition("=") - fixed = "--" + _strings.replace_delimeter_in_part(arg[2:]) + "=" + val + + # We wrap our type with a dummy dataclass if it can't be treated as a nested type. + # For example: passing in f=int will result in a dataclass with a single field + # typed as int. + if not _fields.is_nested_type(cast(type, f), default_instance_internal): + dummy_field = cast( + dataclasses.Field, + dataclasses.field(), + ) + f = dataclasses.make_dataclass( + cls_name="dummy", + fields=[(_strings.dummy_field_name, cast(type, f), dummy_field)], + frozen=True, + ) + default_instance_internal = f(default_instance_internal) # type: ignore + dummy_wrapped = True else: - fixed = "--" + _strings.replace_delimeter_in_part(arg[2:]) - if ( - return_unknown_args - and fixed in modified_args - and modified_args[fixed] != arg - ): - raise RuntimeError( - "Ambiguous arguments: " + modified_args[fixed] + " and " + arg + dummy_wrapped = False + + # Read and fix arguments. If the user passes in --field_name instead of + # --field-name, correct for them. + args = list(sys.argv[1:]) if args is None else list(args) + + # Fix arguments. This will modify all option-style arguments replacing + # underscores with hyphens, or vice versa if use_underscores=True. + # If two options are ambiguous, e.g., --a_b and --a-b, raise a runtime error. + modified_args: Dict[str, str] = {} + for index, arg in enumerate(args): + if not arg.startswith("--"): + continue + + if "=" in arg: + arg, _, val = arg.partition("=") + fixed = "--" + _strings.replace_delimeter_in_part(arg[2:]) + "=" + val + else: + fixed = "--" + _strings.replace_delimeter_in_part(arg[2:]) + if ( + return_unknown_args + and fixed in modified_args + and modified_args[fixed] != arg + ): + raise RuntimeError( + "Ambiguous arguments: " + modified_args[fixed] + " and " + arg + ) + modified_args[fixed] = arg + args[index] = fixed + + # If we pass in the --tyro-print-completion or --tyro-write-completion flags: turn + # formatting tags, and get the shell we want to generate a completion script for + # (bash/zsh/tcsh). + # + # shtab also offers an add_argument_to() functions that fulfills a similar goal, but + # manual parsing of argv is convenient for turning off formatting. + # + # Note: --tyro-print-completion is deprecated! --tyro-write-completion is less prone + # to errors from accidental logging, print statements, etc. + print_completion = False + write_completion = False + if len(args) >= 2: + # We replace underscores with hyphens to accomodate for `use_undercores`. + print_completion = args[0].replace("_", "-") == "--tyro-print-completion" + write_completion = ( + len(args) >= 3 + and args[0].replace("_", "-") == "--tyro-write-completion" ) - modified_args[fixed] = arg - args[index] = fixed - - # If we pass in the --tyro-print-completion or --tyro-write-completion flags: turn - # formatting tags, and get the shell we want to generate a completion script for - # (bash/zsh/tcsh). - # - # shtab also offers an add_argument_to() functions that fulfills a similar goal, but - # manual parsing of argv is convenient for turning off formatting. - # - # Note: --tyro-print-completion is deprecated! --tyro-write-completion is less prone - # to errors from accidental logging, print statements, etc. - print_completion = False - write_completion = False - if len(args) >= 2: - # We replace underscores with hyphens to accomodate for `use_undercores`. - print_completion = args[0].replace("_", "-") == "--tyro-print-completion" - write_completion = ( - len(args) >= 3 and args[0].replace("_", "-") == "--tyro-write-completion" - ) - # Note: setting USE_RICH must happen before the parser specification is generated. - # TODO: revisit this. Ideally we should be able to eliminate the global state - # changes. - completion_shell = None - completion_target_path = None - if print_completion or write_completion: - completion_shell = args[1] - if write_completion: - completion_target_path = pathlib.Path(args[2]) - if print_completion or write_completion or return_parser: - _arguments.USE_RICH = False - else: - _arguments.USE_RICH = True - - # Map a callable to the relevant CLI arguments + subparsers. - parser_spec = _parsers.ParserSpecification.from_callable_or_type( - f, - description=description, - parent_classes=set(), # Used for recursive calls. - default_instance=default_instance_internal, # Overrides for default values. - intern_prefix="", # Used for recursive calls. - extern_prefix="", # Used for recursive calls. - subcommand_prefix="", # Used for recursive calls. - ) + # Note: setting USE_RICH must happen before the parser specification is generated. + # TODO: revisit this. Ideally we should be able to eliminate the global state + # changes. + completion_shell = None + completion_target_path = None + if print_completion or write_completion: + completion_shell = args[1] + if write_completion: + completion_target_path = pathlib.Path(args[2]) + if print_completion or write_completion or return_parser: + _arguments.USE_RICH = False + else: + _arguments.USE_RICH = True + + # Map a callable to the relevant CLI arguments + subparsers. + parser_spec = _parsers.ParserSpecification.from_callable_or_type( + f, + description=description, + parent_classes=set(), # Used for recursive calls. + default_instance=default_instance_internal, # Overrides for default values. + intern_prefix="", # Used for recursive calls. + extern_prefix="", # Used for recursive calls. + subcommand_prefix="", # Used for recursive calls. + ) # Generate parser! with _argparse_formatter.ansi_context(): diff --git a/src/tyro/_docstrings.py b/src/tyro/_docstrings.py index 3d461927b..9b7cbbae3 100644 --- a/src/tyro/_docstrings.py +++ b/src/tyro/_docstrings.py @@ -301,7 +301,7 @@ def get_callable_description(f: Callable) -> str: the fields of the class if a docstring is not specified; this helper will ignore these docstrings.""" - f, _unused = _resolver.resolve_generic_types(f) + f, _ = _resolver.resolve_generic_types(f) f = _resolver.unwrap_origin_strip_extras(f) if f in _callable_description_blocklist: return "" diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index a9e59b143..067cd6372 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -28,7 +28,6 @@ Optional, Set, Tuple, - TypeVar, Union, cast, ) @@ -64,6 +63,7 @@ class FieldDefinition: intern_name: str extern_name: str type_or_callable: Union[TypeForm[Any], Callable] + typevar_context: _resolver.TypeParamAssignmentContext """Type or callable for this field. This should have all Annotated[] annotations stripped.""" default: Any @@ -109,8 +109,19 @@ def make( *, markers: Tuple[_markers.Marker, ...] = (), ): - # Resolve generic aliases. - type_or_callable = _resolver.apply_type_from_typevar(type_or_callable, {}) + # Resolve generics. + type_or_callable = _resolver.TypeParamResolver.concretize_type_params( + type_or_callable + ) + typevar_context = _resolver.TypeParamResolver.get_assignment_context( + type_or_callable + ) + type_or_callable = typevar_context.origin_type + + # Narrow types. + type_or_callable = _resolver.type_from_typevar_constraints(type_or_callable) + type_or_callable = _resolver.narrow_collection_types(type_or_callable, default) + type_or_callable = _resolver.narrow_union_type(type_or_callable, default) # Try to extract argconf overrides from type. _, argconfs = _resolver.unwrap_annotated_and_aliases( @@ -147,12 +158,39 @@ def make( for context_markers in _field_context_markers: markers += context_markers + # Check that the default value matches the final resolved type. + # There's some similar Union-specific logic for this in narrow_union_type(). We + # may be able to consolidate this. + if ( + # Be relatively conservative: isinstance() can be checked on non-type + # types (like unions in Python >=3.10), but we'll only consider single types + # for now. + type(type_or_callable) is type + and not isinstance(default, type_or_callable) # type: ignore + # If a custom constructor is set, type_or_callable may not be + # matched to the annotated type. + and argconf.constructor_factory is None + and default not in DEFAULT_SENTINEL_SINGLETONS + # The numeric tower in Python is wacky. This logic is non-critical, so + # we'll just skip it (+the complexity) for numbers. + and not isinstance(default, numbers.Number) + ): + # If the default value doesn't match the resolved type, we expand the + # type. This is inspired by https://github.com/brentyi/tyro/issues/88. + warnings.warn( + f"The field {name} is annotated with type {type_or_callable}, " + f"but the default value {default} has type {type(default)}. " + f"We'll try to handle this gracefully, but it may cause unexpected behavior." + ) + type_or_callable = Union[type_or_callable, type(default)] # type: ignore + out = FieldDefinition( intern_name=name, extern_name=name if argconf.name is None else argconf.name, type_or_callable=type_or_callable if argconf.constructor_factory is None else argconf.constructor_factory(), + typevar_context=typevar_context, default=default, is_default_from_default_instance=is_default_from_default_instance, helptext=helptext, @@ -261,8 +299,9 @@ def is_nested_type( TODO: we should come up with a better name than 'nested type', which is a little bit misleading.""" + list_or_error = _try_field_list_from_callable(typ, default_instance) return not isinstance( - _try_field_list_from_callable(typ, default_instance), + list_or_error, UnsupportedNestedTypeMessage, ) @@ -271,9 +310,7 @@ def field_list_from_callable( f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance, support_single_arg_types: bool, -) -> Tuple[ - Union[Callable, TypeForm[Any]], Dict[TypeVar, TypeForm], List[FieldDefinition] -]: +) -> Tuple[Union[Callable, TypeForm[Any]], List[FieldDefinition]]: """Generate a list of generic 'field' objects corresponding to the inputs of some annotated callable. @@ -283,7 +320,6 @@ def field_list_from_callable( A list of field definitions. """ # Resolve generic types. - f, type_from_typevar = _resolver.resolve_generic_types(f) f = _resolver.unwrap_newtype_and_narrow_subtypes(f, default_instance) # Try to generate field list. @@ -296,67 +332,21 @@ def field_list_from_callable( if support_single_arg_types: return ( f, - type_from_typevar, [ - FieldDefinition( - intern_name="value", - extern_name="value", # Doesn't matter. + FieldDefinition.make( + name="value", type_or_callable=f, default=default_instance, is_default_from_default_instance=True, helptext="", - custom_constructor=False, - markers={_markers.Positional, _markers._PositionalCall}, - argconf=_confstruct._ArgConfiguration( - None, None, None, None, None, None, None - ), - call_argname="", + markers=(_markers.Positional, _markers._PositionalCall), ) ], ) else: raise _instantiators.UnsupportedTypeAnnotationError(field_list.message) - # Try to resolve types in our list of fields. - def resolve(field: FieldDefinition) -> FieldDefinition: - typ = field.type_or_callable - typ = _resolver.apply_type_from_typevar(typ, type_from_typevar) - typ = _resolver.type_from_typevar_constraints(typ) - typ = _resolver.narrow_collection_types(typ, field.default) - typ = _resolver.narrow_union_type(typ, field.default) - - # Check that the default value matches the final resolved type. - # There's some similar Union-specific logic for this in narrow_union_type(). We - # may be able to consolidate this. - if ( - # Be relatively conservative: isinstance() can be checked on non-type - # types (like unions in Python >=3.10), but we'll only consider single types - # for now. - type(typ) is type - and not isinstance(field.default, typ) # type: ignore - # If a custom constructor is set, field.type_or_callable may not be - # matched to the annotated type. - and not field.custom_constructor - and field.default not in DEFAULT_SENTINEL_SINGLETONS - # The numeric tower in Python is wacky. This logic is non-critical, so - # we'll just skip it (+the complexity) for numbers. - and not isinstance(field.default, numbers.Number) - ): - # If the default value doesn't match the resolved type, we expand the - # type. This is inspired by https://github.com/brentyi/tyro/issues/88. - warnings.warn( - f"The field {field.intern_name} is annotated with type {field.type_or_callable}, " - f"but the default value {field.default} has type {type(field.default)}. " - f"We'll try to handle this gracefully, but it may cause unexpected behavior." - ) - typ = Union[typ, type(field.default)] # type: ignore - - field = dataclasses.replace(field, type_or_callable=typ) - return field - - field_list = list(map(resolve, field_list)) - - return f, type_from_typevar, field_list + return f, field_list # Implementation details below. @@ -383,91 +373,99 @@ def _try_field_list_from_callable( f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - # Apply constructor_factory override for type. - f = _resolver.swap_type_using_confstruct(f) - - # Check for default instances in subcommand configs. This is needed for - # is_nested_type() when arguments are not valid without a default, and this - # default is specified in the subcommand config. - f, found_subcommand_configs = _resolver.unwrap_annotated_and_aliases( - f, conf._confstruct._SubcommandConfiguration - ) - if len(found_subcommand_configs) > 0: - default_instance = found_subcommand_configs[0].default - - # Unwrap generics. - f, type_from_typevar = _resolver.resolve_generic_types(f) - f = _resolver.apply_type_from_typevar(f, type_from_typevar) - f = _resolver.unwrap_newtype_and_narrow_subtypes(f, default_instance) - f = _resolver.narrow_collection_types(f, default_instance) - f_origin = _resolver.unwrap_origin_strip_extras(cast(TypeForm, f)) + # Resolve generic types. + resolve_context = _resolver.TypeParamResolver.get_assignment_context(f) + with resolve_context: + f = resolve_context.origin_type + + # Apply constructor_factory override for type. + f = _resolver.swap_type_using_confstruct(f) + + # Check for default instances in subcommand configs. This is needed for + # is_nested_type() when arguments are not valid without a default, and this + # default is specified in the subcommand config. + f, found_subcommand_configs = _resolver.unwrap_annotated_and_aliases( + f, conf._confstruct._SubcommandConfiguration + ) + if len(found_subcommand_configs) > 0: + default_instance = found_subcommand_configs[0].default + + # Unwrap generics. + f = _resolver.unwrap_newtype_and_narrow_subtypes(f, default_instance) + f = _resolver.narrow_collection_types(f, default_instance) + f_origin = _resolver.unwrap_origin_strip_extras(cast(TypeForm, f)) + + # If `f` is a type: + # 1. Set cls to the type. + # 2. Consider `f` to be `cls.__init__`. + cls: Optional[TypeForm[Any]] = None + if inspect.isclass(f): + cls = f + if hasattr(cls, "__init__") and cls.__init__ is not object.__init__: + f = cls.__init__ # type: ignore + elif hasattr(cls, "__new__") and cls.__new__ is not object.__new__: + f = cls.__new__ + else: + return UnsupportedNestedTypeMessage( + f"Cannot instantiate class {cls} with no unique __init__ or __new__" + " method." + ) + f_origin = cls # type: ignore + + # Try field generation from class inputs. + if cls is not None: + if is_typeddict(cls): + return _field_list_from_typeddict(cls, default_instance) + if _resolver.is_namedtuple(cls): + return _field_list_from_namedtuple(cls, default_instance) + if _resolver.is_dataclass(cls): + return _field_list_from_dataclass(cls, default_instance) + if _is_attrs(cls): + return _field_list_from_attrs(cls, default_instance) + if _is_pydantic(cls): + return _field_list_from_pydantic(cls, default_instance) + + # Standard container types. These are different because they can be nested structures + # if they contain other nested types (eg Tuple[Struct, Struct]), or treated as + # single arguments otherwise (eg Tuple[int, int]). + # + # Note that f_origin will be populated if we annotate as `Tuple[..]`, and cls will + # be populated if we annotate as just `tuple`. + if f_origin is tuple or cls is tuple: + return _field_list_from_tuple(f, default_instance) + elif f_origin in (collections.abc.Mapping, dict) or cls in ( + collections.abc.Mapping, + dict, + ): + return _field_list_from_dict(f, default_instance) + elif f_origin in ( + list, + set, + typing.Sequence, + collections.abc.Sequence, + ) or cls in ( + list, + set, + typing.Sequence, + collections.abc.Sequence, + ): + return _field_list_from_nontuple_sequence_checked(f, default_instance) - # If `f` is a type: - # 1. Set cls to the type. - # 2. Consider `f` to be `cls.__init__`. - cls: Optional[TypeForm[Any]] = None - if inspect.isclass(f): - cls = f - if hasattr(cls, "__init__") and cls.__init__ is not object.__init__: - f = cls.__init__ # type: ignore - elif hasattr(cls, "__new__") and cls.__new__ is not object.__new__: - f = cls.__new__ - else: + # General cases. + if ( + cls is not None and cls in _known_parsable_types + ) or _resolver.unwrap_origin_strip_extras(f) in _known_parsable_types: + return UnsupportedNestedTypeMessage(f"{f} should be parsed directly!") + elif ( + cls is not None + and issubclass(_resolver.unwrap_origin_strip_extras(cls), os.PathLike) + and _instantiators.is_type_string_converter(cls) + ): return UnsupportedNestedTypeMessage( - f"Cannot instantiate class {cls} with no unique __init__ or __new__" - " method." + f"PathLike {cls} should be parsed directly!" ) - f_origin = cls # type: ignore - - # Try field generation from class inputs. - if cls is not None: - if is_typeddict(cls): - return _field_list_from_typeddict(cls, default_instance) - if _resolver.is_namedtuple(cls): - return _field_list_from_namedtuple(cls, default_instance) - if _resolver.is_dataclass(cls): - return _field_list_from_dataclass(cls, default_instance) - if _is_attrs(cls): - return _field_list_from_attrs(cls, default_instance) - if _is_pydantic(cls): - return _field_list_from_pydantic(cls, default_instance) - - # Standard container types. These are different because they can be nested structures - # if they contain other nested types (eg Tuple[Struct, Struct]), or treated as - # single arguments otherwise (eg Tuple[int, int]). - # - # Note that f_origin will be populated if we annotate as `Tuple[..]`, and cls will - # be populated if we annotate as just `tuple`. - if f_origin is tuple or cls is tuple: - return _field_list_from_tuple(f, default_instance) - elif f_origin in (collections.abc.Mapping, dict) or cls in ( - collections.abc.Mapping, - dict, - ): - return _field_list_from_dict(f, default_instance) - elif f_origin in (list, set, typing.Sequence, collections.abc.Sequence) or cls in ( - list, - set, - typing.Sequence, - collections.abc.Sequence, - ): - return _field_list_from_nontuple_sequence_checked(f, default_instance) - - # General cases. - if ( - cls is not None and cls in _known_parsable_types - ) or _resolver.unwrap_origin_strip_extras(f) in _known_parsable_types: - return UnsupportedNestedTypeMessage(f"{f} should be parsed directly!") - elif ( - cls is not None - and issubclass(_resolver.unwrap_origin_strip_extras(cls), os.PathLike) - and _instantiators.is_type_string_converter(cls) - ): - return UnsupportedNestedTypeMessage( - f"PathLike {cls} should be parsed directly!" - ) - else: - return _try_field_list_from_general_callable(f, cls, default_instance) + else: + return _try_field_list_from_general_callable(f, cls, default_instance) def _field_list_from_typeddict( diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index af1b40b73..0e1107e99 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -129,10 +129,7 @@ def is_type_string_converter(typ: Union[Callable, TypeForm[Any]]) -> bool: # Some checks we can do if the signature is available! for i, param in enumerate(signature.parameters.values()): annotation = type_annotations.get(param.name, param.annotation) - - # Hack: apply_type_from_typevar applies shims, like UnionType => Union - # conversion. - annotation = _resolver.apply_type_from_typevar(annotation, {}) + annotation = _resolver.TypeParamResolver.concretize_type_params(annotation) if i == 0 and not ( (get_origin(annotation) is Union and str in get_args(annotation)) or annotation in (str, inspect.Parameter.empty) @@ -153,7 +150,6 @@ def is_type_string_converter(typ: Union[Callable, TypeForm[Any]]) -> bool: def instantiator_from_type( typ: Union[TypeForm[Any], Callable], - type_from_typevar: Dict[TypeVar, TypeForm[Any]], markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: """Recursive helper for parsing type annotations. @@ -201,9 +197,7 @@ def instantiator(strings: List[str]) -> None: # Address container types. If a matching container is found, this will recursively # call instantiator_from_type(). - container_out = _instantiator_from_container_type( - cast(TypeForm[Any], typ), type_from_typevar, markers - ) + container_out = _instantiator_from_container_type(cast(TypeForm[Any], typ), markers) if container_out is not None: return container_out @@ -335,7 +329,6 @@ def instantiator_base_case(strings: List[str]) -> Any: @overload def _instantiator_from_type_inner( typ: TypeForm, - type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal["fixed_length"], markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: ... @@ -344,7 +337,6 @@ def _instantiator_from_type_inner( @overload def _instantiator_from_type_inner( typ: TypeForm, - type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal[False], markers: Set[_markers.Marker], ) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: ... @@ -353,7 +345,6 @@ def _instantiator_from_type_inner( @overload def _instantiator_from_type_inner( typ: TypeForm, - type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal[True], markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: ... @@ -361,13 +352,12 @@ def _instantiator_from_type_inner( def _instantiator_from_type_inner( typ: TypeForm, - type_from_typevar: Dict[TypeVar, TypeForm[Any]], allow_sequences: Literal["fixed_length", True, False], markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: """Thin wrapper over instantiator_from_type, with some extra asserts for catching errors.""" - out = instantiator_from_type(typ, type_from_typevar, markers) + out = instantiator_from_type(typ, markers) if out[1].nargs == "*": # We currently only use allow_sequences=False for options in Literal types, # which are evaluated using `type()`. It should not be possible to hit this @@ -384,7 +374,6 @@ def _instantiator_from_type_inner( def _instantiator_from_container_type( typ: TypeForm[Any], - type_from_typevar: Dict[TypeVar, TypeForm[Any]], markers: Set[_markers.Marker], ) -> Optional[Tuple[Instantiator, InstantiatorMetadata]]: """Attempt to create an instantiator from a container type. Returns `None` if no @@ -418,13 +407,12 @@ def _instantiator_from_container_type( _instantiator_from_literal: (Literal, LiteralAlternate), }.items(): if type_origin in matched_origins: - return make(typ, type_from_typevar, markers) + return make(typ, markers) return None def _instantiator_from_tuple( typ: TypeForm, - type_from_typevar: Dict[TypeVar, TypeForm[Any]], markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: types = get_args(typ) @@ -435,7 +423,7 @@ def _instantiator_from_tuple( # Ellipsis: variable argument counts. When an ellipsis is used, tuples must # contain only one type. assert len(typeset_no_ellipsis) == 1 - return _instantiator_from_sequence(typ, type_from_typevar, markers) + return _instantiator_from_sequence(typ, markers) else: instantiators: List[_StandardInstantiator] = [] @@ -443,7 +431,7 @@ def _instantiator_from_tuple( nargs = 0 for t in types: a, b = _instantiator_from_type_inner( - t, type_from_typevar, allow_sequences="fixed_length", markers=markers + t, allow_sequences="fixed_length", markers=markers ) instantiators.append(a) # type: ignore metas.append(b) @@ -509,7 +497,6 @@ def _join_union_metavars(metavars: Iterable[str]) -> str: def _instantiator_from_union( typ: TypeForm, - type_from_typevar: Dict[TypeVar, TypeForm[Any]], markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: options = list(get_args(typ)) @@ -528,9 +515,7 @@ def _instantiator_from_union( nargs: Optional[Union[int, Literal["*"]]] = 1 first = True for t in options: - a, b = _instantiator_from_type_inner( - t, type_from_typevar, allow_sequences=True, markers=markers - ) + a, b = _instantiator_from_type_inner(t, allow_sequences=True, markers=markers) instantiators.append(a) metas.append(b) if b.choices is None: @@ -591,18 +576,16 @@ def union_instantiator(strings: List[str]) -> Any: def _instantiator_from_dict( typ: TypeForm, - type_from_typevar: Dict[TypeVar, TypeForm[Any]], markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: key_type, val_type = get_args(typ) key_instantiator, key_meta = _instantiator_from_type_inner( - key_type, type_from_typevar, allow_sequences="fixed_length", markers=markers + key_type, allow_sequences="fixed_length", markers=markers ) if _markers.UseAppendAction in markers: val_instantiator, val_meta = _instantiator_from_type_inner( val_type, - type_from_typevar, allow_sequences=True, markers=markers - {_markers.UseAppendAction}, ) @@ -625,7 +608,7 @@ def append_dict_instantiator(strings: List[List[str]]) -> Any: ) else: val_instantiator, val_meta = _instantiator_from_type_inner( - val_type, type_from_typevar, allow_sequences="fixed_length", markers=markers + val_type, allow_sequences="fixed_length", markers=markers ) pair_metavar = f"{key_meta.metavar} {val_meta.metavar}" key_nargs = cast(int, key_meta.nargs) # Casts needed for mypy but not pyright! @@ -673,7 +656,6 @@ def dict_instantiator(strings: List[str]) -> Any: def _instantiator_from_sequence( typ: TypeForm, - type_from_typevar: Dict[TypeVar, TypeForm[Any]], markers: Set[_markers.Marker], ) -> Tuple[Instantiator, InstantiatorMetadata]: """Instantiator for variable-length sequences: list, sets, Tuple[T, ...], etc.""" @@ -691,7 +673,6 @@ def _instantiator_from_sequence( if _markers.UseAppendAction in markers: make, inner_meta = _instantiator_from_type_inner( contained_type, - type_from_typevar, allow_sequences=True, markers=markers - {_markers.UseAppendAction}, ) @@ -709,7 +690,6 @@ def append_sequence_instantiator(strings: List[List[str]]) -> Any: else: make, inner_meta = _instantiator_from_type_inner( contained_type, - type_from_typevar, allow_sequences="fixed_length", markers=markers, ) @@ -743,7 +723,6 @@ def sequence_instantiator(strings: List[str]) -> Any: def _instantiator_from_literal( typ: TypeForm, - type_from_typevar: Dict[TypeVar, TypeForm[Any]], markers: Set[_markers.Marker], ) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: choices = get_args(typ) diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 47b461474..76c1b2257 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -80,11 +80,14 @@ def from_callable_or_type( # Resolve the type of `f`, generate a field list. with _fields.FieldDefinition.marker_context(markers): - f, type_from_typevar, field_list = _fields.field_list_from_callable( - f=f, - default_instance=default_instance, - support_single_arg_types=support_single_arg_types, - ) + context = _resolver.TypeParamResolver.get_assignment_context(f) + with context: + f = context.origin_type + f, field_list = _fields.field_list_from_callable( + f=f, + default_instance=default_instance, + support_single_arg_types=support_single_arg_types, + ) # Cycle detection. # @@ -110,14 +113,14 @@ def from_callable_or_type( subparsers_from_prefix = {} for field in field_list: - field_out = handle_field( - field, - type_from_typevar=type_from_typevar, - parent_classes=parent_classes, - intern_prefix=intern_prefix, - extern_prefix=extern_prefix, - subcommand_prefix=subcommand_prefix, - ) + with field.typevar_context: + field_out = handle_field( + field, + parent_classes=parent_classes, + intern_prefix=intern_prefix, + extern_prefix=extern_prefix, + subcommand_prefix=subcommand_prefix, + ) if isinstance(field_out, _arguments.ArgumentDefinition): # Handle single arguments. args.append(field_out) @@ -287,7 +290,6 @@ def format_group_name(prefix: str) -> str: def handle_field( field: _fields.FieldDefinition, - type_from_typevar: Dict[TypeVar, TypeForm[Any]], parent_classes: Set[Type[Any]], intern_prefix: str, extern_prefix: str, @@ -308,7 +310,6 @@ def handle_field( # (1) Handle Unions over callables; these result in subparsers. subparsers_attempt = SubparsersSpecification.from_field( field, - type_from_typevar=type_from_typevar, parent_classes=parent_classes, intern_prefix=_strings.make_field_name([intern_prefix, field.intern_name]), extern_prefix=_strings.make_field_name([extern_prefix, field.extern_name]), @@ -362,7 +363,6 @@ def handle_field( extern_prefix=extern_prefix, subcommand_prefix=subcommand_prefix, field=field, - type_from_typevar=type_from_typevar, ) @@ -381,7 +381,6 @@ class SubparsersSpecification: @staticmethod def from_field( field: _fields.FieldDefinition, - type_from_typevar: Dict[TypeVar, TypeForm[Any]], parent_classes: Set[Type[Any]], intern_prefix: str, extern_prefix: str, @@ -393,10 +392,7 @@ def from_field( # We don't use sets here to retain order of subcommands. options: List[Union[type, Callable]] - options = [ - _resolver.apply_type_from_typevar(typ, type_from_typevar) - for typ in get_args(typ) - ] + options = [typ for typ in get_args(typ)] options = [ ( # Cast seems unnecessary but needed in mypy... (1.4.1) diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index b1b76467e..560354767 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -1,5 +1,7 @@ """Utilities for resolving types and forward references.""" +from __future__ import annotations + import collections.abc import copy import dataclasses @@ -43,10 +45,16 @@ TypeOrCallable = TypeVar("TypeOrCallable", TypeForm[Any], Callable) +def unwrap_aliases(typ: TypeOrCallable) -> TypeOrCallable: + """Unwrap type aliases.""" + if isinstance(typ, TypeAliasType): + return unwrap_aliases(cast(Any, typ.__value__)) + return typ + + def unwrap_origin_strip_extras(typ: TypeOrCallable) -> TypeOrCallable: """Returns the origin, ignoring typing.Annotated, of typ if it exists. Otherwise, returns typ.""" - # TODO: Annotated[] handling should be revisited... typ = unwrap_annotated_and_aliases(typ) origin = get_origin(typ) @@ -61,63 +69,6 @@ def is_dataclass(cls: Union[TypeForm, Callable]) -> bool: return dataclasses.is_dataclass(unwrap_origin_strip_extras(cls)) # type: ignore -def resolve_generic_types( - cls: TypeOrCallable, -) -> Tuple[TypeOrCallable, Dict[TypeVar, TypeForm[Any]]]: - """If the input is a class: no-op. If it's a generic alias: returns the origin - class, and a mapping from typevars to concrete types.""" - - annotations: Tuple[Any, ...] = () - if get_origin(cls) is Annotated: - # ^We need this `if` statement for an obscure edge case: when `cls` is a - # function with `__tyro_markers__` set, we don't want/need to return - # Annotated[func, markers]. - cls, annotations = unwrap_annotated_and_aliases(cls, "all") - - # We'll ignore NewType when getting the origin + args for generics. - origin_cls = get_origin(unwrap_newtype_and_aliases(cls)[0]) - type_from_typevar: Dict[TypeVar, TypeForm[Any]] = {} - - # Support typing.Self. - # We'll do this by pretending that `Self` is a TypeVar... - if hasattr(cls, "__self__"): - self_type = getattr(cls, "__self__") - if inspect.isclass(self_type): - type_from_typevar[cast(TypeVar, Self)] = self_type # type: ignore - else: - type_from_typevar[cast(TypeVar, Self)] = self_type.__class__ # type: ignore - - if ( - # Apply some heuristics for generic types. Should revisit this. - origin_cls is not None - and hasattr(origin_cls, "__parameters__") - and hasattr(origin_cls.__parameters__, "__len__") - ): - typevars = origin_cls.__parameters__ - typevar_values = get_args(unwrap_newtype_and_aliases(cls)[0]) - assert len(typevars) == len(typevar_values) - cls = origin_cls - type_from_typevar.update(dict(zip(typevars, typevar_values))) - - if hasattr(cls, "__orig_bases__"): - bases = getattr(cls, "__orig_bases__") - for base in bases: - origin_base = unwrap_origin_strip_extras(base) - if origin_base is base or not hasattr(origin_base, "__parameters__"): - continue - typevars = origin_base.__parameters__ - typevar_values = get_args(base) - type_from_typevar.update(dict(zip(typevars, typevar_values))) - - if len(annotations) == 0: - return cls, type_from_typevar - else: - return ( - Annotated.__class_getitem__((cls, *annotations)), # type: ignore - type_from_typevar, - ) - - @_unsafe_cache.unsafe_cache(maxsize=1024) def resolved_fields(cls: TypeForm) -> List[dataclasses.Field]: """Similar to dataclasses.fields(), but includes dataclasses.InitVar types and @@ -389,63 +340,123 @@ def unwrap_annotated_and_aliases( return args[0], targets # type: ignore -def apply_type_from_typevar( - typ: TypeOrCallable, type_from_typevar: Dict[TypeVar, TypeForm[Any]] -) -> TypeOrCallable: - GenericAlias = getattr(types, "GenericAlias", None) - if ( - GenericAlias is not None - and isinstance(typ, GenericAlias) - and len(getattr(typ, "__type_params__", ())) > 0 - ): - type_from_typevar = type_from_typevar.copy() - for k, v in zip(typ.__type_params__, typ.__args__): # type: ignore - type_from_typevar[k] = v # type: ignore - typ = typ.__value__ # type: ignore +class TypeParamResolver: + param_assignments: List[Dict[TypeVar, TypeForm[Any]]] = [] + + @classmethod + def get_assignment_context(cls, typ: TypeOrCallable) -> TypeParamAssignmentContext: + """Context manager for resolving type parameters.""" + typ, type_from_typevar = resolve_generic_types(typ) + return TypeParamAssignmentContext(typ, type_from_typevar) + + @staticmethod + def concretize_type_params(typ: TypeOrCallable) -> TypeOrCallable: + """Apply type parameter assignments based on the current context.""" + typ = unwrap_aliases(typ) + + GenericAlias = getattr(types, "GenericAlias", None) + if ( + GenericAlias is not None + and isinstance(typ, GenericAlias) + and len(getattr(typ, "__type_params__", ())) > 0 + ): + type_from_typevar = {} + for k, v in zip(typ.__type_params__, get_args(typ)): # type: ignore + type_from_typevar[k] = v # type: ignore + typ = typ.__value__ # type: ignore + + with TypeParamAssignmentContext(typ, type_from_typevar): + return TypeParamResolver._concretize_type_params(typ) + else: + return TypeParamResolver._concretize_type_params(typ) + + @staticmethod + def _concretize_type_params(typ: TypeOrCallable) -> TypeOrCallable: + for type_from_typevar in reversed(TypeParamResolver.param_assignments): + if typ in type_from_typevar: + return type_from_typevar[typ] # type: ignore + + origin = get_origin(typ) + args = get_args(typ) + if len(args) > 0: + if origin is Annotated: + args = args[:1] + if origin is collections.abc.Callable: + assert isinstance(args[0], list) + args = tuple(args[0]) + args[1:] + + new_args_list = [] + for x in args: + for type_from_typevar in reversed(TypeParamResolver.param_assignments): + if x in type_from_typevar: + x = type_from_typevar[x] + break + new_args_list.append(x) + + new_args = tuple( + TypeParamResolver.concretize_type_params(x) for x in new_args_list + ) + + # Apply shims to convert from types.UnionType to typing.Union, list to typing.List, etc. + # This will let us call `copy_with()` next. + typ = standardize_builtin_generics(typ) + + # Standard generic aliases have a `copy_with()`! + if hasattr(typ, "copy_with"): + return typ.copy_with(new_args) # type: ignore + else: + # `collections` types, like collections.abc.Sequence. + assert hasattr(origin, "__class_getitem__") + return origin.__class_getitem__(new_args) # type: ignore + + return typ # type: ignore - if typ in type_from_typevar: - return type_from_typevar[typ] # type: ignore +def standardize_builtin_generics(typ: TypeOrCallable) -> TypeOrCallable: + """Apply shims to convert `types.UnionType` to `typing.Union`, `list` to + `typing.List`, etc.""" origin = get_origin(typ) args = get_args(typ) - if len(args) > 0: - if origin is Annotated: - args = args[:1] - if origin is collections.abc.Callable: - assert isinstance(args[0], list) - args = tuple(args[0]) + args[1:] - - # Convert Python 3.9 and 3.10 types to their typing library equivalents, which - # support `.copy_with()`. This is not really the right place for this logic... - if sys.version_info[:2] >= (3, 9): - shim_table = { - # PEP 585. Requires Python 3.9. - tuple: Tuple, - list: List, - dict: Dict, - set: Set, - frozenset: FrozenSet, - type: Type, - } - if hasattr(types, "UnionType"): # type: ignore - # PEP 604. Requires Python 3.10. - shim_table[types.UnionType] = Union # type: ignore - - for new, old in shim_table.items(): - if origin is new: # type: ignore - typ = old.__getitem__(args) # type: ignore - - new_args = tuple(apply_type_from_typevar(x, type_from_typevar) for x in args) - - # Standard generic aliases have a `copy_with()`! - if hasattr(typ, "copy_with"): - return typ.copy_with(new_args) # type: ignore - else: - # `collections` types, like collections.abc.Sequence. - assert hasattr(origin, "__class_getitem__") - return origin.__class_getitem__(new_args) # type: ignore - return typ # type: ignore + if origin is None or len(args) == 0: + return typ + + # Convert Python 3.9 and 3.10 types to their typing library equivalents, which + # support `.copy_with()`. This is not really the right place for this logic... + if sys.version_info[:2] >= (3, 9): + shim_table = { + # PEP 585. Requires Python 3.9. + tuple: Tuple, + list: List, + dict: Dict, + set: Set, + frozenset: FrozenSet, + type: Type, + } + if hasattr(types, "UnionType"): # type: ignore + # PEP 604. Requires Python 3.10. + shim_table[types.UnionType] = Union # type: ignore + + if origin in shim_table: # type: ignore + typ = shim_table[origin].__getitem__(args) # type: ignore + return typ + + +class TypeParamAssignmentContext: + def __init__( + self, + origin_type: TypeOrCallable, + type_from_typevar: Dict[TypeVar, TypeForm[Any]], + ): + # `Any` is needed for mypy... + self.origin_type: Any = origin_type + self.type_from_typevar = type_from_typevar + + def __enter__(self): + TypeParamResolver.param_assignments.append(self.type_from_typevar) + + def __exit__(self, exc_type, exc_value, traceback): + TypeParamResolver.param_assignments.pop() @_unsafe_cache.unsafe_cache(maxsize=1024) @@ -483,6 +494,67 @@ def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCalla NoneType = type(None) +def resolve_generic_types( + typ: TypeOrCallable, +) -> Tuple[TypeOrCallable, Dict[TypeVar, TypeForm[Any]]]: + """If the input is a class: no-op. If it's a generic alias: returns the origin + class, and a mapping from typevars to concrete types.""" + + annotations: Tuple[Any, ...] = () + if get_origin(typ) is Annotated: + # ^We need this `if` statement for an obscure edge case: when `cls` is a + # function with `__tyro_markers__` set, we don't want/need to return + # Annotated[func, markers]. + typ, annotations = unwrap_annotated_and_aliases(typ, "all") + + # Apply shims to convert from types.UnionType to typing.Union, list to + # typing.List, etc. + typ = standardize_builtin_generics(typ) + + # We'll ignore NewType when getting the origin + args for generics. + origin_cls = get_origin(unwrap_newtype_and_aliases(typ)[0]) + type_from_typevar: Dict[TypeVar, TypeForm[Any]] = {} + + # Support typing.Self. + # We'll do this by pretending that `Self` is a TypeVar... + if hasattr(typ, "__self__"): + self_type = getattr(typ, "__self__") + if inspect.isclass(self_type): + type_from_typevar[cast(TypeVar, Self)] = self_type # type: ignore + else: + type_from_typevar[cast(TypeVar, Self)] = self_type.__class__ # type: ignore + + if ( + # Apply some heuristics for generic types. Should revisit this. + origin_cls is not None + and hasattr(origin_cls, "__parameters__") + and hasattr(origin_cls.__parameters__, "__len__") + ): + typevars = origin_cls.__parameters__ + typevar_values = get_args(unwrap_newtype_and_aliases(typ)[0]) + assert len(typevars) == len(typevar_values) + typ = origin_cls + type_from_typevar.update(dict(zip(typevars, typevar_values))) + + if hasattr(typ, "__orig_bases__"): + bases = getattr(typ, "__orig_bases__") + for base in bases: + origin_base = unwrap_origin_strip_extras(base) + if origin_base is base or not hasattr(origin_base, "__parameters__"): + continue + typevars = origin_base.__parameters__ + typevar_values = get_args(base) + type_from_typevar.update(dict(zip(typevars, typevar_values))) + + if len(annotations) == 0: + return typ, type_from_typevar + else: + return ( + Annotated.__class_getitem__((typ, *annotations)), # type: ignore + type_from_typevar, + ) + + def get_type_hints_with_backported_syntax( obj: Callable[..., Any], include_extras: bool = False ) -> Dict[str, Any]: diff --git a/src/tyro/_subcommand_matching.py b/src/tyro/_subcommand_matching.py index ff21e4817..02daccc5f 100644 --- a/src/tyro/_subcommand_matching.py +++ b/src/tyro/_subcommand_matching.py @@ -72,7 +72,7 @@ def make( ) -> _TypeTree: """From an object instance, return a data structure representing the types in the object.""" try: - typ, _type_from_typevar, field_list = _fields.field_list_from_callable( + typ, field_list = _fields.field_list_from_callable( typ, default_instance=default_instance, support_single_arg_types=False ) except _instantiators.UnsupportedTypeAnnotationError: diff --git a/tests/test_new_style_annotations_min_py312.py b/tests/test_new_style_annotations_min_py312.py index 120d5d09b..b7b6ea055 100644 --- a/tests/test_new_style_annotations_min_py312.py +++ b/tests/test_new_style_annotations_min_py312.py @@ -77,3 +77,16 @@ class Config: arg: TT[bool] assert tyro.cli(Config, args=[]) == Config(arg=True) + + +type Renamed[T] = Annotated[T, tyro.conf.arg(name="renamed")] + + +def test_pep695_generic_alias_rename() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: Renamed[bool] + + assert tyro.cli(Config, args=["--renamed", "True"]) == Config(arg=True) diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py index 9189e585c..b7b6ea055 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py @@ -64,3 +64,29 @@ class Container: a: AnnotatedBasic assert tyro.cli(Container, args="--basic 1".split(" ")) == Container(1) + + +type TT[T] = Annotated[T, tyro.conf.arg(name="", constructor=lambda: True)] + + +def test_pep695_generic_alias() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: TT[bool] + + assert tyro.cli(Config, args=[]) == Config(arg=True) + + +type Renamed[T] = Annotated[T, tyro.conf.arg(name="renamed")] + + +def test_pep695_generic_alias_rename() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: Renamed[bool] + + assert tyro.cli(Config, args=["--renamed", "True"]) == Config(arg=True) diff --git a/tests/test_py311_generated/test_tuple_with_subcommands_aliases_min_py312_generated.py b/tests/test_py311_generated/test_tuple_with_subcommands_aliases_min_py312_generated.py new file mode 100644 index 000000000..ed84fe81c --- /dev/null +++ b/tests/test_py311_generated/test_tuple_with_subcommands_aliases_min_py312_generated.py @@ -0,0 +1,164 @@ +# mypy: ignore-errors +"""Tests adapted from https://github.com/brentyi/tyro/issues/89, which catches edge +cases when combining nested tuple types, renamed arguments, and subcommands. + +Largely written by @wookayin. +""" + +import dataclasses +from pathlib import Path +from typing import Annotated, cast + +import tyro.conf + + +@dataclasses.dataclass +class Checkout[T]: + """Check out a branch.""" + + branch: T + + +@dataclasses.dataclass +class Commit: + """Commit something.""" + + input: tyro.conf.Positional[Path] + + +@dataclasses.dataclass +class Arg: + verbose: bool = True + + +def test_case1() -> None: + o = tyro.cli( + Checkout[str] | Commit, + args=["commit", "./path.txt"], + ) + assert o == Commit(input=Path("path.txt")) + + +def test_case2() -> None: + arg, action = tyro.cli( + tuple[ + Arg, + Annotated[ + Checkout[str] | Commit, + tyro.conf.arg(name=""), + ], + ], + args=["commit", "./path.txt"], + ) + + assert isinstance(arg, Arg) + assert isinstance(action, Commit) + assert action.input == Path("./path.txt") + + +def test_case3() -> None: + o = tyro.cli( + tuple[ + Annotated[ + Arg, + tyro.conf.arg(name=""), + ], + Annotated[ + Annotated[Checkout[str], tyro.conf.subcommand(name="checkout")] + | Annotated[Commit, tyro.conf.subcommand(name="commit")], + tyro.conf.arg(name=""), + ], + ], + args=["commit", "./path.txt"], + ) + assert o == (Arg(), Commit(Path("./path.txt"))) + + +def test_case4() -> None: + o = tyro.cli( + tuple[ + Annotated[ + Annotated[Checkout[str], tyro.conf.subcommand(name="checkout")] + | Annotated[Commit, tyro.conf.subcommand(name="commit")], + tyro.conf.arg(name=""), + ] + ], + args=["commit", "./path.txt"], + ) + assert o == (Commit(Path("./path.txt")),) + + +def test_case5() -> None: + assert tyro.cli( + tyro.conf.OmitArgPrefixes[ + tuple[ + Annotated[ + Checkout[str], + tyro.conf.subcommand(prefix_name=False), + ] + | Annotated[Commit, tyro.conf.subcommand(prefix_name=False)], + Arg, + ] + ], + args=["--no-verbose", "checkout-str", "--branch", "branch"], + ) == (Checkout("branch"), Arg(False)) + + +# New tests using PEP 695 type aliases: + +type CheckoutAlias[T] = Annotated[Checkout[T], tyro.conf.subcommand(name="checkout")] +type CommitAlias = Annotated[Commit, tyro.conf.subcommand(name="commit")] +type ActionUnion = CheckoutAlias[str] | CommitAlias +type RenamedArg = Annotated[Arg, tyro.conf.arg(name="global")] + + +def test_case6() -> None: + o = tyro.cli( + tuple[ + RenamedArg, + Annotated[ActionUnion, tyro.conf.arg(name="")], + ], + args=["--global.no-verbose", "commit", "./path.txt"], + ) + assert o == (Arg(verbose=False), Commit(Path("./path.txt"))) + + +# https://github.com/microsoft/pyright/issues/9261 +type PositionalPath = tyro.conf.Positional[Path] # type: ignore + + +def test_case7() -> None: + @dataclasses.dataclass + class NewCommit: + """Commit something with a message.""" + + input: PositionalPath + message: str = "Default commit message" + + o = tyro.cli( + cast( + type, + CheckoutAlias[str] + | Annotated[NewCommit, tyro.conf.subcommand(name="commit")], + ), + args=["commit", "./path.txt", "--message", "New commit"], + ) + assert o == NewCommit(input=Path("./path.txt"), message="New commit") + + +type VerboseArg = Annotated[ + bool, tyro.conf.arg(name="verbose", help="Enable verbose output") +] + + +def test_case8() -> None: + @dataclasses.dataclass + class ConfigWithVerbose: + action: ActionUnion + verbose: VerboseArg = False + + o = tyro.cli( + ConfigWithVerbose, + args=["--verbose", "action:checkout", "--action.branch", "main"], + ) + assert o == ConfigWithVerbose(verbose=True, action=Checkout(branch="main")) diff --git a/tests/test_tuple_with_subcommands_aliases_min_py312.py b/tests/test_tuple_with_subcommands_aliases_min_py312.py new file mode 100644 index 000000000..8c0f33a8a --- /dev/null +++ b/tests/test_tuple_with_subcommands_aliases_min_py312.py @@ -0,0 +1,166 @@ +# mypy: ignore-errors +"""Tests adapted from https://github.com/brentyi/tyro/issues/89, which catches edge +cases when combining nested tuple types, renamed arguments, and subcommands. + +Largely written by @wookayin. +""" + +import dataclasses +from pathlib import Path +from typing import cast + +from typing_extensions import Annotated + +import tyro.conf + + +@dataclasses.dataclass +class Checkout[T]: + """Check out a branch.""" + + branch: T + + +@dataclasses.dataclass +class Commit: + """Commit something.""" + + input: tyro.conf.Positional[Path] + + +@dataclasses.dataclass +class Arg: + verbose: bool = True + + +def test_case1() -> None: + o = tyro.cli( + Checkout[str] | Commit, + args=["commit", "./path.txt"], + ) + assert o == Commit(input=Path("path.txt")) + + +def test_case2() -> None: + arg, action = tyro.cli( + tuple[ + Arg, + Annotated[ + Checkout[str] | Commit, + tyro.conf.arg(name=""), + ], + ], + args=["commit", "./path.txt"], + ) + + assert isinstance(arg, Arg) + assert isinstance(action, Commit) + assert action.input == Path("./path.txt") + + +def test_case3() -> None: + o = tyro.cli( + tuple[ + Annotated[ + Arg, + tyro.conf.arg(name=""), + ], + Annotated[ + Annotated[Checkout[str], tyro.conf.subcommand(name="checkout")] + | Annotated[Commit, tyro.conf.subcommand(name="commit")], + tyro.conf.arg(name=""), + ], + ], + args=["commit", "./path.txt"], + ) + assert o == (Arg(), Commit(Path("./path.txt"))) + + +def test_case4() -> None: + o = tyro.cli( + tuple[ + Annotated[ + Annotated[Checkout[str], tyro.conf.subcommand(name="checkout")] + | Annotated[Commit, tyro.conf.subcommand(name="commit")], + tyro.conf.arg(name=""), + ] + ], + args=["commit", "./path.txt"], + ) + assert o == (Commit(Path("./path.txt")),) + + +def test_case5() -> None: + assert tyro.cli( + tyro.conf.OmitArgPrefixes[ + tuple[ + Annotated[ + Checkout[str], + tyro.conf.subcommand(prefix_name=False), + ] + | Annotated[Commit, tyro.conf.subcommand(prefix_name=False)], + Arg, + ] + ], + args=["--no-verbose", "checkout-str", "--branch", "branch"], + ) == (Checkout("branch"), Arg(False)) + + +# New tests using PEP 695 type aliases: + +type CheckoutAlias[T] = Annotated[Checkout[T], tyro.conf.subcommand(name="checkout")] +type CommitAlias = Annotated[Commit, tyro.conf.subcommand(name="commit")] +type ActionUnion = CheckoutAlias[str] | CommitAlias +type RenamedArg = Annotated[Arg, tyro.conf.arg(name="global")] + + +def test_case6() -> None: + o = tyro.cli( + tuple[ + RenamedArg, + Annotated[ActionUnion, tyro.conf.arg(name="")], + ], + args=["--global.no-verbose", "commit", "./path.txt"], + ) + assert o == (Arg(verbose=False), Commit(Path("./path.txt"))) + + +# https://github.com/microsoft/pyright/issues/9261 +type PositionalPath = tyro.conf.Positional[Path] # type: ignore + + +def test_case7() -> None: + @dataclasses.dataclass + class NewCommit: + """Commit something with a message.""" + + input: PositionalPath + message: str = "Default commit message" + + o = tyro.cli( + cast( + type, + CheckoutAlias[str] + | Annotated[NewCommit, tyro.conf.subcommand(name="commit")], + ), + args=["commit", "./path.txt", "--message", "New commit"], + ) + assert o == NewCommit(input=Path("./path.txt"), message="New commit") + + +type VerboseArg = Annotated[ + bool, tyro.conf.arg(name="verbose", help="Enable verbose output") +] + + +def test_case8() -> None: + @dataclasses.dataclass + class ConfigWithVerbose: + action: ActionUnion + verbose: VerboseArg = False + + o = tyro.cli( + ConfigWithVerbose, + args=["--verbose", "action:checkout", "--action.branch", "main"], + ) + assert o == ConfigWithVerbose(verbose=True, action=Checkout(branch="main")) From 74d977ead765c674fa6e317eae44ad7e56ae7257 Mon Sep 17 00:00:00 2001 From: brentyi Date: Sat, 19 Oct 2024 12:29:52 -0700 Subject: [PATCH 462/491] `0.8.13` --- pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 71bdffb82..e6b98dae2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.12" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.13" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index 7fd0b1827..74c0da4af 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.12" +__version__ = "0.8.13" From 1f76596a8b42ca81142634b6b26df7deb989eb39 Mon Sep 17 00:00:00 2001 From: brentyi Date: Sat, 19 Oct 2024 12:34:33 -0700 Subject: [PATCH 463/491] Fix example usage commands --- examples/04_additional/12_aliases.py | 14 +++++++------- examples/04_additional/13_counters.py | 8 ++++---- examples/04_additional/16_type_statement.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/04_additional/12_aliases.py b/examples/04_additional/12_aliases.py index 5ee33de9c..16c5b3b3c 100644 --- a/examples/04_additional/12_aliases.py +++ b/examples/04_additional/12_aliases.py @@ -3,13 +3,13 @@ :func:`tyro.conf.arg()` can be used to attach aliases to arguments. Usage: -`python ./11_aliases.py --help` -`python ./11_aliases.py commit --help` -`python ./11_aliases.py commit --message hello --all` -`python ./11_aliases.py commit -m hello -a` -`python ./11_aliases.py checkout --help` -`python ./11_aliases.py checkout --branch main` -`python ./11_aliases.py checkout -b main` +`python ./12_aliases.py --help` +`python ./12_aliases.py commit --help` +`python ./12_aliases.py commit --message hello --all` +`python ./12_aliases.py commit -m hello -a` +`python ./12_aliases.py checkout --help` +`python ./12_aliases.py checkout --branch main` +`python ./12_aliases.py checkout -b main` """ from typing_extensions import Annotated diff --git a/examples/04_additional/13_counters.py b/examples/04_additional/13_counters.py index 2a6dd2bbf..83562b1ac 100644 --- a/examples/04_additional/13_counters.py +++ b/examples/04_additional/13_counters.py @@ -3,10 +3,10 @@ Repeatable 'counter' arguments can be specified via :data:`tyro.conf.UseCounterAction`. Usage: -`python ./12_counters.py --help` -`python ./12_counters.py --verbosity` -`python ./12_counters.py --verbosity --verbosity` -`python ./12_counters.py -vvv` +`python ./13_counters.py --help` +`python ./13_counters.py --verbosity` +`python ./13_counters.py --verbosity --verbosity` +`python ./13_counters.py -vvv` """ from typing_extensions import Annotated diff --git a/examples/04_additional/16_type_statement.py b/examples/04_additional/16_type_statement.py index 023d5c6f6..9aa08cf93 100644 --- a/examples/04_additional/16_type_statement.py +++ b/examples/04_additional/16_type_statement.py @@ -6,7 +6,7 @@ In Python 3.12, the :code:`type` statement is introduced to create type aliases. Usage: -`python ./13_type_statement.py --help` +`python ./16_type_statement.py --help` """ import dataclasses From 15c368634793ae9c781d29c7d468bc39fdd307eb Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 22 Oct 2024 02:35:13 -0700 Subject: [PATCH 464/491] Fix edge cases associated with nested/recursive PEP 695 aliases (#181) * Fix edge cases with recursive type aliases * Generated tests * Add RecursionError check for recursive types * NewType / alias cleanup * Docs * Nits, more tests * Bump version --- docs/source/index.md | 1 + docs/source/supported_types.md | 43 +++++++ pyproject.toml | 2 +- src/tyro/__init__.py | 2 +- src/tyro/_calling.py | 4 +- src/tyro/_fields.py | 12 +- src/tyro/_instantiators.py | 18 +-- src/tyro/_parsers.py | 22 ++-- src/tyro/_resolver.py | 102 ++++++++++------ src/tyro/_strings.py | 14 ++- src/tyro/_subcommand_matching.py | 6 +- src/tyro/extras/_serialization.py | 2 +- tests/test_nested.py | 38 ++++++ tests/test_new_style_annotations_min_py312.py | 113 +++++++++++++++++- .../test_nested_generated.py | 38 ++++++ ...w_style_annotations_min_py312_generated.py | 113 +++++++++++++++++- 16 files changed, 451 insertions(+), 79 deletions(-) create mode 100644 docs/source/supported_types.md diff --git a/docs/source/index.md b/docs/source/index.md index 05d7e25db..1144b6090 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -70,6 +70,7 @@ To get started, we recommend browsing the examples to the left. installation your_first_cli + supported_types .. toctree:: :caption: Basics diff --git a/docs/source/supported_types.md b/docs/source/supported_types.md new file mode 100644 index 000000000..6e3a95100 --- /dev/null +++ b/docs/source/supported_types.md @@ -0,0 +1,43 @@ +# Supported types + +For minimum-boilerplate CLIs, `tyro` aims to maximize support of +Python's standard [typing](https://docs.python.org/3/library/typing.html) +features. + +As a partial list, inputs can be annotated with: + +- Basic types like `int`, `str`, `float`, `bool`, `pathlib.Path`, `None`. +- `datetime.date`, `datetime.datetime`, and `datetime.time`. +- Container types like `list`, `dict`, `tuple`, and `set`. +- Union types, like `X | Y`, `Union[X, Y]`, and `Optional[T]`. +- `typing.Literal` and `enum.Enum` . +- Type aliases, for example using Python 3.12's [PEP 695](https://peps.python.org/pep-0695/) `type` statement. +- Generics, such as those annotated with `typing.TypeVar` or with the type parameter syntax introduced by Python 3.12's [PEP 695](https://peps.python.org/pep-0695/). +- etc + +Compositions of the above types, like `tuple[int | str, ...] | None`, are also supported. + +Types can also be placed and nested in various structures, such as: + +- `dataclasses.dataclass`. +- `attrs`, `pydantic`, and `flax.linen` models. +- `typing.NamedTuple`. +- `typing.TypedDict`, flags like `total=`, and associated annotations like `typing.Required`, `typing.NotRequired`, `typing.ReadOnly`, + +### What's not supported + +There are some limitations. We currently _do not_ support: + +- Variable-length sequences over nested structures, unless a default is + provided. For types like `list[Dataclass]`, we require a default value to + infer length from. The length of the corresponding field cannot be changed + from the CLI interface. +- Nesting variable-length sequences in other sequences. `tuple[int, ...]` and + `tuple[tuple[int, int, int], ...]` are supported, as the variable-length + sequence is the outermost type. However, `tuple[tuple[int, ...], ...]` is + ambiguous to parse and not supported. +- Self-referential types, like `type RecursiveList[T] = T | list[RecursiveList[T]]`. + +In each of these cases, a [custom +constructor](https://brentyi.github.io/tyro/examples/04_additional/11_custom_constructors/) +can be defined as a workaround. diff --git a/pyproject.toml b/pyproject.toml index e6b98dae2..f1978a113 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tyro" authors = [ {name = "brentyi", email = "brentyi@berkeley.edu"}, ] -version = "0.8.13" # TODO: currently needs to be synchronized manually with __init__.py. +version = "0.8.14" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" license = { text="MIT" } diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index 74c0da4af..14155a7c4 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -14,4 +14,4 @@ # TODO: this should be synchronized automatically with the pyproject.toml. -__version__ = "0.8.13" +__version__ = "0.8.14" diff --git a/src/tyro/_calling.py b/src/tyro/_calling.py index 632b5bc18..ee4cdd78f 100644 --- a/src/tyro/_calling.py +++ b/src/tyro/_calling.py @@ -233,9 +233,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: unwrapped_f = f unwrapped_f = _resolver.swap_type_using_confstruct(unwrapped_f) unwrapped_f = _resolver.unwrap_origin_strip_extras(unwrapped_f) - unwrapped_f = _resolver.unwrap_newtype_and_narrow_subtypes( - unwrapped_f, default_instance - ) + unwrapped_f = _resolver.narrow_subtypes(unwrapped_f, default_instance) unwrapped_f = _resolver.unwrap_origin_strip_extras(unwrapped_f) if unwrapped_f in (tuple, list, set): diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 067cd6372..6329737c2 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -124,7 +124,7 @@ def make( type_or_callable = _resolver.narrow_union_type(type_or_callable, default) # Try to extract argconf overrides from type. - _, argconfs = _resolver.unwrap_annotated_and_aliases( + _, argconfs = _resolver.unwrap_annotated( type_or_callable, _confstruct._ArgConfiguration ) argconf = _confstruct._ArgConfiguration( @@ -149,7 +149,7 @@ def make( if argconf.help is not None: helptext = argconf.help - type_or_callable, inferred_markers = _resolver.unwrap_annotated_and_aliases( + type_or_callable, inferred_markers = _resolver.unwrap_annotated( type_or_callable, _markers._Marker ) markers = inferred_markers + markers @@ -320,11 +320,11 @@ def field_list_from_callable( A list of field definitions. """ # Resolve generic types. - f = _resolver.unwrap_newtype_and_narrow_subtypes(f, default_instance) + f = _resolver.narrow_subtypes(f, default_instance) # Try to generate field list. # We recursively apply markers. - _, parent_markers = _resolver.unwrap_annotated_and_aliases(f, _markers._Marker) + _, parent_markers = _resolver.unwrap_annotated(f, _markers._Marker) with FieldDefinition.marker_context(parent_markers): field_list = _try_field_list_from_callable(f, default_instance) @@ -384,14 +384,14 @@ def _try_field_list_from_callable( # Check for default instances in subcommand configs. This is needed for # is_nested_type() when arguments are not valid without a default, and this # default is specified in the subcommand config. - f, found_subcommand_configs = _resolver.unwrap_annotated_and_aliases( + f, found_subcommand_configs = _resolver.unwrap_annotated( f, conf._confstruct._SubcommandConfiguration ) if len(found_subcommand_configs) > 0: default_instance = found_subcommand_configs[0].default # Unwrap generics. - f = _resolver.unwrap_newtype_and_narrow_subtypes(f, default_instance) + f = _resolver.narrow_subtypes(f, default_instance) f = _resolver.narrow_collection_types(f, default_instance) f_origin = _resolver.unwrap_origin_strip_extras(cast(TypeForm, f)) diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py index 0e1107e99..aaf37cd26 100644 --- a/src/tyro/_instantiators.py +++ b/src/tyro/_instantiators.py @@ -126,12 +126,16 @@ def is_type_string_converter(typ: Union[Callable, TypeForm[Any]]) -> bool: return True type_annotations = _resolver.get_type_hints_with_backported_syntax(typ) + # Some checks we can do if the signature is available! for i, param in enumerate(signature.parameters.values()): annotation = type_annotations.get(param.name, param.annotation) annotation = _resolver.TypeParamResolver.concretize_type_params(annotation) if i == 0 and not ( - (get_origin(annotation) is Union and str in get_args(annotation)) + ( + get_origin(annotation) in (Union, _resolver.UnionType) + and str in get_args(annotation) + ) or annotation in (str, inspect.Parameter.empty) ): return False @@ -189,11 +193,11 @@ def instantiator(strings: List[str]) -> None: # `isinstance(x, NewType)` doesn't work because NewType isn't a class until # Python 3.10, so we instead do a duck typing-style check. metavar = getattr(typ, "__name__", "").upper() - typ, maybe_newtype_name = _resolver.unwrap_newtype_and_aliases(typ) - if maybe_newtype_name is not None: - metavar = maybe_newtype_name.upper() - - typ = _resolver.unwrap_annotated_and_aliases(typ) + typ, type_alias_breadcrumbs = _resolver.unwrap_annotated( + typ, _resolver.TyroTypeAliasBreadCrumb + ) + if len(type_alias_breadcrumbs) > 0: + metavar = type_alias_breadcrumbs[-1].name # Address container types. If a matching container is found, this will recursively # call instantiator_from_type(). @@ -403,7 +407,7 @@ def _instantiator_from_container_type( ), _instantiator_from_tuple: (tuple,), _instantiator_from_dict: (dict, collections.abc.Mapping), - _instantiator_from_union: (Union,), + _instantiator_from_union: (Union, _resolver.UnionType), _instantiator_from_literal: (Literal, LiteralAlternate), }.items(): if type_origin in matched_origins: diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 76c1b2257..0be0b4837 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -75,7 +75,7 @@ def from_callable_or_type( """Create a parser definition from a callable or type.""" # Consolidate subcommand types. - markers = _resolver.unwrap_annotated_and_aliases(f, _markers._Marker)[1] + markers = _resolver.unwrap_annotated(f, _markers._Marker)[1] consolidate_subcommand_args = _markers.ConsolidateSubcommandArgs in markers # Resolve the type of `f`, generate a field list. @@ -328,7 +328,7 @@ def handle_field( if _fields.is_nested_type(field.type_or_callable, field.default): field = dataclasses.replace( field, - type_or_callable=_resolver.unwrap_newtype_and_narrow_subtypes( + type_or_callable=_resolver.narrow_subtypes( field.type_or_callable, field.default, ), @@ -386,8 +386,8 @@ def from_field( extern_prefix: str, ) -> Optional[SubparsersSpecification]: # Union of classes should create subparsers. - typ = _resolver.unwrap_annotated_and_aliases(field.type_or_callable) - if get_origin(typ) is not Union: + typ = _resolver.unwrap_annotated(field.type_or_callable) + if get_origin(typ) not in (Union, _resolver.UnionType): return None # We don't use sets here to retain order of subcommands. @@ -403,7 +403,7 @@ def from_field( # If specified, swap types using tyro.conf.subcommand(constructor=...). for i, option in enumerate(options): - _, found_subcommand_configs = _resolver.unwrap_annotated_and_aliases( + _, found_subcommand_configs = _resolver.unwrap_annotated( option, _confstruct._SubcommandConfiguration ) if ( @@ -413,7 +413,7 @@ def from_field( options[i] = Annotated.__class_getitem__( # type: ignore ( found_subcommand_configs[0].constructor_factory(), - *_resolver.unwrap_annotated_and_aliases(option, "all")[1], + *_resolver.unwrap_annotated(option, "all")[1], ) ) @@ -439,10 +439,8 @@ def from_field( else extern_prefix, type(None) if option is none_proxy else cast(type, option), ) - option_unwrapped, found_subcommand_configs = ( - _resolver.unwrap_annotated_and_aliases( - option, _confstruct._SubcommandConfiguration - ) + option_unwrapped, found_subcommand_configs = _resolver.unwrap_annotated( + option, _confstruct._SubcommandConfiguration ) if len(found_subcommand_configs) != 0: # Explicitly annotated default. @@ -506,9 +504,7 @@ def from_field( # Strip the subcommand config from the option type. # Relevant: https://github.com/brentyi/tyro/pull/117 - option_origin, annotations = _resolver.unwrap_annotated_and_aliases( - option, "all" - ) + option_origin, annotations = _resolver.unwrap_annotated(option, "all") annotations = tuple( a for a in annotations diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 560354767..4483a6955 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -16,7 +16,6 @@ Dict, FrozenSet, List, - Optional, Sequence, Set, Tuple, @@ -42,20 +41,27 @@ from . import _fields, _unsafe_cache, conf from ._typing import TypeForm +UnionType = getattr(types, "UnionType", Union) +"""Same as types.UnionType, but points to typing.Union for older versions of +Python. types.UnionType was added in Python 3.10, and is created when the `X | +Y` syntax is used for unions.""" + TypeOrCallable = TypeVar("TypeOrCallable", TypeForm[Any], Callable) -def unwrap_aliases(typ: TypeOrCallable) -> TypeOrCallable: - """Unwrap type aliases.""" - if isinstance(typ, TypeAliasType): - return unwrap_aliases(cast(Any, typ.__value__)) - return typ +@dataclasses.dataclass(frozen=True) +class TyroTypeAliasBreadCrumb: + """A breadcrumb we can leave behind to track names of type aliases and + `NewType` types. We can use type alias names to auto-populate + subcommands.""" + + name: str def unwrap_origin_strip_extras(typ: TypeOrCallable) -> TypeOrCallable: """Returns the origin, ignoring typing.Annotated, of typ if it exists. Otherwise, returns typ.""" - typ = unwrap_annotated_and_aliases(typ) + typ = unwrap_annotated(typ) origin = get_origin(typ) if origin is not None: @@ -124,12 +130,17 @@ def type_from_typevar_constraints(typ: TypeOrCallable) -> TypeOrCallable: TypeOrCallableOrNone = TypeVar("TypeOrCallableOrNone", Callable, TypeForm[Any], None) -def unwrap_newtype_and_aliases( +def resolve_newtype_and_aliases( typ: TypeOrCallableOrNone, -) -> Tuple[TypeOrCallableOrNone, Optional[str]]: +) -> TypeOrCallableOrNone: # Handle type aliases, eg via the `type` statement in Python 3.12. if isinstance(typ, TypeAliasType): - return unwrap_newtype_and_aliases(typ.__value__) # type: ignore + return Annotated.__class_getitem__( + ( + cast(Any, resolve_newtype_and_aliases(typ.__value__)), + TyroTypeAliasBreadCrumb(typ.__name__), + ) + ) # We'll unwrap NewType annotations here; this is needed before issubclass # checks! @@ -140,13 +151,16 @@ def unwrap_newtype_and_aliases( while hasattr(typ, "__name__") and hasattr(typ, "__supertype__"): if return_name is None: return_name = getattr(typ, "__name__") - typ = getattr(typ, "__supertype__") + typ = resolve_newtype_and_aliases(getattr(typ, "__supertype__")) - return typ, return_name + if return_name is not None: + typ = Annotated.__class_getitem__((typ, TyroTypeAliasBreadCrumb(return_name))) # type: ignore + + return cast(TypeOrCallableOrNone, typ) @_unsafe_cache.unsafe_cache(maxsize=1024) -def unwrap_newtype_and_narrow_subtypes( +def narrow_subtypes( typ: TypeOrCallable, default_instance: Any, ) -> TypeOrCallable: @@ -158,8 +172,7 @@ def unwrap_newtype_and_narrow_subtypes( string default is passed in, we don't want to narrow the type to always be strings!)""" - typ, unused_name = unwrap_newtype_and_aliases(typ) - del unused_name + typ = resolve_newtype_and_aliases(typ) try: potential_subclass = type(default_instance) @@ -169,7 +182,7 @@ def unwrap_newtype_and_narrow_subtypes( # it doesn't really make sense to parse this case. return typ - superclass = unwrap_annotated_and_aliases(typ) + superclass = unwrap_annotated(typ) # For Python 3.10. if get_origin(superclass) is Union: @@ -194,7 +207,7 @@ def swap_type_using_confstruct(typ: TypeOrCallable) -> TypeOrCallable: `tyro.conf.arg` and `tyro.conf.subcommand`. Runtime annotations are kept, but the type is swapped.""" # Need to swap types. - _, annotations = unwrap_annotated_and_aliases(typ, search_type="all") + _, annotations = unwrap_annotated(typ, search_type="all") for anno in reversed(annotations): if ( isinstance( @@ -256,27 +269,27 @@ def narrow_collection_types( @overload -def unwrap_annotated_and_aliases( +def unwrap_annotated( typ: TypeOrCallable, search_type: TypeForm[MetadataType], ) -> Tuple[TypeOrCallable, Tuple[MetadataType, ...]]: ... @overload -def unwrap_annotated_and_aliases( +def unwrap_annotated( typ: TypeOrCallable, search_type: Literal["all"], ) -> Tuple[TypeOrCallable, Tuple[Any, ...]]: ... @overload -def unwrap_annotated_and_aliases( +def unwrap_annotated( typ: TypeOrCallable, search_type: None = None, ) -> TypeOrCallable: ... -def unwrap_annotated_and_aliases( +def unwrap_annotated( typ: TypeOrCallable, search_type: Union[TypeForm[MetadataType], Literal["all"], object, None] = None, ) -> Union[Tuple[TypeOrCallable, Tuple[MetadataType, ...]], TypeOrCallable]: @@ -289,8 +302,7 @@ def unwrap_annotated_and_aliases( """ # Unwrap aliases defined using Python 3.12's `type` syntax. - if isinstance(typ, TypeAliasType): - return unwrap_annotated_and_aliases(typ.__value__, search_type) + typ = resolve_newtype_and_aliases(typ) # `Final` and `ReadOnly` types are ignored in tyro. while get_origin(typ) in STRIP_WRAPPER_TYPES: @@ -350,28 +362,41 @@ def get_assignment_context(cls, typ: TypeOrCallable) -> TypeParamAssignmentConte return TypeParamAssignmentContext(typ, type_from_typevar) @staticmethod - def concretize_type_params(typ: TypeOrCallable) -> TypeOrCallable: + def concretize_type_params( + typ: TypeOrCallable, seen: set[Any] | None = None + ) -> TypeOrCallable: """Apply type parameter assignments based on the current context.""" - typ = unwrap_aliases(typ) + if seen is None: + seen = set() + elif seen is not None and typ in seen: + # Found a cycle. We don't (currently) support recursive types. + return typ + else: + seen.add(typ) + + typ = resolve_newtype_and_aliases(typ) + type_from_typevar = {} GenericAlias = getattr(types, "GenericAlias", None) - if ( + while ( GenericAlias is not None and isinstance(typ, GenericAlias) and len(getattr(typ, "__type_params__", ())) > 0 ): - type_from_typevar = {} for k, v in zip(typ.__type_params__, get_args(typ)): # type: ignore - type_from_typevar[k] = v # type: ignore + type_from_typevar[k] = TypeParamResolver.concretize_type_params( + v, seen=seen + ) typ = typ.__value__ # type: ignore - with TypeParamAssignmentContext(typ, type_from_typevar): - return TypeParamResolver._concretize_type_params(typ) + if len(type_from_typevar) == 0: + return TypeParamResolver._concretize_type_params(typ, seen=seen) else: - return TypeParamResolver._concretize_type_params(typ) + with TypeParamAssignmentContext(typ, type_from_typevar): + return TypeParamResolver._concretize_type_params(typ, seen=seen) @staticmethod - def _concretize_type_params(typ: TypeOrCallable) -> TypeOrCallable: + def _concretize_type_params(typ: TypeOrCallable, seen: set[Any]) -> TypeOrCallable: for type_from_typevar in reversed(TypeParamResolver.param_assignments): if typ in type_from_typevar: return type_from_typevar[typ] # type: ignore @@ -394,7 +419,8 @@ def _concretize_type_params(typ: TypeOrCallable) -> TypeOrCallable: new_args_list.append(x) new_args = tuple( - TypeParamResolver.concretize_type_params(x) for x in new_args_list + TypeParamResolver.concretize_type_params(x, seen=seen) + for x in new_args_list ) # Apply shims to convert from types.UnionType to typing.Union, list to typing.List, etc. @@ -432,10 +458,8 @@ def standardize_builtin_generics(typ: TypeOrCallable) -> TypeOrCallable: set: Set, frozenset: FrozenSet, type: Type, + UnionType: Union, } - if hasattr(types, "UnionType"): # type: ignore - # PEP 604. Requires Python 3.10. - shim_table[types.UnionType] = Union # type: ignore if origin in shim_table: # type: ignore typ = shim_table[origin].__getitem__(args) # type: ignore @@ -505,14 +529,14 @@ def resolve_generic_types( # ^We need this `if` statement for an obscure edge case: when `cls` is a # function with `__tyro_markers__` set, we don't want/need to return # Annotated[func, markers]. - typ, annotations = unwrap_annotated_and_aliases(typ, "all") + typ, annotations = unwrap_annotated(typ, "all") # Apply shims to convert from types.UnionType to typing.Union, list to # typing.List, etc. typ = standardize_builtin_generics(typ) # We'll ignore NewType when getting the origin + args for generics. - origin_cls = get_origin(unwrap_newtype_and_aliases(typ)[0]) + origin_cls = get_origin(resolve_newtype_and_aliases(typ)) type_from_typevar: Dict[TypeVar, TypeForm[Any]] = {} # Support typing.Self. @@ -531,7 +555,7 @@ def resolve_generic_types( and hasattr(origin_cls.__parameters__, "__len__") ): typevars = origin_cls.__parameters__ - typevar_values = get_args(unwrap_newtype_and_aliases(typ)[0]) + typevar_values = get_args(resolve_newtype_and_aliases(typ)) assert len(typevars) == len(typevar_values) typ = origin_cls type_from_typevar.update(dict(zip(typevars, typevar_values))) diff --git a/src/tyro/_strings.py b/src/tyro/_strings.py index 9a3953e7d..a8427b05f 100644 --- a/src/tyro/_strings.py +++ b/src/tyro/_strings.py @@ -80,11 +80,14 @@ def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: from .conf import _confstruct # Prevent circular imports cls, type_from_typevar = _resolver.resolve_generic_types(cls) - cls, found_subcommand_configs = _resolver.unwrap_annotated_and_aliases( + _, type_alias_breadcrumbs = _resolver.unwrap_annotated( + cls, _resolver.TyroTypeAliasBreadCrumb + ) + cls, found_subcommand_configs = _resolver.unwrap_annotated( cls, _confstruct._SubcommandConfiguration ) - # Subparser name from `tyro.metadata.subcommand()`. + # Subparser name from `tyro.conf.subcommand()`. found_name = None prefix_name = True if len(found_subcommand_configs) > 0: @@ -94,6 +97,13 @@ def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: if found_name is not None: return found_name, prefix_name + # Subparser name from type alias. This is lower priority thant he name from + # `tyro.conf.subcommand()`. + if len(type_alias_breadcrumbs) > 0: + return hyphen_separated_from_camel_case( + type_alias_breadcrumbs[-1].name + ), prefix_name + # Subparser name from class name. def get_name(cls: Type) -> str: orig = get_origin(cls) diff --git a/src/tyro/_subcommand_matching.py b/src/tyro/_subcommand_matching.py index 02daccc5f..bc01dceb1 100644 --- a/src/tyro/_subcommand_matching.py +++ b/src/tyro/_subcommand_matching.py @@ -96,12 +96,10 @@ def _get_type_options(typ: _typing.TypeForm) -> Tuple[_typing.TypeForm, ...]: # Check against supertypes. for self_type in self_types: - self_type = _resolver.unwrap_annotated_and_aliases(self_type) - self_type, _ = _resolver.unwrap_newtype_and_aliases(self_type) + self_type = _resolver.unwrap_annotated(self_type) ok = False for super_type in super_types: - super_type = _resolver.unwrap_annotated_and_aliases(super_type) - self_type, _ = _resolver.unwrap_newtype_and_aliases(self_type) + super_type = _resolver.unwrap_annotated(super_type) if issubclass(self_type, super_type): ok = True if not ok: diff --git a/src/tyro/extras/_serialization.py b/src/tyro/extras/_serialization.py index ae77abb40..bfb5a28d2 100644 --- a/src/tyro/extras/_serialization.py +++ b/src/tyro/extras/_serialization.py @@ -35,7 +35,7 @@ def _get_contained_special_types_from_type( else _parent_contained_dataclasses ) - cls = _resolver.unwrap_annotated_and_aliases(cls) + cls = _resolver.unwrap_annotated(cls) cls, type_from_typevar = _resolver.resolve_generic_types(cls) contained_special_types = {cls} diff --git a/tests/test_nested.py b/tests/test_nested.py index 0264057df..0eec9d258 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -323,6 +323,44 @@ class Subparser: tyro.cli(Subparser, args=["--x", "1", "bc:smtp-server", "--bc.y", "3"]) +def test_subparser_newtype() -> None: + @dataclasses.dataclass + class HTTPServer: + y: int + + @dataclasses.dataclass + class SMTPServer: + z: int + + HTTPServer1 = NewType("HTTPServer1", HTTPServer) + HTTPServer2 = NewType("HTTPServer2", HTTPServer) + + @dataclasses.dataclass + class Subparser: + x: int + bc: Union[HTTPServer1, HTTPServer2, SMTPServer] + + assert tyro.cli( + Subparser, args=["--x", "1", "bc:http-server1", "--bc.y", "3"] + ) == Subparser(x=1, bc=HTTPServer1(HTTPServer(y=3))) + assert tyro.cli( + Subparser, args=["--x", "1", "bc:http-server2", "--bc.y", "3"] + ) == Subparser(x=1, bc=HTTPServer2(HTTPServer(y=3))) + assert tyro.cli( + Subparser, args=["--x", "1", "bc:smtp-server", "--bc.z", "3"] + ) == Subparser(x=1, bc=SMTPServer(z=3)) + + with pytest.raises(SystemExit): + # Missing subcommand. + tyro.cli(Subparser, args=["--x", "1"]) + with pytest.raises(SystemExit): + # Wrong field. + tyro.cli(Subparser, args=["--x", "1", "bc:http-server1", "--bc.z", "3"]) + with pytest.raises(SystemExit): + # Wrong field. + tyro.cli(Subparser, args=["--x", "1", "bc:smtp-server", "--bc.y", "3"]) + + def test_subparser_root() -> None: @dataclasses.dataclass class HTTPServer: diff --git a/tests/test_new_style_annotations_min_py312.py b/tests/test_new_style_annotations_min_py312.py index b7b6ea055..5e6a7ef0a 100644 --- a/tests/test_new_style_annotations_min_py312.py +++ b/tests/test_new_style_annotations_min_py312.py @@ -2,9 +2,12 @@ # # PEP 695 isn't yet supported in mypy. (April 4, 2024) from dataclasses import dataclass -from typing import Annotated +from typing import Annotated, Any, NewType + +import pytest import tyro +from tyro.conf._markers import OmitArgPrefixes def test_simple_generic(): @@ -90,3 +93,111 @@ class Config: arg: Renamed[bool] assert tyro.cli(Config, args=["--renamed", "True"]) == Config(arg=True) + + +def test_pep695_generic_alias_rename_override() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: Annotated[Renamed[bool], tyro.conf.arg(name="renamed2")] + + assert tyro.cli(Config, args=["--renamed2", "True"]) == Config(arg=True) + + +type RenamedTwice[T] = Renamed[Renamed[T]] + + +def test_pep695_generic_alias_rename_twice() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: RenamedTwice[bool] + + assert tyro.cli(Config, args=["--renamed", "True"]) == Config(arg=True) + + +type SomeUnion = bool | int +type RenamedThreeTimes[T] = Renamed[Renamed[Renamed[T]]] +type SomeUnionRenamed = RenamedThreeTimes[SomeUnion] + + +def test_pep695_generic_alias_rename_three_times() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: Annotated[SomeUnionRenamed, tyro.conf.arg(name="renamed_override")] + + assert tyro.cli(Config, args=["--renamed-override", "True"]) == Config(arg=True) + + +type RecursiveList[T] = T | list[RecursiveList[T]] + + +def test_pep695_recursive_types() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: RecursiveList[str] + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(Config, args=["--arg", "True"]) + + +def test_pep695_recursive_types_custom_constructor() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: Annotated[RecursiveList[str], tyro.conf.arg(constructor=str)] + + assert tyro.cli(Config, args=["--arg", "True"]) == Config(arg="True") + + +type UnprefixedSubcommandPair[T1, T2, T3] = ( + Annotated[OmitArgPrefixes[T1], tyro.conf.subcommand(prefix_name=False)] # type: ignore + | Annotated[T2, tyro.conf.subcommand(prefix_name=False)] # type: ignore + | Annotated[T3, tyro.conf.subcommand(prefix_name=True)] +) +type IntContainer = Inner[int] +type IntContainerIntermediate = Inner[int] +type IntContainer2 = IntContainerIntermediate +type StrContainer = Inner[str] + + +def test_pep695_alias_subcommand() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: UnprefixedSubcommandPair[IntContainer, IntContainer2, StrContainer] + + assert tyro.cli(Config, args=["int-container", "--a", "3", "--b", "5"]) == Config( + Inner(3, 5) + ) + assert tyro.cli( + Config, args=["int-container2", "--arg.a", "3", "--arg.b", "5"] + ) == Config(Inner(3, 5)) + assert tyro.cli( + Config, args=["arg:str-container", "--arg.a", "3", "--arg.b", "5"] + ) == Config(Inner("3", "5")) + + +type Int0 = int +Int1 = NewType("Int1", Int0) +type Int2 = Int1 +Int3 = NewType("Int3", Int2) +type Int4 = Int3 +Int5 = NewType("Int5", Int4) +type Int6 = Int5 +Int7 = NewType("Int7", Int6) + + +def test_pep695_new_type_alias() -> None: + def main(arg: list[Int7], /) -> Any: + return arg + + assert tyro.cli(main, args=["1", "2"]) == [1, 2] diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index 5fb5b31be..d48b096df 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -333,6 +333,44 @@ class Subparser: tyro.cli(Subparser, args=["--x", "1", "bc:smtp-server", "--bc.y", "3"]) +def test_subparser_newtype() -> None: + @dataclasses.dataclass + class HTTPServer: + y: int + + @dataclasses.dataclass + class SMTPServer: + z: int + + HTTPServer1 = NewType("HTTPServer1", HTTPServer) + HTTPServer2 = NewType("HTTPServer2", HTTPServer) + + @dataclasses.dataclass + class Subparser: + x: int + bc: HTTPServer1 | HTTPServer2 | SMTPServer + + assert tyro.cli( + Subparser, args=["--x", "1", "bc:http-server1", "--bc.y", "3"] + ) == Subparser(x=1, bc=HTTPServer1(HTTPServer(y=3))) + assert tyro.cli( + Subparser, args=["--x", "1", "bc:http-server2", "--bc.y", "3"] + ) == Subparser(x=1, bc=HTTPServer2(HTTPServer(y=3))) + assert tyro.cli( + Subparser, args=["--x", "1", "bc:smtp-server", "--bc.z", "3"] + ) == Subparser(x=1, bc=SMTPServer(z=3)) + + with pytest.raises(SystemExit): + # Missing subcommand. + tyro.cli(Subparser, args=["--x", "1"]) + with pytest.raises(SystemExit): + # Wrong field. + tyro.cli(Subparser, args=["--x", "1", "bc:http-server1", "--bc.z", "3"]) + with pytest.raises(SystemExit): + # Wrong field. + tyro.cli(Subparser, args=["--x", "1", "bc:smtp-server", "--bc.y", "3"]) + + def test_subparser_root() -> None: @dataclasses.dataclass class HTTPServer: diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py index b7b6ea055..5e6a7ef0a 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py @@ -2,9 +2,12 @@ # # PEP 695 isn't yet supported in mypy. (April 4, 2024) from dataclasses import dataclass -from typing import Annotated +from typing import Annotated, Any, NewType + +import pytest import tyro +from tyro.conf._markers import OmitArgPrefixes def test_simple_generic(): @@ -90,3 +93,111 @@ class Config: arg: Renamed[bool] assert tyro.cli(Config, args=["--renamed", "True"]) == Config(arg=True) + + +def test_pep695_generic_alias_rename_override() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: Annotated[Renamed[bool], tyro.conf.arg(name="renamed2")] + + assert tyro.cli(Config, args=["--renamed2", "True"]) == Config(arg=True) + + +type RenamedTwice[T] = Renamed[Renamed[T]] + + +def test_pep695_generic_alias_rename_twice() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: RenamedTwice[bool] + + assert tyro.cli(Config, args=["--renamed", "True"]) == Config(arg=True) + + +type SomeUnion = bool | int +type RenamedThreeTimes[T] = Renamed[Renamed[Renamed[T]]] +type SomeUnionRenamed = RenamedThreeTimes[SomeUnion] + + +def test_pep695_generic_alias_rename_three_times() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: Annotated[SomeUnionRenamed, tyro.conf.arg(name="renamed_override")] + + assert tyro.cli(Config, args=["--renamed-override", "True"]) == Config(arg=True) + + +type RecursiveList[T] = T | list[RecursiveList[T]] + + +def test_pep695_recursive_types() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: RecursiveList[str] + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(Config, args=["--arg", "True"]) + + +def test_pep695_recursive_types_custom_constructor() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: Annotated[RecursiveList[str], tyro.conf.arg(constructor=str)] + + assert tyro.cli(Config, args=["--arg", "True"]) == Config(arg="True") + + +type UnprefixedSubcommandPair[T1, T2, T3] = ( + Annotated[OmitArgPrefixes[T1], tyro.conf.subcommand(prefix_name=False)] # type: ignore + | Annotated[T2, tyro.conf.subcommand(prefix_name=False)] # type: ignore + | Annotated[T3, tyro.conf.subcommand(prefix_name=True)] +) +type IntContainer = Inner[int] +type IntContainerIntermediate = Inner[int] +type IntContainer2 = IntContainerIntermediate +type StrContainer = Inner[str] + + +def test_pep695_alias_subcommand() -> None: + """Adapted from: https://github.com/brentyi/tyro/issues/177""" + + @dataclass(frozen=True) + class Config: + arg: UnprefixedSubcommandPair[IntContainer, IntContainer2, StrContainer] + + assert tyro.cli(Config, args=["int-container", "--a", "3", "--b", "5"]) == Config( + Inner(3, 5) + ) + assert tyro.cli( + Config, args=["int-container2", "--arg.a", "3", "--arg.b", "5"] + ) == Config(Inner(3, 5)) + assert tyro.cli( + Config, args=["arg:str-container", "--arg.a", "3", "--arg.b", "5"] + ) == Config(Inner("3", "5")) + + +type Int0 = int +Int1 = NewType("Int1", Int0) +type Int2 = Int1 +Int3 = NewType("Int3", Int2) +type Int4 = Int3 +Int5 = NewType("Int5", Int4) +type Int6 = Int5 +Int7 = NewType("Int7", Int6) + + +def test_pep695_new_type_alias() -> None: + def main(arg: list[Int7], /) -> Any: + return arg + + assert tyro.cli(main, args=["1", "2"]) == [1, 2] From 1a9448a68e2e9f70df183554909fabc2d4434528 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 22 Oct 2024 11:44:03 -0700 Subject: [PATCH 465/491] Support cyclic generics (#182) * Less brute-force generics handling * Add back cycle detection * Type narrowing for `Any` * Improve subcommand matching, tests * Cleanup * Cleanup * attrs/pydantic import cleanup * mypy * More cleanup * Cleanup * Cleanup * Fix set mutation --- src/tyro/_cli.py | 196 +++++++++--------- src/tyro/_docstrings.py | 11 +- src/tyro/_fields.py | 49 +++-- src/tyro/_parsers.py | 80 ++++--- src/tyro/_resolver.py | 13 +- src/tyro/_strings.py | 9 +- src/tyro/_subcommand_matching.py | 51 +++-- tests/test_nested.py | 6 +- tests/test_new_style_annotations_min_py312.py | 93 ++++++++- .../test_nested_generated.py | 6 +- ...w_style_annotations_min_py312_generated.py | 93 ++++++++- 11 files changed, 405 insertions(+), 202 deletions(-) diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index 760d78e86..eda141863 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -20,8 +20,6 @@ import shtab from typing_extensions import Literal -from tyro._resolver import TypeParamResolver - from . import _argparse as argparse from . import ( _argparse_formatter, @@ -318,110 +316,106 @@ def _cli_impl( stacklevel=2, ) - resolve_context = TypeParamResolver.get_assignment_context(f) - with resolve_context: - f = resolve_context.origin_type - - # Internally, we distinguish between two concepts: - # - "default", which is used for individual arguments. - # - "default_instance", which is used for _fields_ (which may be broken down into - # one or many arguments, depending on various factors). - # - # This could be revisited. - default_instance_internal: Union[_fields.NonpropagatingMissingType, OutT] = ( - _fields.MISSING_NONPROP if default is None else default - ) + # Internally, we distinguish between two concepts: + # - "default", which is used for individual arguments. + # - "default_instance", which is used for _fields_ (which may be broken down into + # one or many arguments, depending on various factors). + # + # This could be revisited. + default_instance_internal: Union[_fields.NonpropagatingMissingType, OutT] = ( + _fields.MISSING_NONPROP if default is None else default + ) - # We wrap our type with a dummy dataclass if it can't be treated as a nested type. - # For example: passing in f=int will result in a dataclass with a single field - # typed as int. - if not _fields.is_nested_type(cast(type, f), default_instance_internal): - dummy_field = cast( - dataclasses.Field, - dataclasses.field(), - ) - f = dataclasses.make_dataclass( - cls_name="dummy", - fields=[(_strings.dummy_field_name, cast(type, f), dummy_field)], - frozen=True, - ) - default_instance_internal = f(default_instance_internal) # type: ignore - dummy_wrapped = True + # We wrap our type with a dummy dataclass if it can't be treated as a nested type. + # For example: passing in f=int will result in a dataclass with a single field + # typed as int. + if not _fields.is_nested_type(cast(type, f), default_instance_internal): + dummy_field = cast( + dataclasses.Field, + dataclasses.field(), + ) + f = dataclasses.make_dataclass( + cls_name="dummy", + fields=[(_strings.dummy_field_name, cast(type, f), dummy_field)], + frozen=True, + ) + default_instance_internal = f(default_instance_internal) # type: ignore + dummy_wrapped = True + else: + dummy_wrapped = False + + # Read and fix arguments. If the user passes in --field_name instead of + # --field-name, correct for them. + args = list(sys.argv[1:]) if args is None else list(args) + + # Fix arguments. This will modify all option-style arguments replacing + # underscores with hyphens, or vice versa if use_underscores=True. + # If two options are ambiguous, e.g., --a_b and --a-b, raise a runtime error. + modified_args: Dict[str, str] = {} + for index, arg in enumerate(args): + if not arg.startswith("--"): + continue + + if "=" in arg: + arg, _, val = arg.partition("=") + fixed = "--" + _strings.replace_delimeter_in_part(arg[2:]) + "=" + val else: - dummy_wrapped = False - - # Read and fix arguments. If the user passes in --field_name instead of - # --field-name, correct for them. - args = list(sys.argv[1:]) if args is None else list(args) - - # Fix arguments. This will modify all option-style arguments replacing - # underscores with hyphens, or vice versa if use_underscores=True. - # If two options are ambiguous, e.g., --a_b and --a-b, raise a runtime error. - modified_args: Dict[str, str] = {} - for index, arg in enumerate(args): - if not arg.startswith("--"): - continue - - if "=" in arg: - arg, _, val = arg.partition("=") - fixed = "--" + _strings.replace_delimeter_in_part(arg[2:]) + "=" + val - else: - fixed = "--" + _strings.replace_delimeter_in_part(arg[2:]) - if ( - return_unknown_args - and fixed in modified_args - and modified_args[fixed] != arg - ): - raise RuntimeError( - "Ambiguous arguments: " + modified_args[fixed] + " and " + arg - ) - modified_args[fixed] = arg - args[index] = fixed - - # If we pass in the --tyro-print-completion or --tyro-write-completion flags: turn - # formatting tags, and get the shell we want to generate a completion script for - # (bash/zsh/tcsh). - # - # shtab also offers an add_argument_to() functions that fulfills a similar goal, but - # manual parsing of argv is convenient for turning off formatting. - # - # Note: --tyro-print-completion is deprecated! --tyro-write-completion is less prone - # to errors from accidental logging, print statements, etc. - print_completion = False - write_completion = False - if len(args) >= 2: - # We replace underscores with hyphens to accomodate for `use_undercores`. - print_completion = args[0].replace("_", "-") == "--tyro-print-completion" - write_completion = ( - len(args) >= 3 - and args[0].replace("_", "-") == "--tyro-write-completion" + fixed = "--" + _strings.replace_delimeter_in_part(arg[2:]) + if ( + return_unknown_args + and fixed in modified_args + and modified_args[fixed] != arg + ): + raise RuntimeError( + "Ambiguous arguments: " + modified_args[fixed] + " and " + arg ) - - # Note: setting USE_RICH must happen before the parser specification is generated. - # TODO: revisit this. Ideally we should be able to eliminate the global state - # changes. - completion_shell = None - completion_target_path = None - if print_completion or write_completion: - completion_shell = args[1] - if write_completion: - completion_target_path = pathlib.Path(args[2]) - if print_completion or write_completion or return_parser: - _arguments.USE_RICH = False - else: - _arguments.USE_RICH = True - - # Map a callable to the relevant CLI arguments + subparsers. - parser_spec = _parsers.ParserSpecification.from_callable_or_type( - f, - description=description, - parent_classes=set(), # Used for recursive calls. - default_instance=default_instance_internal, # Overrides for default values. - intern_prefix="", # Used for recursive calls. - extern_prefix="", # Used for recursive calls. - subcommand_prefix="", # Used for recursive calls. + modified_args[fixed] = arg + args[index] = fixed + + # If we pass in the --tyro-print-completion or --tyro-write-completion flags: turn + # formatting tags, and get the shell we want to generate a completion script for + # (bash/zsh/tcsh). + # + # shtab also offers an add_argument_to() functions that fulfills a similar goal, but + # manual parsing of argv is convenient for turning off formatting. + # + # Note: --tyro-print-completion is deprecated! --tyro-write-completion is less prone + # to errors from accidental logging, print statements, etc. + print_completion = False + write_completion = False + if len(args) >= 2: + # We replace underscores with hyphens to accomodate for `use_undercores`. + print_completion = args[0].replace("_", "-") == "--tyro-print-completion" + write_completion = ( + len(args) >= 3 and args[0].replace("_", "-") == "--tyro-write-completion" ) + # Note: setting USE_RICH must happen before the parser specification is generated. + # TODO: revisit this. Ideally we should be able to eliminate the global state + # changes. + completion_shell = None + completion_target_path = None + if print_completion or write_completion: + completion_shell = args[1] + if write_completion: + completion_target_path = pathlib.Path(args[2]) + if print_completion or write_completion or return_parser: + _arguments.USE_RICH = False + else: + _arguments.USE_RICH = True + + # Map a callable to the relevant CLI arguments + subparsers. + parser_spec = _parsers.ParserSpecification.from_callable_or_type( + f, + markers=set(), + description=description, + parent_classes=set(), # Used for recursive calls. + default_instance=default_instance_internal, # Overrides for default values. + intern_prefix="", # Used for recursive calls. + extern_prefix="", # Used for recursive calls. + subcommand_prefix="", # Used for recursive calls. + ) + # Generate parser! with _argparse_formatter.ansi_context(): parser = _argparse_formatter.TyroArgumentParser( diff --git a/src/tyro/_docstrings.py b/src/tyro/_docstrings.py index 9b7cbbae3..6eadae785 100644 --- a/src/tyro/_docstrings.py +++ b/src/tyro/_docstrings.py @@ -7,6 +7,7 @@ import inspect import io import itertools +import sys import tokenize from typing import Callable, Dict, Generic, Hashable, List, Optional, Set, Type, TypeVar @@ -311,9 +312,13 @@ def get_callable_description(f: Callable) -> str: if isinstance(f, functools.partial): f = f.func - try: - import pydantic - except ImportError: + if "pydantic" in sys.modules.keys(): + try: + import pydantic + except ImportError: + # Needed for mock import test. + pydantic = None # type: ignore + else: pydantic = None # type: ignore # Note inspect.getdoc() causes some corner cases with TypedDicts. diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 6329737c2..275595dd1 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -55,7 +55,7 @@ from ._typing import TypeForm from .conf import _confstruct, _markers -_field_context_markers: List[Tuple[_markers.Marker, ...]] = [] +global_context_markers: List[Tuple[_markers.Marker, ...]] = [] @dataclasses.dataclass @@ -63,9 +63,6 @@ class FieldDefinition: intern_name: str extern_name: str type_or_callable: Union[TypeForm[Any], Callable] - typevar_context: _resolver.TypeParamAssignmentContext - """Type or callable for this field. This should have all Annotated[] annotations - stripped.""" default: Any # We need to record whether defaults are from default instances to # determine if they should override the default in @@ -94,9 +91,9 @@ def __post_init__(self): def marker_context(markers: Tuple[_markers.Marker, ...]): """Context for setting markers on fields. All fields created within the context will have the specified markers.""" - _field_context_markers.append(markers) + global_context_markers.append(markers) yield - _field_context_markers.pop() + global_context_markers.pop() @staticmethod def make( @@ -113,15 +110,19 @@ def make( type_or_callable = _resolver.TypeParamResolver.concretize_type_params( type_or_callable ) - typevar_context = _resolver.TypeParamResolver.get_assignment_context( - type_or_callable - ) - type_or_callable = typevar_context.origin_type # Narrow types. - type_or_callable = _resolver.type_from_typevar_constraints(type_or_callable) - type_or_callable = _resolver.narrow_collection_types(type_or_callable, default) - type_or_callable = _resolver.narrow_union_type(type_or_callable, default) + if type_or_callable is Any and default not in MISSING_SINGLETONS: + type_or_callable = type(default) + else: + # TypeVar constraints are already applied in + # TypeParamResolver.concretize_type_params(), but that won't be + # called for functions. + type_or_callable = _resolver.type_from_typevar_constraints(type_or_callable) + type_or_callable = _resolver.narrow_collection_types( + type_or_callable, default + ) + type_or_callable = _resolver.narrow_union_type(type_or_callable, default) # Try to extract argconf overrides from type. _, argconfs = _resolver.unwrap_annotated( @@ -155,7 +156,7 @@ def make( markers = inferred_markers + markers # Include markers set via context manager. - for context_markers in _field_context_markers: + for context_markers in global_context_markers: markers += context_markers # Check that the default value matches the final resolved type. @@ -190,7 +191,6 @@ def make( type_or_callable=type_or_callable if argconf.constructor_factory is None else argconf.constructor_factory(), - typevar_context=typevar_context, default=default, is_default_from_default_instance=is_default_from_default_instance, helptext=helptext, @@ -373,14 +373,11 @@ def _try_field_list_from_callable( f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance, ) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - # Resolve generic types. - resolve_context = _resolver.TypeParamResolver.get_assignment_context(f) - with resolve_context: - f = resolve_context.origin_type - - # Apply constructor_factory override for type. - f = _resolver.swap_type_using_confstruct(f) - + # Apply constructor_factory override for type. + f = _resolver.swap_type_using_confstruct(f) + typevar_context = _resolver.TypeParamResolver.get_assignment_context(f) + f = typevar_context.origin_type + with typevar_context: # Check for default instances in subcommand configs. This is needed for # is_nested_type() when arguments are not valid without a default, and this # default is specified in the subcommand config. @@ -618,7 +615,7 @@ def _field_list_from_dataclass( def _is_pydantic(cls: TypeForm[Any]) -> bool: # pydantic will already be imported if it's used. - if "pydantic" not in sys.modules.keys(): + if "pydantic" not in sys.modules.keys(): # pragma: no cover # This is needed for the mock import test in # test_missing_optional_packages.py to pass. return False @@ -626,6 +623,8 @@ def _is_pydantic(cls: TypeForm[Any]) -> bool: try: import pydantic except ImportError: + # This is needed for the mock import test in + # test_missing_optional_packages.py to pass. return False try: @@ -722,7 +721,7 @@ def _field_list_from_pydantic( def _is_attrs(cls: TypeForm[Any]) -> bool: # attr will already be imported if it's used. - if "attr" not in sys.modules.keys(): + if "attr" not in sys.modules.keys(): # pragma: no cover return False try: diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 0be0b4837..ef6bf0136 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -62,6 +62,7 @@ class ParserSpecification: @staticmethod def from_callable_or_type( f: Callable[..., T], + markers: Set[_markers._Marker], description: Optional[str], parent_classes: Set[Type[Any]], default_instance: Union[ @@ -75,19 +76,16 @@ def from_callable_or_type( """Create a parser definition from a callable or type.""" # Consolidate subcommand types. - markers = _resolver.unwrap_annotated(f, _markers._Marker)[1] + markers = markers | set(_resolver.unwrap_annotated(f, _markers._Marker)[1]) consolidate_subcommand_args = _markers.ConsolidateSubcommandArgs in markers # Resolve the type of `f`, generate a field list. - with _fields.FieldDefinition.marker_context(markers): - context = _resolver.TypeParamResolver.get_assignment_context(f) - with context: - f = context.origin_type - f, field_list = _fields.field_list_from_callable( - f=f, - default_instance=default_instance, - support_single_arg_types=support_single_arg_types, - ) + with _fields.FieldDefinition.marker_context(tuple(markers)): + f, field_list = _fields.field_list_from_callable( + f=f, + default_instance=default_instance, + support_single_arg_types=support_single_arg_types, + ) # Cycle detection. # @@ -113,14 +111,13 @@ def from_callable_or_type( subparsers_from_prefix = {} for field in field_list: - with field.typevar_context: - field_out = handle_field( - field, - parent_classes=parent_classes, - intern_prefix=intern_prefix, - extern_prefix=extern_prefix, - subcommand_prefix=subcommand_prefix, - ) + field_out = handle_field( + field, + parent_classes=parent_classes, + intern_prefix=intern_prefix, + extern_prefix=extern_prefix, + subcommand_prefix=subcommand_prefix, + ) if isinstance(field_out, _arguments.ArgumentDefinition): # Handle single arguments. args.append(field_out) @@ -334,14 +331,8 @@ def handle_field( ), ) return ParserSpecification.from_callable_or_type( - ( - # Recursively apply marker types. - field.type_or_callable - if len(field.markers) == 0 - else Annotated.__class_getitem__( # type: ignore - (field.type_or_callable,) + tuple(field.markers) - ) - ), + field.type_or_callable, + markers=field.markers, description=None, parent_classes=parent_classes, default_instance=field.default, @@ -458,16 +449,18 @@ def from_field( default_name = None else: default_name = _subcommand_matching.match_subcommand( - field.default, subcommand_config_from_name, subcommand_type_from_name + field.default, + subcommand_config_from_name, + subcommand_type_from_name, ) - # This should never be triggered because union types are expanded when - # a bad default value is provided. assert default_name is not None, ( f"`{extern_prefix}` was provided a default value of type" f" {type(field.default)} but no matching subcommand was found. A" " type may be missing in the Union type declaration for" - f" `{extern_prefix}`, which currently expects {options}." + f" `{extern_prefix}`, which currently expects {options}. " + "The types may also be too complex for tyro's subcommand matcher; support " + "is particularly limited for custom generic types." ) # Add subcommands for each option. @@ -517,21 +510,18 @@ def from_field( (option_origin,) + annotations ) - subparser = ParserSpecification.from_callable_or_type( - ( - # Recursively apply markers. - Annotated.__class_getitem__((option,) + tuple(field.markers)) # type: ignore - if len(field.markers) > 0 - else option - ), - description=subcommand_config.description, - parent_classes=parent_classes, - default_instance=subcommand_config.default, - intern_prefix=intern_prefix, - extern_prefix=extern_prefix, - subcommand_prefix=intern_prefix, - support_single_arg_types=True, - ) + with _fields.FieldDefinition.marker_context(tuple(field.markers)): + subparser = ParserSpecification.from_callable_or_type( + option, + markers=field.markers, + description=subcommand_config.description, + parent_classes=parent_classes, + default_instance=subcommand_config.default, + intern_prefix=intern_prefix, + extern_prefix=extern_prefix, + subcommand_prefix=intern_prefix, + support_single_arg_types=True, + ) # Apply prefix to helptext in nested classes in subparsers. subparser = dataclasses.replace( diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 4483a6955..74ca303c5 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -124,6 +124,8 @@ def type_from_typevar_constraints(typ: TypeOrCallable) -> TypeOrCallable: elif len(typ.__constraints__) > 0: # Try to infer type from TypeVar constraints. return Union.__getitem__(typ.__constraints__) # type: ignore + else: + return Any return typ @@ -534,9 +536,10 @@ def resolve_generic_types( # Apply shims to convert from types.UnionType to typing.Union, list to # typing.List, etc. typ = standardize_builtin_generics(typ) + typ = resolve_newtype_and_aliases(typ) # We'll ignore NewType when getting the origin + args for generics. - origin_cls = get_origin(resolve_newtype_and_aliases(typ)) + origin_cls = get_origin(typ) type_from_typevar: Dict[TypeVar, TypeForm[Any]] = {} # Support typing.Self. @@ -559,6 +562,14 @@ def resolve_generic_types( assert len(typevars) == len(typevar_values) typ = origin_cls type_from_typevar.update(dict(zip(typevars, typevar_values))) + elif ( + # Apply some heuristics for generic types. Should revisit this. + hasattr(typ, "__parameters__") and hasattr(typ.__parameters__, "__len__") # type: ignore + ): + typevars = typ.__parameters__ # type: ignore + typevar_values = tuple(type_from_typevar_constraints(x) for x in typevars) + assert len(typevars) == len(typevar_values) + type_from_typevar.update(dict(zip(typevars, typevar_values))) if hasattr(typ, "__orig_bases__"): bases = getattr(typ, "__orig_bases__") diff --git a/src/tyro/_strings.py b/src/tyro/_strings.py index a8427b05f..2882a1986 100644 --- a/src/tyro/_strings.py +++ b/src/tyro/_strings.py @@ -122,9 +122,12 @@ def get_name(cls: Type) -> str: return ( get_delimeter().join( - map( - lambda x: _subparser_name_from_type(x)[0], - [cls] + list(type_from_typevar.values()), + [get_name(cls)] + + list( + map( + lambda x: _subparser_name_from_type(x)[0], + list(type_from_typevar.values()), + ) ) ), prefix_name, diff --git a/src/tyro/_subcommand_matching.py b/src/tyro/_subcommand_matching.py index bc01dceb1..2e6f523bb 100644 --- a/src/tyro/_subcommand_matching.py +++ b/src/tyro/_subcommand_matching.py @@ -15,14 +15,11 @@ def match_subcommand( subcommand_type_from_name: Dict[str, type], ) -> Optional[str]: """Given a subcommand mapping and a default, return which subcommand the default - corresponds to.""" + corresponds to. - # It's really hard to concretize a generic type at runtime, so we just... - # don't. :-) - if hasattr(type(default), "__parameters__"): - raise _instantiators.UnsupportedTypeAnnotationError( - "Default values for generic subparsers are not supported." - ) + TOOD: this function is based on heuristics. While it should be robust to + most real-world scenarios, there's room for improvement for generic types. + """ # Get default subcommand name: by default hash. default_hash = object.__hash__(default) @@ -49,10 +46,17 @@ def match_subcommand( return subcommand_name # Get default subcommand name: by annotated type tree. - for subcommand_name, subcommand_type in subcommand_type_from_name.items(): - if typetree.is_subtype_of( - _TypeTree.make(subcommand_type, _fields.MISSING_NONPROP) - ): + typetree_from_name = { + subcommand_name: _TypeTree.make(subcommand_type, _fields.MISSING_NONPROP) + for subcommand_name, subcommand_type in subcommand_type_from_name.items() + } + for subcommand_name in subcommand_type_from_name.keys(): + # Equality check for type tree. + if typetree == typetree_from_name[subcommand_name]: + return subcommand_name + for subcommand_name in subcommand_type_from_name.keys(): + # Leaky subtype check. + if typetree.is_subtype_of(typetree_from_name[subcommand_name]): return subcommand_name # Failed. This should never happen, we'll raise an error outside of this function if @@ -71,15 +75,18 @@ def make( default_instance: _fields.DefaultInstance, ) -> _TypeTree: """From an object instance, return a data structure representing the types in the object.""" + + typ_unwrap = _resolver.resolve_generic_types(typ)[0] + try: typ, field_list = _fields.field_list_from_callable( typ, default_instance=default_instance, support_single_arg_types=False ) except _instantiators.UnsupportedTypeAnnotationError: - return _TypeTree(typ, {}) + return _TypeTree(typ_unwrap, {}) return _TypeTree( - typ, + typ_unwrap, { field.intern_name: _TypeTree.make(field.type_or_callable, field.default) for field in field_list @@ -87,6 +94,8 @@ def make( ) def is_subtype_of(self, supertype: _TypeTree) -> bool: + """Best-effort subcommand matching.""" + # Generalize to unions. def _get_type_options(typ: _typing.TypeForm) -> Tuple[_typing.TypeForm, ...]: return get_args(typ) if get_origin(typ) is Union else (typ,) @@ -94,15 +103,25 @@ def _get_type_options(typ: _typing.TypeForm) -> Tuple[_typing.TypeForm, ...]: self_types = _get_type_options(self.typ) super_types = _get_type_options(supertype.typ) - # Check against supertypes. + # Check against supertypes. TODO: the heuristics below could be more + # principled. We should revisit. for self_type in self_types: self_type = _resolver.unwrap_annotated(self_type) ok = False for super_type in super_types: super_type = _resolver.unwrap_annotated(super_type) - if issubclass(self_type, super_type): - ok = True + try: + if issubclass(self_type, super_type): + ok = True + except TypeError: + pass if not ok: return False + for child_name, child in self.children.items(): + if child_name not in supertype.children or not child.is_subtype_of( + supertype.children[child_name] + ): + return False + return True diff --git a/tests/test_nested.py b/tests/test_nested.py index 0eec9d258..f9a57c094 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -980,11 +980,11 @@ def main(x: Union[A[int], A[float]]) -> Any: assert tyro.cli(main, args="x:a-float --x.x 3.2".split(" ")) == A(3.2) assert tyro.cli(main, args="x:a-int --x.x 3".split(" ")) == A(3) - def main_with_default(x: Union[A[int], A[float]] = A(5)) -> Any: + def main_with_default(x: Union[A[str], A[int]] = A(5)) -> Any: return x - with pytest.raises(tyro.UnsupportedTypeAnnotationError): - tyro.cli(main_with_default, args=[]) + assert tyro.cli(main_with_default, args=[]) == A(5) + assert tyro.cli(main_with_default, args=["x:a-str", "--x.x", "3"]) == A("3") def test_generic_inherited() -> None: diff --git a/tests/test_new_style_annotations_min_py312.py b/tests/test_new_style_annotations_min_py312.py index 5e6a7ef0a..7efee58b8 100644 --- a/tests/test_new_style_annotations_min_py312.py +++ b/tests/test_new_style_annotations_min_py312.py @@ -2,7 +2,7 @@ # # PEP 695 isn't yet supported in mypy. (April 4, 2024) from dataclasses import dataclass -from typing import Annotated, Any, NewType +from typing import Annotated, Any, Literal, NewType import pytest @@ -201,3 +201,94 @@ def main(arg: list[Int7], /) -> Any: return arg assert tyro.cli(main, args=["1", "2"]) == [1, 2] + + +def test_generic_config(): + @dataclass(frozen=True) + class Container[T]: + a: Inner[T] + + assert tyro.cli( + Container[bool], + args="--a.a True --a.b False".split(" "), + config=(tyro.conf.FlagConversionOff,), + ) == Container(Inner(True, False)) + + +def test_generic_config_subcommand(): + @dataclass(frozen=True) + class Container[T]: + a: T + + assert tyro.cli( + Container[Container[bool] | Container[str]], + args="a:container-bool --a.a True".split(" "), + default=Container(Container(a="30")), + config=(tyro.conf.FlagConversionOff,), + ) == Container(Container(True)) + + assert tyro.cli( + Container[Container[bool] | Container[str]], + args=[], + default=Container(Container(a="30")), + config=(tyro.conf.FlagConversionOff,), + ) == Container(Container("30")) + + assert tyro.cli( + Container[Container[bool] | Container[str]], + args=[], + default=Container(Container(a=False)), + config=(tyro.conf.FlagConversionOff,), + ) == Container(Container(False)) + + +def test_generic_config_subcommand2(): + @dataclass(frozen=True) + class Container[T]: + a: tyro.conf.OmitSubcommandPrefixes[T] + + assert tyro.cli( + Container[Container[bool] | Container[str]], + args="container-bool --a True".split(" "), + ) == Container(Container(True)) + + +def test_generic_config_subcommand3(): + @dataclass(frozen=True) + class Container[T]: + a: T + + assert tyro.cli( + Container[Container[bool] | Container[str]], + args=[], + default=Container(Container(a=True)), + config=(tyro.conf.OmitSubcommandPrefixes,), + ) == Container(Container(True)) + + +def test_generic_config_subcommand4(): + @dataclass(frozen=True) + class Container[T]: + a: T + + assert tyro.cli( + Container[Container[bool] | Container[Literal["1", "2"]]], + args="container-literal-1-2 --a 2".split(" "), + config=(tyro.conf.OmitSubcommandPrefixes,), + ) == Container(Container("2")) + + assert tyro.cli( + Container[Container[bool] | Container[Literal["1", "2"]]], + args=[], + default=Container(Container(a=True)), + config=(tyro.conf.OmitSubcommandPrefixes,), + ) == Container(Container(True)) + + # This case is currently too hard for tyro's subcommand matcher. + with pytest.raises(AssertionError): + tyro.cli( + Container[Container[bool] | Container[Literal["1", "2"]]], + args=[], + default=Container(Container(a="1")), + config=(tyro.conf.OmitSubcommandPrefixes,), + ) diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index d48b096df..4ae8a4e78 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -990,11 +990,11 @@ def main(x: A[int] | A[float]) -> Any: assert tyro.cli(main, args="x:a-float --x.x 3.2".split(" ")) == A(3.2) assert tyro.cli(main, args="x:a-int --x.x 3".split(" ")) == A(3) - def main_with_default(x: A[int] | A[float] = A(5)) -> Any: + def main_with_default(x: A[str] | A[int] = A(5)) -> Any: return x - with pytest.raises(tyro.UnsupportedTypeAnnotationError): - tyro.cli(main_with_default, args=[]) + assert tyro.cli(main_with_default, args=[]) == A(5) + assert tyro.cli(main_with_default, args=["x:a-str", "--x.x", "3"]) == A("3") def test_generic_inherited() -> None: diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py index 5e6a7ef0a..7efee58b8 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py @@ -2,7 +2,7 @@ # # PEP 695 isn't yet supported in mypy. (April 4, 2024) from dataclasses import dataclass -from typing import Annotated, Any, NewType +from typing import Annotated, Any, Literal, NewType import pytest @@ -201,3 +201,94 @@ def main(arg: list[Int7], /) -> Any: return arg assert tyro.cli(main, args=["1", "2"]) == [1, 2] + + +def test_generic_config(): + @dataclass(frozen=True) + class Container[T]: + a: Inner[T] + + assert tyro.cli( + Container[bool], + args="--a.a True --a.b False".split(" "), + config=(tyro.conf.FlagConversionOff,), + ) == Container(Inner(True, False)) + + +def test_generic_config_subcommand(): + @dataclass(frozen=True) + class Container[T]: + a: T + + assert tyro.cli( + Container[Container[bool] | Container[str]], + args="a:container-bool --a.a True".split(" "), + default=Container(Container(a="30")), + config=(tyro.conf.FlagConversionOff,), + ) == Container(Container(True)) + + assert tyro.cli( + Container[Container[bool] | Container[str]], + args=[], + default=Container(Container(a="30")), + config=(tyro.conf.FlagConversionOff,), + ) == Container(Container("30")) + + assert tyro.cli( + Container[Container[bool] | Container[str]], + args=[], + default=Container(Container(a=False)), + config=(tyro.conf.FlagConversionOff,), + ) == Container(Container(False)) + + +def test_generic_config_subcommand2(): + @dataclass(frozen=True) + class Container[T]: + a: tyro.conf.OmitSubcommandPrefixes[T] + + assert tyro.cli( + Container[Container[bool] | Container[str]], + args="container-bool --a True".split(" "), + ) == Container(Container(True)) + + +def test_generic_config_subcommand3(): + @dataclass(frozen=True) + class Container[T]: + a: T + + assert tyro.cli( + Container[Container[bool] | Container[str]], + args=[], + default=Container(Container(a=True)), + config=(tyro.conf.OmitSubcommandPrefixes,), + ) == Container(Container(True)) + + +def test_generic_config_subcommand4(): + @dataclass(frozen=True) + class Container[T]: + a: T + + assert tyro.cli( + Container[Container[bool] | Container[Literal["1", "2"]]], + args="container-literal-1-2 --a 2".split(" "), + config=(tyro.conf.OmitSubcommandPrefixes,), + ) == Container(Container("2")) + + assert tyro.cli( + Container[Container[bool] | Container[Literal["1", "2"]]], + args=[], + default=Container(Container(a=True)), + config=(tyro.conf.OmitSubcommandPrefixes,), + ) == Container(Container(True)) + + # This case is currently too hard for tyro's subcommand matcher. + with pytest.raises(AssertionError): + tyro.cli( + Container[Container[bool] | Container[Literal["1", "2"]]], + args=[], + default=Container(Container(a="1")), + config=(tyro.conf.OmitSubcommandPrefixes,), + ) From b358dfae43e7eb711fbacf6139fe2356295bf1b5 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Wed, 23 Oct 2024 02:58:53 -0700 Subject: [PATCH 466/491] Minor helptext generation improvements (#184) * Adjust helptext behavior * Tests --- src/tyro/_parsers.py | 36 ++++--- tests/test_conf.py | 4 +- tests/test_nested.py | 6 ++ tests/test_py311_generated/ok.py | 12 +++ .../test_conf_generated.py | 4 +- .../test_nested_generated.py | 6 ++ ...st_pydantic_helptext_advanced_generated.py | 98 ++++++++++++++++++ tests/test_pydantic_helptext_advanced.py | 99 +++++++++++++++++++ 8 files changed, 249 insertions(+), 16 deletions(-) create mode 100644 tests/test_py311_generated/ok.py create mode 100644 tests/test_py311_generated/test_pydantic_helptext_advanced_generated.py create mode 100644 tests/test_pydantic_helptext_advanced.py diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index ef6bf0136..228021132 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -145,7 +145,9 @@ def from_callable_or_type( ) # Helptext for this field; used as description for grouping arguments. - class_field_name = _strings.make_field_name([field.intern_name]) + class_field_name = _strings.make_field_name( + [intern_prefix, field.intern_name] + ) if field.helptext is not None: helptext_from_intern_prefixed_field_name[class_field_name] = ( field.helptext @@ -235,10 +237,20 @@ def apply_args( """Create defined arguments and subparsers.""" # Make argument groups. - def format_group_name(prefix: str) -> str: - return (prefix + " options").strip() + def format_group_name(group_name: str) -> str: + return (group_name + " options").strip() + + def group_name_from_arg(arg: _arguments.ArgumentDefinition) -> str: + prefix = arg.lowered.name_or_flag + if prefix.startswith("--"): + prefix = prefix[2:] + if "." in prefix: + prefix = prefix.rpartition(".")[0] + else: + prefix = "" + return prefix - group_from_prefix: Dict[str, argparse._ArgumentGroup] = { + group_from_group_name: Dict[str, argparse._ArgumentGroup] = { "": parser._action_groups[1], **{ cast(str, group.title).partition(" ")[0]: group @@ -251,9 +263,10 @@ def format_group_name(prefix: str) -> str: # Add each argument group. Note that groups with only suppressed arguments won't # be added. for arg in self.args: + group_name = group_name_from_arg(arg) if ( arg.lowered.help is not argparse.SUPPRESS - and arg.extern_prefix not in group_from_prefix + and group_name not in group_from_group_name ): description = ( parent.helptext_from_intern_prefixed_field_name.get( @@ -262,24 +275,23 @@ def format_group_name(prefix: str) -> str: if parent is not None else None ) - group_from_prefix[arg.extern_prefix] = parser.add_argument_group( - format_group_name(arg.extern_prefix), + group_from_group_name[group_name] = parser.add_argument_group( + format_group_name(group_name), description=description, ) - # Add each argument. - for arg in self.args: + # Add each argument. if arg.field.is_positional(): arg.add_argument(positional_group) continue - if arg.extern_prefix in group_from_prefix: - arg.add_argument(group_from_prefix[arg.extern_prefix]) + if group_name in group_from_group_name: + arg.add_argument(group_from_group_name[group_name]) else: # Suppressed argument: still need to add them, but they won't show up in # the helptext so it doesn't matter which group. assert arg.lowered.help is argparse.SUPPRESS - arg.add_argument(group_from_prefix[""]) + arg.add_argument(group_from_group_name[""]) for child in self.child_from_prefix.values(): child.apply_args(parser, parent=self) diff --git a/tests/test_conf.py b/tests/test_conf.py index 1750e8db1..54dd9aa35 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -1497,8 +1497,8 @@ class DatasetConfig: with pytest.raises(SystemExit), contextlib.redirect_stdout(target): instantiate_dataclasses((OptimizerConfig, DatasetConfig), args=["--help"]) helptext = target.getvalue() - assert "OptimizerConfig options" in helptext - assert "DatasetConfig options" in helptext + assert "OptimizerConfig options" not in helptext + assert "DatasetConfig options" not in helptext def test_counter_action() -> None: diff --git a/tests/test_nested.py b/tests/test_nested.py index f9a57c094..98ec5cb6e 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -18,11 +18,17 @@ class B: class Nested: x: int b: B + """Helptext for b""" assert tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) with pytest.raises(SystemExit): tyro.cli(Nested, args=["--x", "1"]) + def main(x: Nested): + return x + + assert "Helptext for b" in get_helptext_with_checks(main) + def test_nested_annotated() -> None: @dataclasses.dataclass diff --git a/tests/test_py311_generated/ok.py b/tests/test_py311_generated/ok.py new file mode 100644 index 000000000..73e5b33bc --- /dev/null +++ b/tests/test_py311_generated/ok.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass +from typing import Literal + +import tyro + + +@dataclass(frozen=True) +class Container[T]: + a: T + + +tyro.cli(Container[Container[bool] | Container[Literal["1", "2"]]]) diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index 04301ed76..13997ecbd 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -1492,8 +1492,8 @@ class DatasetConfig: with pytest.raises(SystemExit), contextlib.redirect_stdout(target): instantiate_dataclasses((OptimizerConfig, DatasetConfig), args=["--help"]) helptext = target.getvalue() - assert "OptimizerConfig options" in helptext - assert "DatasetConfig options" in helptext + assert "OptimizerConfig options" not in helptext + assert "DatasetConfig options" not in helptext def test_counter_action() -> None: diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index 4ae8a4e78..5a611b42c 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -28,11 +28,17 @@ class B: class Nested: x: int b: B + """Helptext for b""" assert tyro.cli(Nested, args=["--x", "1", "--b.y", "3"]) == Nested(x=1, b=B(y=3)) with pytest.raises(SystemExit): tyro.cli(Nested, args=["--x", "1"]) + def main(x: Nested): + return x + + assert "Helptext for b" in get_helptext_with_checks(main) + def test_nested_annotated() -> None: @dataclasses.dataclass diff --git a/tests/test_py311_generated/test_pydantic_helptext_advanced_generated.py b/tests/test_py311_generated/test_pydantic_helptext_advanced_generated.py new file mode 100644 index 000000000..a7da8accc --- /dev/null +++ b/tests/test_py311_generated/test_pydantic_helptext_advanced_generated.py @@ -0,0 +1,98 @@ +"""Adapted from: https://github.com/brentyi/tyro/issues/183""" + +from typing import Annotated, NamedTuple, Set + +from helptext_utils import get_helptext_with_checks +from pydantic import BaseModel, Field + +import tyro + + +class MyRange(NamedTuple): + low: int + high: int + + def __str__(self): + return f"<{self.low}, {self.high}>" + + @staticmethod + def tyro_constructor( + range_str: Annotated[ + str, + tyro.conf.arg(name=""), + ], + ): + import re + + m = re.match("([0-9]+)(-([0-9]+))*", range_str) + low = m[1] # type: ignore + high = low if not m[3] else m[3] # type: ignore + + return MyRange(int(low), int(high)) + + @staticmethod + def tyro_constructor_set( + range_str_set: Annotated[ + Set[str], + tyro.conf.arg(name=""), + ], + ): + return {MyRange.tyro_constructor(r) for r in range_str_set} + + +class MySpec(BaseModel): + some_set: Set[int] = Field( + default={1, 2, 3}, + description="Some set of integers", + title="Some set", + ) + + some_string: str = Field( + description="Some string without a default value.", title="SomeSTR" + ) + + here_comes_the_trouble: Annotated[ + Set[MyRange], + tyro.conf.arg(constructor=MyRange.tyro_constructor_set), + ] = Field( + default={MyRange(0, 1024)}, + description="I would like this one in the same group as others", + title="Please help", + ) + + +def add_spec(spec: MySpec) -> MySpec: + return spec + + +def test_functionality() -> None: + assert tyro.cli( + add_spec, args=["--spec.some-set", "1", "2", "3", "--spec.some-string", "hello"] + ) == MySpec( + some_set={1, 2, 3}, + some_string="hello", + here_comes_the_trouble={MyRange(0, 1024)}, + ) + assert tyro.cli( + add_spec, + args=[ + "--spec.some-set", + "1", + "2", + "3", + "--spec.some-string", + "hello", + "--spec.here-comes-the-trouble", + "0-512", + ], + ) == MySpec( + some_set={1, 2, 3}, + some_string="hello", + here_comes_the_trouble={MyRange(0, 512)}, + ) + + +def test_helptext() -> None: + helptext = get_helptext_with_checks(add_spec) + assert "spec options" in helptext + assert "spec.here-comes-the-trouble-options" not in helptext diff --git a/tests/test_pydantic_helptext_advanced.py b/tests/test_pydantic_helptext_advanced.py new file mode 100644 index 000000000..245a2150a --- /dev/null +++ b/tests/test_pydantic_helptext_advanced.py @@ -0,0 +1,99 @@ +"""Adapted from: https://github.com/brentyi/tyro/issues/183""" + +from typing import NamedTuple, Set + +from helptext_utils import get_helptext_with_checks +from pydantic import BaseModel, Field +from typing_extensions import Annotated + +import tyro + + +class MyRange(NamedTuple): + low: int + high: int + + def __str__(self): + return f"<{self.low}, {self.high}>" + + @staticmethod + def tyro_constructor( + range_str: Annotated[ + str, + tyro.conf.arg(name=""), + ], + ): + import re + + m = re.match("([0-9]+)(-([0-9]+))*", range_str) + low = m[1] # type: ignore + high = low if not m[3] else m[3] # type: ignore + + return MyRange(int(low), int(high)) + + @staticmethod + def tyro_constructor_set( + range_str_set: Annotated[ + Set[str], + tyro.conf.arg(name=""), + ], + ): + return {MyRange.tyro_constructor(r) for r in range_str_set} + + +class MySpec(BaseModel): + some_set: Set[int] = Field( + default={1, 2, 3}, + description="Some set of integers", + title="Some set", + ) + + some_string: str = Field( + description="Some string without a default value.", title="SomeSTR" + ) + + here_comes_the_trouble: Annotated[ + Set[MyRange], + tyro.conf.arg(constructor=MyRange.tyro_constructor_set), + ] = Field( + default={MyRange(0, 1024)}, + description="I would like this one in the same group as others", + title="Please help", + ) + + +def add_spec(spec: MySpec) -> MySpec: + return spec + + +def test_functionality() -> None: + assert tyro.cli( + add_spec, args=["--spec.some-set", "1", "2", "3", "--spec.some-string", "hello"] + ) == MySpec( + some_set={1, 2, 3}, + some_string="hello", + here_comes_the_trouble={MyRange(0, 1024)}, + ) + assert tyro.cli( + add_spec, + args=[ + "--spec.some-set", + "1", + "2", + "3", + "--spec.some-string", + "hello", + "--spec.here-comes-the-trouble", + "0-512", + ], + ) == MySpec( + some_set={1, 2, 3}, + some_string="hello", + here_comes_the_trouble={MyRange(0, 512)}, + ) + + +def test_helptext() -> None: + helptext = get_helptext_with_checks(add_spec) + assert "spec options" in helptext + assert "spec.here-comes-the-trouble-options" not in helptext From ea22883b166000bbd246bce7b19af3a1caea8d0f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 24 Oct 2024 13:58:04 -0700 Subject: [PATCH 467/491] `typing.Annotated` workaround for pydantic v1 (#187) * Workaround for pydantic v1, which strips `typing.Annotated` metadata * docstring_parser doesn't seem to work with Pydantic v1 + Python 3.8 * sys version check --- src/tyro/_fields.py | 6 +++- .../test_pydantic_generated.py | 27 +++++++++++++++ tests/test_pydantic.py | 33 +++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 275595dd1..ed0b82767 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -674,6 +674,10 @@ def _field_list_from_pydantic( cls_cast = cast(pydantic_v1.BaseModel, cls) # type: ignore else: cls_cast = cls + + hints = _resolver.get_type_hints_with_backported_syntax( + cls, include_extras=True + ) for pd1_field in cls_cast.__fields__.values(): helptext = pd1_field.field_info.description if helptext is None: @@ -686,7 +690,7 @@ def _field_list_from_pydantic( field_list.append( FieldDefinition.make( name=pd1_field.name, - type_or_callable=pd1_field.outer_type_, + type_or_callable=hints[pd1_field.name], default=default, is_default_from_default_instance=is_default_from_default_instance, helptext=helptext, diff --git a/tests/test_py311_generated/test_pydantic_generated.py b/tests/test_py311_generated/test_pydantic_generated.py index c54ef87df..c0892be15 100644 --- a/tests/test_py311_generated/test_pydantic_generated.py +++ b/tests/test_py311_generated/test_pydantic_generated.py @@ -6,6 +6,7 @@ from typing import Annotated, cast import pytest +from helptext_utils import get_helptext_with_checks from pydantic import BaseModel, Field, v1 import tyro @@ -51,6 +52,32 @@ class ManyTypesA(v1.BaseModel): ) == ManyTypesA(i=5, s="hello", f=3.0, p=pathlib.Path("~")) +def test_pydantic_v1_conf() -> None: + class ManyTypesA(v1.BaseModel): + i: int + """This is a docstring.""" + s: tyro.conf.Suppress[str] = "hello" + f: float = v1.Field(default_factory=lambda: 3.0) + + class ManyTypesB(ManyTypesA): + p: pathlib.Path + + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( + ManyTypesB, + args=[ + "--i", + "5", + "--p", + "~", + ], + ) == ManyTypesB(i=5, s="hello", f=3.0, p=pathlib.Path("~")) + helptext = get_helptext_with_checks(ManyTypesB) + assert "--s" not in helptext + assert "--i" in helptext + assert "This is a docstring" in helptext + + def test_pydantic_helptext() -> None: class Helptext(BaseModel): """This docstring should be printed as a description.""" diff --git a/tests/test_pydantic.py b/tests/test_pydantic.py index 5a588e71c..a64ca28cf 100644 --- a/tests/test_pydantic.py +++ b/tests/test_pydantic.py @@ -3,9 +3,11 @@ import contextlib import io import pathlib +import sys from typing import cast import pytest +from helptext_utils import get_helptext_with_checks from pydantic import BaseModel, Field, v1 from typing_extensions import Annotated @@ -52,6 +54,37 @@ class ManyTypesA(v1.BaseModel): ) == ManyTypesA(i=5, s="hello", f=3.0, p=pathlib.Path("~")) +def test_pydantic_v1_conf() -> None: + class ManyTypesA(v1.BaseModel): + i: int + """This is a docstring.""" + s: tyro.conf.Suppress[str] = "hello" + f: float = v1.Field(default_factory=lambda: 3.0) + + class ManyTypesB(ManyTypesA): + p: pathlib.Path + + # We can directly pass a dataclass to `tyro.cli()`: + assert tyro.cli( + ManyTypesB, + args=[ + "--i", + "5", + "--p", + "~", + ], + ) == ManyTypesB(i=5, s="hello", f=3.0, p=pathlib.Path("~")) + helptext = get_helptext_with_checks(ManyTypesB) + assert "--s" not in helptext + assert "--i" in helptext + + # This doesn't work when combining older versions of Python with older + # versions of pydantic. The root cause may be in the docstring_parser + # dependency. + if sys.version_info >= (3, 10): + assert "This is a docstring" in helptext + + def test_pydantic_helptext() -> None: class Helptext(BaseModel): """This docstring should be printed as a description.""" From dd11c7700a0417b1b16e68ae0a649c5e5f5c97e0 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 29 Oct 2024 23:10:46 -0700 Subject: [PATCH 468/491] Primitive handling rewrite (#189) * PrimitiveSpec API prototype * Revert example change * Delete unused * ruff * mypy * housekeeping * tests * Append cleanup, public API * mypy * typing.Annotated support, tyro.cli() registry arg * Fixes * mypy * Less busy main API * Gen * Remove unused error --- src/tyro/__init__.py | 3 +- src/tyro/_argparse_formatter.py | 16 +- src/tyro/_arguments.py | 195 ++--- src/tyro/_calling.py | 4 +- src/tyro/_cli.py | 122 ++- src/tyro/_fields.py | 197 ++--- src/tyro/_instantiators.py | 749 ------------------ src/tyro/_parsers.py | 57 +- src/tyro/_resolver.py | 4 +- src/tyro/_strings.py | 47 +- src/tyro/_subcommand_matching.py | 7 +- src/tyro/conf/_confstruct.py | 24 +- src/tyro/constructors/__init__.py | 5 + src/tyro/constructors/_primitive_spec.py | 705 +++++++++++++++++ src/tyro/extras/__init__.py | 3 +- tests/helptext_utils.py | 4 + tests/test_conf.py | 23 + tests/test_custom_primitive.py | 44 + tests/test_errors.py | 18 + tests/test_helptext.py | 17 +- tests/test_is_nested_type.py | 52 -- tests/test_is_struct_type.py | 52 ++ tests/test_new_style_annotations_min_py310.py | 16 +- tests/test_new_style_annotations_unions.py | 16 +- .../test_conf_generated.py | 23 + .../test_custom_primitive_generated.py | 42 + .../test_errors_generated.py | 18 + .../test_helptext_generated.py | 17 +- .../test_is_nested_type_generated.py | 46 +- .../test_is_struct_type_generated.py | 52 ++ ...w_style_annotations_min_py310_generated.py | 16 +- ..._new_style_annotations_unions_generated.py | 16 +- .../test_pydantic_generated.py | 8 +- 33 files changed, 1444 insertions(+), 1174 deletions(-) delete mode 100644 src/tyro/_instantiators.py create mode 100644 src/tyro/constructors/__init__.py create mode 100644 src/tyro/constructors/_primitive_spec.py create mode 100644 tests/test_custom_primitive.py delete mode 100644 tests/test_is_nested_type.py create mode 100644 tests/test_is_struct_type.py create mode 100644 tests/test_py311_generated/test_custom_primitive_generated.py create mode 100644 tests/test_py311_generated/test_is_struct_type_generated.py diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index 14155a7c4..49f5e40f2 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -1,10 +1,11 @@ from typing import TYPE_CHECKING from . import conf as conf +from . import constructors as constructors from . import extras as extras from ._cli import cli as cli from ._fields import MISSING as MISSING -from ._instantiators import ( +from .constructors._primitive_spec import ( UnsupportedTypeAnnotationError as UnsupportedTypeAnnotationError, ) diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index 69753e4f0..1f632eb88 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -852,9 +852,11 @@ def get_score(option_string: str) -> float: console.print( Panel( Group( - f"{message[0].upper() + message[1:]}" - if len(message) > 0 - else "", + ( + f"{message[0].upper() + message[1:]}" + if len(message) > 0 + else "" + ), *extra_info, Rule(style=Style(color="red")), f"For full helptext, run [bold]{self.prog} --help[/bold]", @@ -1343,9 +1345,11 @@ def _format_usage( new_actions.append( argparse.Action( [ - "OPTIONS" - if len(prog_parts) == 1 - else prog_parts[-1].upper() + " OPTIONS" + ( + "OPTIONS" + if len(prog_parts) == 1 + else prog_parts[-1].upper() + " OPTIONS" + ) ], dest="", ) diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index 1dc29e986..44e36dcdb 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -4,8 +4,6 @@ from __future__ import annotations import dataclasses -import enum -import itertools import json import shlex from typing import ( @@ -13,7 +11,6 @@ Any, Callable, Iterable, - Mapping, Optional, Sequence, Tuple, @@ -24,10 +21,16 @@ import rich.markup import shtab +from typing_extensions import get_origin from . import _argparse as argparse -from . import _fields, _instantiators, _strings +from . import _fields, _strings from .conf import _markers +from .constructors._primitive_spec import ( + PrimitiveConstructorRegistry, + PrimitiveTypeInfo, + UnsupportedTypeAnnotationError, +) if TYPE_CHECKING: cached_property = property @@ -114,7 +117,7 @@ def add_argument( # Get keyword arguments, with None values removed. kwargs = dict(self.lowered.__dict__) # type: ignore - kwargs.pop("instantiator") + kwargs.pop("instance_from_str") kwargs = {k: v for k, v in kwargs.items() if v is not None} name_or_flag = kwargs.pop("name_or_flag") if len(name_or_flag) == 0: @@ -184,10 +187,8 @@ def lowered(self) -> LoweredArgumentDefinition: # Each rule will mutate the lowered object. This is (unfortunately) # much faster than a functional approach. lowered = LoweredArgumentDefinition() - _rule_handle_defaults(self, lowered) _rule_handle_boolean_flags(self, lowered) - _rule_recursive_instantiator_from_type(self, lowered) - _rule_convert_defaults_to_strings(self, lowered) + _rule_apply_primitive_specs(self, lowered) _rule_counters(self, lowered) _rule_generate_helptext(self, lowered) _rule_set_name_or_flag_and_dest(self, lowered) @@ -204,14 +205,14 @@ class LoweredArgumentDefinition: # # The main reason we use this instead of the standard 'type' argument is to enable # mixed-type tuples. - instantiator: Optional[_instantiators.Instantiator] = None + instance_from_str: Optional[Callable] = None def is_fixed(self) -> bool: """If the instantiator is set to `None`, even after all argument transformations, it means that we don't have a valid instantiator for an argument. We then mark the argument as 'fixed', with a value always equal to the field default.""" - return self.instantiator is None + return self.instance_from_str is None # From here on out, all fields correspond 1:1 to inputs to argparse's # add_argument() method. @@ -228,25 +229,6 @@ def is_fixed(self) -> bool: help: Optional[str] = None -def _rule_handle_defaults( - arg: ArgumentDefinition, - lowered: LoweredArgumentDefinition, -) -> None: - """Set `required=False` if a default value is set.""" - - # Mark lowered as required if a default is set. - if ( - arg.field.default in _fields.MISSING_SINGLETONS - and _markers._OPTIONAL_GROUP not in arg.field.markers - ): - lowered.default = None - lowered.required = True - return - - lowered.default = arg.field.default - return - - def _rule_handle_boolean_flags( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, @@ -265,16 +247,18 @@ def _rule_handle_boolean_flags( elif arg.field.default in (True, False): # Default `False` => --flag passed in flips to `True`. lowered.action = BooleanOptionalAction - lowered.instantiator = lambda x: x # argparse will directly give us a bool! + lowered.instance_from_str = ( + lambda x: x + ) # argparse will directly give us a bool! return assert False, ( f"Expected a boolean as a default for {arg.field.intern_name}, but got" - f" {lowered.default}." + f" {arg.field.default}." ) -def _rule_recursive_instantiator_from_type( +def _rule_apply_primitive_specs( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> None: @@ -286,26 +270,35 @@ def _rule_recursive_instantiator_from_type( Conversions from strings to our desired types happen in the instantiator; this is a bit more flexible, and lets us handle more complex types like enums and multi-type tuples.""" + if _markers.Fixed in arg.field.markers: - lowered.instantiator = None + lowered.instance_from_str = None lowered.metavar = "{fixed}" lowered.required = False lowered.default = _fields.MISSING_PROP return - if lowered.instantiator is not None: + if lowered.instance_from_str is not None: return + try: - instantiator, metadata = _instantiators.instantiator_from_type( - arg.field.type_or_callable, - arg.field.markers, - ) - except _instantiators.UnsupportedTypeAnnotationError as e: + if arg.field.primitive_spec is not None: + spec = arg.field.primitive_spec + else: + registry = PrimitiveConstructorRegistry._get_active_registry() + spec = registry.get_spec( + PrimitiveTypeInfo.make( + cast(type, arg.field.type_or_callable), + arg.field.markers, + source_registry=registry, + ) + ) + except UnsupportedTypeAnnotationError as e: if arg.field.default in _fields.MISSING_SINGLETONS: field_name = _strings.make_field_name( [arg.extern_prefix, arg.field.intern_name] ) if field_name != "": - raise _instantiators.UnsupportedTypeAnnotationError( + raise UnsupportedTypeAnnotationError( f"Unsupported type annotation for the field {field_name}; " f"{e.args[0]} " "To suppress this error, assign the field either a default value or a different type." @@ -323,59 +316,76 @@ def _rule_recursive_instantiator_from_type( lowered.default = _fields.MISSING_PROP return - if metadata.action == "append": - - def append_instantiator(x: Any) -> Any: - out = instantiator(x) - if arg.field.default in _fields.MISSING_SINGLETONS: - return instantiator(x) - - return type(out)(arg.field.default) + out - - lowered.instantiator = append_instantiator + # Mark lowered as required if a default is missing. + if ( + arg.field.default in _fields.MISSING_SINGLETONS + and _markers._OPTIONAL_GROUP not in arg.field.markers + ): lowered.default = None - lowered.choices = metadata.choices - lowered.nargs = metadata.nargs - lowered.metavar = metadata.metavar - lowered.action = metadata.action - lowered.required = False - return + lowered.required = True + elif ( + arg.field.default is not _fields.EXCLUDE_FROM_CALL + and arg.field.default not in _fields.MISSING_SINGLETONS + ): + # Set default. + lowered.default = spec.str_from_instance(arg.field.default) else: - lowered.instantiator = instantiator - lowered.choices = metadata.choices - lowered.nargs = metadata.nargs - lowered.metavar = metadata.metavar - lowered.action = metadata.action - return - + lowered.default = arg.field.default + + if spec._action == "append": + + def append_instantiator(x: list[list[str]]) -> Any: + """Handle UseAppendAction effects.""" + # We'll assume that the type is annotated as Dict[...], Tuple[...], List[...], etc. + container_type = get_origin(arg.field.type_or_callable) + if container_type is None: + # Raw annotation, like `UseAppendAction[list]`. It's unlikely + # that a user would use this but we can handle it. + container_type = arg.field.type_or_callable + + # Instantiate initial output. + out = ( + arg.field.default + if arg.field.default not in _fields.MISSING_SINGLETONS + else None + ) + if out is None: + out = {} if container_type is dict else [] + elif isinstance(out, dict): + out = out.copy() + else: + # All sequence types will be lists for now to make sure we can + # append to them. + out = list(out) + + # Get + merge parts. + parts = [spec.instance_from_str(arg_list) for arg_list in x] + for part in parts: + if isinstance(out, dict): + out.update(part) + else: + out.append(part) -def _rule_convert_defaults_to_strings( - arg: ArgumentDefinition, - lowered: LoweredArgumentDefinition, -) -> None: - """Sets all default values to strings, as required as input to our instantiator - functions. Special-cased for enums.""" - - def as_str(x: Any) -> Tuple[str, ...]: - if isinstance(x, str): - return (x,) - elif isinstance(x, enum.Enum): - return (x.name,) - elif isinstance(x, Mapping): - return tuple(itertools.chain(*map(as_str, itertools.chain(*x.items())))) - elif isinstance(x, Sequence): - return tuple(itertools.chain(*map(as_str, x))) - else: - return (str(x),) + # Return output with correct type. + if isinstance(out, dict): + return out + else: + return container_type(out) - if ( - lowered.default is None - or lowered.default in _fields.MISSING_SINGLETONS - or lowered.action is not None - ): + lowered.instance_from_str = append_instantiator + lowered.default = None + lowered.choices = spec.choices + lowered.nargs = spec.nargs + lowered.metavar = spec.metavar + lowered.action = spec._action + lowered.required = False return else: - lowered.default = as_str(lowered.default) + lowered.instance_from_str = spec.instance_from_str + lowered.choices = spec.choices + lowered.nargs = spec.nargs + lowered.metavar = spec.metavar + lowered.action = spec._action return @@ -407,7 +417,9 @@ def _rule_counters( lowered.action = "count" lowered.default = 0 lowered.required = False - lowered.instantiator = lambda x: x # argparse will directly give us an int! + lowered.instance_from_str = ( + lambda x: x + ) # argparse will directly give us an int! return @@ -438,8 +450,7 @@ def _rule_generate_helptext( if not lowered.required: # Get the default value. - # Note: lowered.default is the stringified version! See - # `_rule_convert_defaults_to_strings()`. + # Note: lowered.default is the stringified version! default = lowered.default if lowered.is_fixed() or lowered.action == "append": # Cases where we'll be missing the lowered default. Use field default instead. @@ -473,7 +484,7 @@ def _rule_generate_helptext( if callable(help_behavior_hint) else help_behavior_hint ) - elif lowered.instantiator is None: + elif lowered.instance_from_str is None: # Intentionally not quoted via shlex, since this can't actually be passed # in via the commandline. behavior_hint = f"(fixed to: {default_label})" @@ -495,8 +506,6 @@ def _rule_generate_helptext( and default in _fields.MISSING_SINGLETONS ): # Argument in an optional group, but with no default. This is typically used - # Note: lowered.default is the stringified version! See - # `_rule_convert_defaults_to_strings()`. # when general (non-argument, non-dataclass) object arguments are given a # default, or when we use `tyro.conf.arg(constructor=...)`. # diff --git a/src/tyro/_calling.py b/src/tyro/_calling.py index ee4cdd78f..76c16d5a3 100644 --- a/src/tyro/_calling.py +++ b/src/tyro/_calling.py @@ -105,8 +105,8 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: value = [value] try: - assert arg.lowered.instantiator is not None - value = arg.lowered.instantiator(value) + assert arg.lowered.instance_from_str is not None + value = arg.lowered.instance_from_str(value) except (ValueError, TypeError) as e: raise InstantiationError( e.args[0], diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index eda141863..6703d65b1 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -1,21 +1,12 @@ """Core public API.""" +from __future__ import annotations + import dataclasses import pathlib import sys import warnings -from typing import ( - Callable, - Dict, - List, - Optional, - Sequence, - Tuple, - TypeVar, - Union, - cast, - overload, -) +from typing import Callable, Sequence, TypeVar, cast, overload import shtab from typing_extensions import Literal @@ -46,14 +37,14 @@ def cli( f: TypeForm[OutT], *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, + prog: None | str = None, + description: None | str = None, + args: None | Sequence[str] = None, + default: None | OutT = None, return_unknown_args: Literal[False] = False, use_underscores: bool = False, console_outputs: bool = True, - config: Optional[Sequence[conf._markers.Marker]] = None, + config: None | Sequence[conf._markers.Marker] = None, ) -> OutT: ... @@ -61,24 +52,24 @@ def cli( def cli( f: TypeForm[OutT], *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, + prog: None | str = None, + description: None | str = None, + args: None | Sequence[str] = None, + default: None | OutT = None, return_unknown_args: Literal[True], use_underscores: bool = False, console_outputs: bool = True, - config: Optional[Sequence[conf._markers.Marker]] = None, -) -> Tuple[OutT, List[str]]: ... + config: None | Sequence[conf._markers.Marker] = None, +) -> tuple[OutT, list[str]]: ... @overload def cli( f: Callable[..., OutT], *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, + prog: None | str = None, + description: None | str = None, + args: None | Sequence[str] = None, # Passing a default makes sense for things like dataclasses, but are not # supported for general callables. These can, however, be specified in the # signature of the callable itself. @@ -86,7 +77,7 @@ def cli( return_unknown_args: Literal[False] = False, use_underscores: bool = False, console_outputs: bool = True, - config: Optional[Sequence[conf._markers.Marker]] = None, + config: None | Sequence[conf._markers.Marker] = None, ) -> OutT: ... @@ -94,9 +85,9 @@ def cli( def cli( f: Callable[..., OutT], *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, + prog: None | str = None, + description: None | str = None, + args: None | Sequence[str] = None, # Passing a default makes sense for things like dataclasses, but are not # supported for general callables. These can, however, be specified in the # signature of the callable itself. @@ -104,23 +95,23 @@ def cli( return_unknown_args: Literal[True], use_underscores: bool = False, console_outputs: bool = True, - config: Optional[Sequence[conf._markers.Marker]] = None, -) -> Tuple[OutT, List[str]]: ... + config: None | Sequence[conf._markers.Marker] = None, +) -> tuple[OutT, list[str]]: ... def cli( - f: Union[TypeForm[OutT], Callable[..., OutT]], + f: TypeForm[OutT] | Callable[..., OutT], *, - prog: Optional[str] = None, - description: Optional[str] = None, - args: Optional[Sequence[str]] = None, - default: Optional[OutT] = None, + prog: None | str = None, + description: None | str = None, + args: None | Sequence[str] = None, + default: None | OutT = None, return_unknown_args: bool = False, use_underscores: bool = False, console_outputs: bool = True, - config: Optional[Sequence[conf._markers.Marker]] = None, + config: None | Sequence[conf._markers.Marker] = None, **deprecated_kwargs, -) -> Union[OutT, Tuple[OutT, List[str]]]: +) -> OutT | tuple[OutT, list[str]]: """Call or instantiate `f`, with inputs populated from an automatically generated CLI interface. @@ -233,9 +224,9 @@ def cli( def get_parser( f: TypeForm[OutT], *, - prog: Optional[str] = None, - description: Optional[str] = None, - default: Optional[OutT] = None, + prog: None | str = None, + description: None | str = None, + default: None | OutT = None, use_underscores: bool = False, console_outputs: bool = True, ) -> argparse.ArgumentParser: ... @@ -245,22 +236,22 @@ def get_parser( def get_parser( f: Callable[..., OutT], *, - prog: Optional[str] = None, - description: Optional[str] = None, - default: Optional[OutT] = None, + prog: None | str = None, + description: None | str = None, + default: None | OutT = None, use_underscores: bool = False, console_outputs: bool = True, ) -> argparse.ArgumentParser: ... def get_parser( - f: Union[TypeForm[OutT], Callable[..., OutT]], + f: TypeForm[OutT] | Callable[..., OutT], *, # We have no `args` argument, since this is only used when # parser.parse_args() is called. - prog: Optional[str] = None, - description: Optional[str] = None, - default: Optional[OutT] = None, + prog: None | str = None, + description: None | str = None, + default: None | OutT = None, use_underscores: bool = False, console_outputs: bool = True, ) -> argparse.ArgumentParser: @@ -287,21 +278,24 @@ def get_parser( def _cli_impl( - f: Union[TypeForm[OutT], Callable[..., OutT]], + f: TypeForm[OutT] | Callable[..., OutT], *, - prog: Optional[str] = None, - description: Optional[str], - args: Optional[Sequence[str]], - default: Optional[OutT], + prog: None | str = None, + description: None | str, + args: None | Sequence[str], + default: None | OutT, return_parser: bool, return_unknown_args: bool, console_outputs: bool, **deprecated_kwargs, -) -> Union[ - OutT, - argparse.ArgumentParser, - Tuple[Callable[[], OutT], List[str]], -]: +) -> ( + OutT + | argparse.ArgumentParser + | tuple[ + Callable[[], OutT], + list[str], + ] +): """Helper for stitching the `tyro` pipeline together.""" if "default_instance" in deprecated_kwargs: warnings.warn( @@ -322,14 +316,14 @@ def _cli_impl( # one or many arguments, depending on various factors). # # This could be revisited. - default_instance_internal: Union[_fields.NonpropagatingMissingType, OutT] = ( + default_instance_internal: _fields.NonpropagatingMissingType | OutT = ( _fields.MISSING_NONPROP if default is None else default ) # We wrap our type with a dummy dataclass if it can't be treated as a nested type. # For example: passing in f=int will result in a dataclass with a single field # typed as int. - if not _fields.is_nested_type(cast(type, f), default_instance_internal): + if not _fields.is_struct_type(cast(type, f), default_instance_internal): dummy_field = cast( dataclasses.Field, dataclasses.field(), @@ -351,7 +345,7 @@ def _cli_impl( # Fix arguments. This will modify all option-style arguments replacing # underscores with hyphens, or vice versa if use_underscores=True. # If two options are ambiguous, e.g., --a_b and --a-b, raise a runtime error. - modified_args: Dict[str, str] = {} + modified_args: dict[str, str] = {} for index, arg in enumerate(args): if not arg.startswith("--"): continue @@ -498,7 +492,7 @@ def _cli_impl( # `field_name_prefix == ""` condition in `callable_with_args()`! # Emulate argparse's error behavior when invalid arguments are passed in. - from rich.console import Console, Group, RenderableType + from rich.console import Console, Group from rich.padding import Padding from rich.panel import Panel from rich.rule import Rule @@ -514,7 +508,7 @@ def _cli_impl( "[bright_red][bold]Error parsing" f" {e.arg.lowered.name_or_flag if isinstance(e.arg, _arguments.ArgumentDefinition) else e.arg}[/bold]:[/bright_red] {e.message}", *cast( # Cast to appease mypy... - List[RenderableType], + list, ( [] if not isinstance(e.arg, _arguments.ArgumentDefinition) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index ed0b82767..e4f2789ed 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -43,17 +43,13 @@ is_typeddict, ) -from . import ( - _docstrings, - _instantiators, - _resolver, - _singleton, - _strings, - _unsafe_cache, - conf, # Avoid circular import. -) +from . import _docstrings, _resolver, _singleton, _strings, _unsafe_cache from ._typing import TypeForm from .conf import _confstruct, _markers +from .constructors._primitive_spec import ( + PrimitiveConstructorSpec, + UnsupportedTypeAnnotationError, +) global_context_markers: List[Tuple[_markers.Marker, ...]] = [] @@ -72,7 +68,8 @@ class FieldDefinition: markers: Set[Any] custom_constructor: bool - argconf: _confstruct._ArgConfiguration + argconf: _confstruct._ArgConfig + primitive_spec: PrimitiveConstructorSpec | None # Override the name in our kwargs. Useful whenever the user-facing argument name # doesn't match the keyword expected by our callable. @@ -82,7 +79,7 @@ def __post_init__(self): if ( _markers.Fixed in self.markers or _markers.Suppress in self.markers ) and self.default in MISSING_SINGLETONS: - raise _instantiators.UnsupportedTypeAnnotationError( + raise UnsupportedTypeAnnotationError( f"Field {self.intern_name} is missing a default value!" ) @@ -98,37 +95,29 @@ def marker_context(markers: Tuple[_markers.Marker, ...]): @staticmethod def make( name: str, - type_or_callable: Union[TypeForm[Any], Callable], + static_type: Union[TypeForm[Any], Callable], default: Any, is_default_from_default_instance: bool, helptext: Optional[str], call_argname_override: Optional[Any] = None, - *, - markers: Tuple[_markers.Marker, ...] = (), ): # Resolve generics. - type_or_callable = _resolver.TypeParamResolver.concretize_type_params( - type_or_callable - ) + static_type = _resolver.TypeParamResolver.concretize_type_params(static_type) # Narrow types. - if type_or_callable is Any and default not in MISSING_SINGLETONS: - type_or_callable = type(default) + if static_type is Any and default not in MISSING_SINGLETONS: + static_type = type(default) else: # TypeVar constraints are already applied in # TypeParamResolver.concretize_type_params(), but that won't be # called for functions. - type_or_callable = _resolver.type_from_typevar_constraints(type_or_callable) - type_or_callable = _resolver.narrow_collection_types( - type_or_callable, default - ) - type_or_callable = _resolver.narrow_union_type(type_or_callable, default) + static_type = _resolver.type_from_typevar_constraints(static_type) + static_type = _resolver.narrow_collection_types(static_type, default) + static_type = _resolver.narrow_union_type(static_type, default) # Try to extract argconf overrides from type. - _, argconfs = _resolver.unwrap_annotated( - type_or_callable, _confstruct._ArgConfiguration - ) - argconf = _confstruct._ArgConfiguration( + _, argconfs = _resolver.unwrap_annotated(static_type, _confstruct._ArgConfig) + argconf = _confstruct._ArgConfig( None, None, help=None, @@ -150,10 +139,15 @@ def make( if argconf.help is not None: helptext = argconf.help - type_or_callable, inferred_markers = _resolver.unwrap_annotated( - type_or_callable, _markers._Marker + _, primitive_specs = _resolver.unwrap_annotated( + static_type, PrimitiveConstructorSpec ) - markers = inferred_markers + markers + if len(primitive_specs) > 0: + primitive_spec = primitive_specs[0] + else: + primitive_spec = None + + static_type, markers = _resolver.unwrap_annotated(static_type, _markers._Marker) # Include markers set via context manager. for context_markers in global_context_markers: @@ -166,9 +160,9 @@ def make( # Be relatively conservative: isinstance() can be checked on non-type # types (like unions in Python >=3.10), but we'll only consider single types # for now. - type(type_or_callable) is type - and not isinstance(default, type_or_callable) # type: ignore - # If a custom constructor is set, type_or_callable may not be + type(static_type) is type + and not isinstance(default, static_type) # type: ignore + # If a custom constructor is set, static_type may not be # matched to the annotated type. and argconf.constructor_factory is None and default not in DEFAULT_SENTINEL_SINGLETONS @@ -179,27 +173,30 @@ def make( # If the default value doesn't match the resolved type, we expand the # type. This is inspired by https://github.com/brentyi/tyro/issues/88. warnings.warn( - f"The field {name} is annotated with type {type_or_callable}, " + f"The field {name} is annotated with type {static_type}, " f"but the default value {default} has type {type(default)}. " f"We'll try to handle this gracefully, but it may cause unexpected behavior." ) - type_or_callable = Union[type_or_callable, type(default)] # type: ignore + static_type = Union[static_type, type(default)] # type: ignore out = FieldDefinition( intern_name=name, extern_name=name if argconf.name is None else argconf.name, - type_or_callable=type_or_callable - if argconf.constructor_factory is None - else argconf.constructor_factory(), + type_or_callable=( + static_type + if argconf.constructor_factory is None + else argconf.constructor_factory() + ), default=default, is_default_from_default_instance=is_default_from_default_instance, helptext=helptext, markers=set(markers), custom_constructor=argconf.constructor_factory is not None, argconf=argconf, - call_argname=call_argname_override - if call_argname_override is not None - else name, + call_argname=( + call_argname_override if call_argname_override is not None else name + ), + primitive_spec=primitive_spec, ) return out @@ -283,20 +280,20 @@ class NotRequiredButWeDontKnowTheValueType(_singleton.Singleton): @dataclasses.dataclass(frozen=True) class UnsupportedNestedTypeMessage: - """Reason why a callable cannot be treated as a nested type.""" + """Reason why a callable cannot be treated as a struct type.""" message: str @_unsafe_cache.unsafe_cache(maxsize=1024) -def is_nested_type( +def is_struct_type( typ: Union[TypeForm[Any], Callable], default_instance: DefaultInstance ) -> bool: - """Determine whether a type should be treated as a 'nested type', where a single + """Determine whether a type should be treated as a 'struct type', where a single type can be broken down into multiple fields (eg for nested dataclasses or classes). - TODO: we should come up with a better name than 'nested type', which is a little bit + TODO: we should come up with a better name than 'struct type', which is a little bit misleading.""" list_or_error = _try_field_list_from_callable(typ, default_instance) @@ -330,21 +327,23 @@ def field_list_from_callable( if isinstance(field_list, UnsupportedNestedTypeMessage): if support_single_arg_types: - return ( - f, - [ - FieldDefinition.make( - name="value", - type_or_callable=f, - default=default_instance, - is_default_from_default_instance=True, - helptext="", - markers=(_markers.Positional, _markers._PositionalCall), - ) - ], - ) + with FieldDefinition.marker_context( + (_markers.Positional, _markers._PositionalCall) + ): + return ( + f, + [ + FieldDefinition.make( + name="value", + static_type=f, + default=default_instance, + is_default_from_default_instance=True, + helptext="", + ) + ], + ) else: - raise _instantiators.UnsupportedTypeAnnotationError(field_list.message) + raise UnsupportedTypeAnnotationError(field_list.message) return f, field_list @@ -381,8 +380,10 @@ def _try_field_list_from_callable( # Check for default instances in subcommand configs. This is needed for # is_nested_type() when arguments are not valid without a default, and this # default is specified in the subcommand config. + from . import conf # Avoid circular import. + f, found_subcommand_configs = _resolver.unwrap_annotated( - f, conf._confstruct._SubcommandConfiguration + f, conf._confstruct._SubcommandConfig ) if len(found_subcommand_configs) > 0: default_instance = found_subcommand_configs[0].default @@ -423,7 +424,7 @@ def _try_field_list_from_callable( return _field_list_from_pydantic(cls, default_instance) # Standard container types. These are different because they can be nested structures - # if they contain other nested types (eg Tuple[Struct, Struct]), or treated as + # if they contain other struct types (eg Tuple[Struct, Struct]), or treated as # single arguments otherwise (eg Tuple[int, int]). # # Note that f_origin will be populated if we annotate as `Tuple[..]`, and cls will @@ -456,7 +457,7 @@ def _try_field_list_from_callable( elif ( cls is not None and issubclass(_resolver.unwrap_origin_strip_extras(cls), os.PathLike) - and _instantiators.is_type_string_converter(cls) + # and _instantiators.is_type_string_converter(cls) ): return UnsupportedNestedTypeMessage( f"PathLike {cls} should be parsed directly!" @@ -491,7 +492,7 @@ def _field_list_from_typeddict( elif total is False: # Support total=False. default = EXCLUDE_FROM_CALL - if is_nested_type(typ, MISSING_NONPROP): + if is_struct_type(typ, MISSING_NONPROP): # total=False behavior is unideal for nested structures. pass # raise _instantiators.UnsupportedTypeAnnotationError( @@ -504,7 +505,7 @@ def _field_list_from_typeddict( default = MISSING_PROP # Nested types need to be populated / can't be excluded from the call. - if default is EXCLUDE_FROM_CALL and is_nested_type(typ, MISSING_NONPROP): + if default is EXCLUDE_FROM_CALL and is_struct_type(typ, MISSING_NONPROP): default = MISSING_NONPROP if typ_origin in (Required, NotRequired): @@ -518,7 +519,7 @@ def _field_list_from_typeddict( field_list.append( FieldDefinition.make( name=name, - type_or_callable=typ, + static_type=typ, default=default, is_default_from_default_instance=is_default_from_default_instance, helptext=_docstrings.get_field_docstring(cls, name), @@ -552,7 +553,7 @@ def _field_list_from_namedtuple( field_list.append( FieldDefinition.make( name=name, - type_or_callable=typ, + static_type=typ, default=default, is_default_from_default_instance=True, helptext=_docstrings.get_field_docstring(cls, name), @@ -604,7 +605,7 @@ def _field_list_from_dataclass( field_list.append( FieldDefinition.make( name=dc_field.name, - type_or_callable=dc_field.type, + static_type=dc_field.type, default=default, is_default_from_default_instance=is_default_from_default_instance, helptext=helptext, @@ -690,7 +691,7 @@ def _field_list_from_pydantic( field_list.append( FieldDefinition.make( name=pd1_field.name, - type_or_callable=hints[pd1_field.name], + static_type=hints[pd1_field.name], default=default, is_default_from_default_instance=is_default_from_default_instance, helptext=helptext, @@ -710,11 +711,13 @@ def _field_list_from_pydantic( field_list.append( FieldDefinition.make( name=name, - type_or_callable=Annotated.__class_getitem__( # type: ignore - (pd2_field.annotation,) + tuple(pd2_field.metadata) - ) - if len(pd2_field.metadata) > 0 - else pd2_field.annotation, + static_type=( + Annotated.__class_getitem__( # type: ignore + (pd2_field.annotation,) + tuple(pd2_field.metadata) + ) + if len(pd2_field.metadata) > 0 + else pd2_field.annotation + ), default=default, is_default_from_default_instance=is_default_from_default_instance, helptext=helptext, @@ -777,7 +780,7 @@ def _field_list_from_attrs( field_list.append( FieldDefinition.make( name=name, - type_or_callable=attr_field.type, + static_type=attr_field.type, default=default, is_default_from_default_instance=is_default_from_default_instance, helptext=_docstrings.get_field_docstring(cls, name), @@ -824,12 +827,12 @@ def _field_list_from_tuple( # argument, but in practice the brackets are annoying because they # require escaping. name=str(i), - type_or_callable=child, + static_type=child, default=default_i, is_default_from_default_instance=True, helptext="", # This should really set the positional marker, but the CLI is more - # intuitive for mixed nested/non-nested types in tuples when we stick + # intuitive for mixed nested/non-struct types in tuples when we stick # with kwargs. Tuples are special-cased in _calling.py. ) ) @@ -840,9 +843,9 @@ def _field_list_from_tuple( for option in get_args(field.type_or_callable): # The second argument here is the default value, which can help with # narrowing but is generall not necessary. - contains_nested |= is_nested_type(option, MISSING_NONPROP) + contains_nested |= is_struct_type(option, MISSING_NONPROP) - contains_nested |= is_nested_type(field.type_or_callable, field.default) + contains_nested |= is_struct_type(field.type_or_callable, field.default) if contains_nested: break @@ -884,7 +887,7 @@ def _try_field_list_from_sequence_inner( # When no default instance is specified: # If we have List[int] => this can be parsed as a single field. # If we have List[SomeStruct] => OK. - if default_instance in MISSING_SINGLETONS and not is_nested_type( + if default_instance in MISSING_SINGLETONS and not is_struct_type( contained_type, MISSING_NONPROP ): return UnsupportedNestedTypeMessage( @@ -895,7 +898,7 @@ def _try_field_list_from_sequence_inner( # [int, int, int] => this can be parsed as a single field. # [SomeStruct, int, int] => OK. if isinstance(default_instance, Iterable) and all( - [not is_nested_type(type(x), x) for x in default_instance] + [not is_struct_type(type(x), x) for x in default_instance] ): return UnsupportedNestedTypeMessage( f"Sequence with default {default_instance} should be parsed directly!" @@ -903,9 +906,9 @@ def _try_field_list_from_sequence_inner( if default_instance in MISSING_SINGLETONS: # We use the broader error type to prevent it from being caught by # is_possibly_nested_type(). This is for sure a bad annotation! - raise _instantiators.UnsupportedTypeAnnotationError( - "tyro currently only supports fixed-length sequences of nested types. For " - "variable-length sequences over nested types, we need a default value to " + raise UnsupportedTypeAnnotationError( + "tyro currently only supports fixed-length sequences of struct types. For " + "variable-length sequences over struct types, we need a default value to " "infer length from. You can also consider a custom constructor, see here " "for an example: https://github.com/brentyi/tyro/issues/151" ) @@ -915,7 +918,7 @@ def _try_field_list_from_sequence_inner( field_list.append( FieldDefinition.make( name=str(i), - type_or_callable=contained_type, + static_type=contained_type, default=default_i, is_default_from_default_instance=True, helptext="", @@ -937,7 +940,7 @@ def _field_list_from_dict( field_list.append( FieldDefinition.make( name=str(k) if not isinstance(k, enum.Enum) else k.name, - type_or_callable=type(v), + static_type=type(v), default=v, is_default_from_default_instance=True, helptext=None, @@ -1026,7 +1029,7 @@ def _field_list_from_params( if param.kind is param.KEYWORD_ONLY: # If keyword only: this can't possibly be an instantiator function # either, so we escalate to an error. - raise _instantiators.UnsupportedTypeAnnotationError(out.message) + raise UnsupportedTypeAnnotationError(out.message) return out # Set markers for positional + variadic arguments. @@ -1054,17 +1057,17 @@ def _field_list_from_params( typ = Dict.__getitem__((str, typ)) # type: ignore default = {} - field_list.append( - FieldDefinition.make( - name=param.name, - # Note that param.annotation doesn't resolve forward references. - type_or_callable=typ, - default=default, - is_default_from_default_instance=False, - helptext=helptext, - markers=markers, + with FieldDefinition.marker_context(markers): + field_list.append( + FieldDefinition.make( + name=param.name, + # Note that param.annotation doesn't resolve forward references. + static_type=typ, + default=default, + is_default_from_default_instance=False, + helptext=helptext, + ) ) - ) return field_list diff --git a/src/tyro/_instantiators.py b/src/tyro/_instantiators.py deleted file mode 100644 index aaf37cd26..000000000 --- a/src/tyro/_instantiators.py +++ /dev/null @@ -1,749 +0,0 @@ -"""Helper for using type annotations to recursively generate instantiator functions, -which map sequences of strings to the annotated type. - -Some examples of type annotations and the desired instantiators: -``` - int - - lambda strings: int(str[0]) - - List[int] - - lambda strings: list( - [int(x) for x in strings] - ) - - List[Color], where Color is an enum - - lambda strings: list( - [Color[x] for x in strings] - ) - - Tuple[int, float] - - lambda strings: tuple( - typ(x) - for typ, x in zip( - (int, float), - strings, - ) - ) -``` -""" - -import builtins -import collections.abc -import dataclasses -import datetime -import enum -import inspect -import os -import pathlib -from collections import deque -from typing import ( - Any, - Callable, - Dict, - Hashable, - Iterable, - List, - Optional, - Sequence, - Set, - Tuple, - Type, - TypeVar, - Union, - cast, - overload, -) - -from typing_extensions import Annotated, Literal, get_args, get_origin - -from . import _resolver - -# There are cases where typing.Literal doesn't match typing_extensions.Literal: -# https://github.com/python/typing_extensions/pull/148 -try: - from typing import Literal as LiteralAlternate -except ImportError: - LiteralAlternate = Literal # type: ignore - -from . import _strings -from ._typing import TypeForm -from .conf import _markers - -_StandardInstantiator = Callable[[List[str]], Any] -_AppendNargsInstantiator = Callable[[List[List[str]]], Any] -# Special case: the only time that argparse doesn't give us a string is when the -# argument action is set to `store_true` or `store_false`. In this case, we get -# a bool directly, and the field action can be a no-op. -_FlagInstantiator = Callable[[bool], bool] - -Instantiator = Union[_StandardInstantiator, _AppendNargsInstantiator, _FlagInstantiator] - -NoneType = type(None) -T = TypeVar("T", bound=enum.Enum) - - -@dataclasses.dataclass -class InstantiatorMetadata: - # Unlike in vanilla argparse, we never set nargs to None. To make things simpler, we - # instead use nargs=1. - nargs: Optional[Union[int, Literal["*"]]] - # Unlike in vanilla argparse, our metavar is always a string. We handle - # sequences, multiple arguments, etc, manually. - metavar: str - choices: Optional[Tuple[str, ...]] - action: Optional[Literal["append"]] - - def check_choices(self, strings: List[str]) -> None: - if self.choices is not None and any(s not in self.choices for s in strings): - raise ValueError(f"invalid choice: {strings} (choose from {self.choices}))") - - -class UnsupportedTypeAnnotationError(Exception): - """Exception raised when an unsupported type annotation is detected.""" - - -_builtin_set: Set[Hashable] = set( - filter( - lambda x: isinstance(x, Hashable), # type: ignore - vars(builtins).values(), # type: ignore - ) -) - - -def is_type_string_converter(typ: Union[Callable, TypeForm[Any]]) -> bool: - """Check if type is a string converter, i.e., (arg: Union[str, Any]) -> T.""" - param_count = 0 - has_var_positional = False - try: - signature = inspect.signature(typ) - except ValueError: - # pybind objects might not have a parsable signature. We try to be tolerant in this case. - # One day this should be fixed with `__text_signature__`. - return True - - type_annotations = _resolver.get_type_hints_with_backported_syntax(typ) - - # Some checks we can do if the signature is available! - for i, param in enumerate(signature.parameters.values()): - annotation = type_annotations.get(param.name, param.annotation) - annotation = _resolver.TypeParamResolver.concretize_type_params(annotation) - if i == 0 and not ( - ( - get_origin(annotation) in (Union, _resolver.UnionType) - and str in get_args(annotation) - ) - or annotation in (str, inspect.Parameter.empty) - ): - return False - if param.kind is inspect.Parameter.VAR_POSITIONAL: - has_var_positional = True - elif param.default is inspect.Parameter.empty and param.kind is not ( - inspect.Parameter.VAR_KEYWORD - ): - param_count += 1 - - # Raise an error if parameters look wrong. - if not (param_count == 1 or (param_count == 0 and has_var_positional)): - return False - return True - - -def instantiator_from_type( - typ: Union[TypeForm[Any], Callable], - markers: Set[_markers.Marker], -) -> Tuple[Instantiator, InstantiatorMetadata]: - """Recursive helper for parsing type annotations. - - Returns two things: - - An instantiator function, for instantiating the type from a string or list of - strings. The latter applies when argparse's `nargs` parameter is set. - - A metadata structure, which specifies parameters for argparse. - """ - - # Handle Any. - if typ is Any: - raise UnsupportedTypeAnnotationError("`Any` is not a parsable type.") - - # Handle NoneType. - if typ is NoneType: - - def instantiator(strings: List[str]) -> None: - # Note that other inputs should be caught by `choices` before the - # instantiator runs. - assert strings == ["None"] - - return instantiator, InstantiatorMetadata( - nargs=1, - metavar="{None}", - choices=("None",), - action=None, - ) - - # Instantiate os.PathLike annotations using pathlib.Path. - # Ideally this should be implemented in a more general way, eg using - # https://github.com/brentyi/tyro/pull/30 - if typ is os.PathLike: - typ = pathlib.Path - - # Unwrap NewType + set metavar based on NewType name. - # `isinstance(x, NewType)` doesn't work because NewType isn't a class until - # Python 3.10, so we instead do a duck typing-style check. - metavar = getattr(typ, "__name__", "").upper() - typ, type_alias_breadcrumbs = _resolver.unwrap_annotated( - typ, _resolver.TyroTypeAliasBreadCrumb - ) - if len(type_alias_breadcrumbs) > 0: - metavar = type_alias_breadcrumbs[-1].name - - # Address container types. If a matching container is found, this will recursively - # call instantiator_from_type(). - container_out = _instantiator_from_container_type(cast(TypeForm[Any], typ), markers) - if container_out is not None: - return container_out - - # At this point: Annotated[] and containers like list[T] are all handled, - # any types with a non-None origin aren't supported. - if get_origin(typ) is not None: - raise UnsupportedTypeAnnotationError( # pragma: no cover - f"Unsupported type {typ} with origin {get_origin(typ)}" - ) - - # Validate that typ is a `(arg: str) -> T` type converter, as expected by argparse. - if typ in (type, Type): - raise UnsupportedTypeAnnotationError( - "We do not support populating `type` instances from the command-line" - ) - elif typ in _builtin_set: - pass - elif not callable(typ): - raise UnsupportedTypeAnnotationError( - f"Expected {typ} to be an `(arg: str) -> T` type converter, but is not" - " callable." - ) - elif not is_type_string_converter(typ): - raise UnsupportedTypeAnnotationError( - f"Expected type to be an `(arg: str) -> T` type converter, but {typ} is not" - " a valid type converter." - ) - - # Use ISO 8601 standard for dates/times. - if typ in (datetime.datetime, datetime.date, datetime.time): - - def instantiate(args: List[str]) -> Any: - (arg,) = args - try: - # Type ignore is unnecessary for pyright but needed for mypy. - return typ.fromisoformat(arg) # type: ignore - except ValueError: - raise ValueError( - f"`{typ.__name__}.fromisoformat('{arg}')` failed. Dates " - "should be specified in ISO-8601 format: " - "https://en.wikipedia.org/wiki/ISO_8601" - ) - - return ( - instantiate, - InstantiatorMetadata( - nargs=1, - metavar={ - # Actually we take any ISO 8601 string here. So these - # metavars are a bit incomplete. - # datetime.datetime: "YYYY-MM-DD[THH:MM:SS]", - # datetime.date: "YYYY-MM-DD", - # datetime.time: "HH:MM[:SS]", - # - # More complete. - datetime.datetime: "YYYY-MM-DD[THH:MM:[SS[…]]]", - datetime.date: "YYYY-MM-DD", - datetime.time: "HH:MM[:SS[…]]", - # - # Even more complete! - # datetime.datetime: "YYYY-MM-DD[THH:MM[:SS[.fff]][±HH:MM|Z]]", - # datetime.date: "YYYY-MM-DD", - # datetime.time: "HH:MM[:SS[.fff]][±HH:MM|Z]", - # - # Not very informative but (1) precise and (2) succinct. - # datetime.datetime: "ISO-DATETIME", - # datetime.date: "ISO-DATE", - # datetime.time: "ISO-TIME", - }[cast(Any, typ)], # cast is for mypy, pyright works fine - choices=None, - action=None, - ), - ) - - # Special case `choices` for some types. Booleans accept {True,False}, - # enums accept based on members. - choices: Union[None, Tuple[str, ...]] = None - autochoice_instantiate: Union[None, Callable[[str], Any]] = None - if typ is bool: - choices = ("True", "False") - autochoice_instantiate = lambda x: {"True": True, "False": False}[x] - elif inspect.isclass(typ) and issubclass(typ, enum.Enum): - if _markers.EnumChoicesFromValues in markers: - value_from_str_choice = {str(member.value): member for member in typ} - choices = tuple(value_from_str_choice.keys()) - autochoice_instantiate = lambda x: value_from_str_choice[x] - else: - choices = tuple(typ.__members__.keys()) - autochoice_instantiate = lambda x: typ[x] - - if choices is not None: - metavar = "{" + ",".join(choices) + "}" - - def instantiator_base_case(strings: List[str]) -> Any: - """Given a type and and a string from the command-line, reconstruct an object. Not - intended to deal with containers. - - This is intended to replace all calls to `type(string)`, which can cause unexpected - behavior. As an example, note that the following argparse code will always print - `True`, because `bool("True") == bool("False") == bool("0") == True`. - ``` - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--flag", type=bool) - - print(parser.parse_args().flag) - ``` - """ - assert len(get_args(typ)) == 0, f"TypeForm {typ} cannot be instantiated." - (string,) = strings - if autochoice_instantiate is not None: - return autochoice_instantiate(string) - elif typ is bytes: - return bytes(string, encoding="ascii") # type: ignore - else: - # We assume this base type can be called on a string to convert - # from a string, eg int("5"), float("5."), Path("/home"), etc. - return typ(string) # type: ignore - - return instantiator_base_case, InstantiatorMetadata( - nargs=1, - metavar=metavar, - choices=choices, - action=None, - ) - - -@overload -def _instantiator_from_type_inner( - typ: TypeForm, - allow_sequences: Literal["fixed_length"], - markers: Set[_markers.Marker], -) -> Tuple[Instantiator, InstantiatorMetadata]: ... - - -@overload -def _instantiator_from_type_inner( - typ: TypeForm, - allow_sequences: Literal[False], - markers: Set[_markers.Marker], -) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: ... - - -@overload -def _instantiator_from_type_inner( - typ: TypeForm, - allow_sequences: Literal[True], - markers: Set[_markers.Marker], -) -> Tuple[Instantiator, InstantiatorMetadata]: ... - - -def _instantiator_from_type_inner( - typ: TypeForm, - allow_sequences: Literal["fixed_length", True, False], - markers: Set[_markers.Marker], -) -> Tuple[Instantiator, InstantiatorMetadata]: - """Thin wrapper over instantiator_from_type, with some extra asserts for catching - errors.""" - out = instantiator_from_type(typ, markers) - if out[1].nargs == "*": - # We currently only use allow_sequences=False for options in Literal types, - # which are evaluated using `type()`. It should not be possible to hit this - # condition from polling a runtime type. - assert allow_sequences - if allow_sequences == "fixed_length" and not isinstance(out[1].nargs, int): - raise UnsupportedTypeAnnotationError( - f"{typ} is a variable-length sequence, which is ambiguous when nested." - " For nesting variable-length sequences (example: List[List[int]])," - " `tyro.conf.UseAppendAction` can help resolve ambiguities." - ) - return out - - -def _instantiator_from_container_type( - typ: TypeForm[Any], - markers: Set[_markers.Marker], -) -> Optional[Tuple[Instantiator, InstantiatorMetadata]]: - """Attempt to create an instantiator from a container type. Returns `None` if no - container type is found.""" - - # Default generic types to strings. - if typ in (dict, Dict): - typ = Dict[str, str] - elif typ in (tuple, Tuple): - typ = Tuple[str, ...] # type: ignore - elif typ in (list, List, collections.abc.Sequence, Sequence): - typ = List[str] - elif typ in (set, Set): - typ = Set[str] - - type_origin = get_origin(typ) - if type_origin in (None, Annotated): - return None - - for make, matched_origins in { - _instantiator_from_sequence: ( - collections.abc.Sequence, - frozenset, - list, - set, - deque, - ), - _instantiator_from_tuple: (tuple,), - _instantiator_from_dict: (dict, collections.abc.Mapping), - _instantiator_from_union: (Union, _resolver.UnionType), - _instantiator_from_literal: (Literal, LiteralAlternate), - }.items(): - if type_origin in matched_origins: - return make(typ, markers) - return None - - -def _instantiator_from_tuple( - typ: TypeForm, - markers: Set[_markers.Marker], -) -> Tuple[Instantiator, InstantiatorMetadata]: - types = get_args(typ) - typeset = set(types) # Note that sets are unordered. - typeset_no_ellipsis = typeset - {Ellipsis} # type: ignore - - if typeset_no_ellipsis != typeset: - # Ellipsis: variable argument counts. When an ellipsis is used, tuples must - # contain only one type. - assert len(typeset_no_ellipsis) == 1 - return _instantiator_from_sequence(typ, markers) - - else: - instantiators: List[_StandardInstantiator] = [] - metas: List[InstantiatorMetadata] = [] - nargs = 0 - for t in types: - a, b = _instantiator_from_type_inner( - t, allow_sequences="fixed_length", markers=markers - ) - instantiators.append(a) # type: ignore - metas.append(b) - assert isinstance(b.nargs, int) - nargs += b.nargs - - def fixed_length_tuple_instantiator(strings: List[str]) -> Any: - assert len(strings) == nargs - - # Make tuple. - out = [] - i = 0 - for make, meta in zip(instantiators, metas): - assert isinstance(meta.nargs, int) - meta.check_choices(strings[i : i + meta.nargs]) - out.append(make(strings[i : i + meta.nargs])) - i += meta.nargs - return tuple(out) - - return fixed_length_tuple_instantiator, InstantiatorMetadata( - nargs=nargs, - metavar=" ".join(m.metavar for m in metas), - choices=None, - action=None, - ) - - -def _join_union_metavars(metavars: Iterable[str]) -> str: - """Metavar generation helper for unions. Could be revisited. - - Examples: - None, INT => NONE|INT - {0,1,2}, {3,4} => {0,1,2,3,4} - {0,1,2}, {3,4}, STR => {0,1,2,3,4}|STR - {None}, INT [INT ...] => {None}|{INT [INT ...]} - STR, INT [INT ...] => STR|{INT [INT ...]} - STR, INT INT => STR|{INT INT} - - The curly brackets are unfortunately overloaded but alternatives all interfere with - argparse internals. - """ - metavars = tuple(metavars) - merged_metavars = [metavars[0]] - for i in range(1, len(metavars)): - prev = merged_metavars[-1] - curr = metavars[i] - if ( - prev.startswith("{") - and prev.endswith("}") - and curr.startswith("{") - and curr.endswith("}") - ): - merged_metavars[-1] = prev[:-1] + "," + curr[1:] - else: - merged_metavars.append(curr) - - for i, m in enumerate(merged_metavars): - if " " in m: - merged_metavars[i] = "{" + m + "}" - - return "|".join(merged_metavars) - - -def _instantiator_from_union( - typ: TypeForm, - markers: Set[_markers.Marker], -) -> Tuple[Instantiator, InstantiatorMetadata]: - options = list(get_args(typ)) - if NoneType in options: - # Move `None` types to the beginning. - # If we have `Optional[str]`, we want this to be parsed as - # `Union[NoneType, str]`. - options.remove(NoneType) - options.insert(0, NoneType) - - # General unions, eg Union[int, bool]. We'll try to convert these from left to - # right. - instantiators = [] - metas = [] - choices: Tuple[str, ...] | None = () - nargs: Optional[Union[int, Literal["*"]]] = 1 - first = True - for t in options: - a, b = _instantiator_from_type_inner(t, allow_sequences=True, markers=markers) - instantiators.append(a) - metas.append(b) - if b.choices is None: - choices = None - elif choices is not None: - choices = choices + b.choices - - if t is not NoneType: - # Enforce that `nargs` is the same for all child types, except for - # NoneType. - if first: - nargs = b.nargs - first = False - elif nargs != b.nargs: - # Just be as general as possible if we see inconsistencies. - nargs = "*" - - metavar: str - metavar = _join_union_metavars(map(lambda x: cast(str, x.metavar), metas)) - - def union_instantiator(strings: List[str]) -> Any: - metadata: InstantiatorMetadata - errors = [] - for i, (instantiator, metadata) in enumerate(zip(instantiators, metas)): - # Check choices. - if metadata.choices is not None and any( - x not in metadata.choices for x in strings - ): - errors.append( - f"{options[i]}: {strings} does not match choices {metadata.choices}" - ) - continue - - # Try passing input into instantiator. - if len(strings) == metadata.nargs or (metadata.nargs == "*"): - try: - return instantiator(strings) # type: ignore - except ValueError as e: - # Failed, try next instantiator. - errors.append(f"{options[i]}: {e.args[0]}") - else: - errors.append( - f"{options[i]}: input length {len(strings)} did not match expected" - f" argument count {metadata.nargs}" - ) - raise ValueError( - f"no type in {options} could be instantiated from" - f" {strings}.\n\nGot errors: \n- " + "\n- ".join(errors) - ) - - return union_instantiator, InstantiatorMetadata( - nargs=nargs, - metavar=metavar, - choices=None if choices is None else tuple(set(choices)), - action=None, - ) - - -def _instantiator_from_dict( - typ: TypeForm, - markers: Set[_markers.Marker], -) -> Tuple[Instantiator, InstantiatorMetadata]: - key_type, val_type = get_args(typ) - key_instantiator, key_meta = _instantiator_from_type_inner( - key_type, allow_sequences="fixed_length", markers=markers - ) - - if _markers.UseAppendAction in markers: - val_instantiator, val_meta = _instantiator_from_type_inner( - val_type, - allow_sequences=True, - markers=markers - {_markers.UseAppendAction}, - ) - pair_metavar = f"{key_meta.metavar} {val_meta.metavar}" - key_nargs = cast(int, key_meta.nargs) # Casts needed for mypy but not pyright! - val_nargs = val_meta.nargs - assert isinstance(key_nargs, int) - - def append_dict_instantiator(strings: List[List[str]]) -> Any: - out = {} - for s in strings: - out[key_instantiator(s[:key_nargs])] = val_instantiator(s[key_nargs:]) # type: ignore - return out - - return append_dict_instantiator, InstantiatorMetadata( - nargs=key_nargs + val_nargs if isinstance(val_nargs, int) else "*", - metavar=pair_metavar, - choices=None, - action="append", - ) - else: - val_instantiator, val_meta = _instantiator_from_type_inner( - val_type, allow_sequences="fixed_length", markers=markers - ) - pair_metavar = f"{key_meta.metavar} {val_meta.metavar}" - key_nargs = cast(int, key_meta.nargs) # Casts needed for mypy but not pyright! - val_nargs = cast(int, val_meta.nargs) - assert isinstance(key_nargs, int) - assert isinstance(val_nargs, int) - pair_nargs = key_nargs + val_nargs - - def dict_instantiator(strings: List[str]) -> Any: - out = {} - if len(strings) % pair_nargs != 0: - raise ValueError("incomplete set of key value pairs!") - - index = 0 - for _ in range(len(strings) // pair_nargs): - assert isinstance(key_nargs, int) - assert isinstance(val_nargs, int) - k = strings[index : index + key_nargs] - index += key_nargs - v = strings[index : index + val_nargs] - index += val_nargs - - if key_meta.choices is not None and any( - kj not in key_meta.choices for kj in k - ): - raise ValueError( - f"invalid choice: {k} (choose from {key_meta.choices}))" - ) - if val_meta.choices is not None and any( - vj not in val_meta.choices for vj in v - ): - raise ValueError( - f"invalid choice: {v} (choose from {val_meta.choices}))" - ) - out[key_instantiator(k)] = val_instantiator(v) # type: ignore - return out - - return dict_instantiator, InstantiatorMetadata( - nargs="*", - metavar=_strings.multi_metavar_from_single(pair_metavar), - choices=None, - action=None, - ) - - -def _instantiator_from_sequence( - typ: TypeForm, - markers: Set[_markers.Marker], -) -> Tuple[Instantiator, InstantiatorMetadata]: - """Instantiator for variable-length sequences: list, sets, Tuple[T, ...], etc.""" - container_type = get_origin(typ) - assert container_type is not None - if container_type is collections.abc.Sequence: - container_type = list - - if container_type is tuple: - (contained_type, ell) = get_args(typ) - assert ell == Ellipsis - else: - (contained_type,) = get_args(typ) - - if _markers.UseAppendAction in markers: - make, inner_meta = _instantiator_from_type_inner( - contained_type, - allow_sequences=True, - markers=markers - {_markers.UseAppendAction}, - ) - - def append_sequence_instantiator(strings: List[List[str]]) -> Any: - assert strings is not None - return container_type(cast(_StandardInstantiator, make)(s) for s in strings) - - return append_sequence_instantiator, InstantiatorMetadata( - nargs=inner_meta.nargs, - metavar=inner_meta.metavar, - choices=inner_meta.choices, - action="append", - ) - else: - make, inner_meta = _instantiator_from_type_inner( - contained_type, - allow_sequences="fixed_length", - markers=markers, - ) - - def sequence_instantiator(strings: List[str]) -> Any: - # Validate nargs. - if ( - isinstance(inner_meta.nargs, int) - and len(strings) % inner_meta.nargs != 0 - ): - raise ValueError( - f"input {strings} is of length {len(strings)}, which is not" - f" divisible by {inner_meta.nargs}." - ) - - # Make tuple. - out = [] - step = inner_meta.nargs if isinstance(inner_meta.nargs, int) else 1 - for i in range(0, len(strings), step): - out.append(make(strings[i : i + inner_meta.nargs])) # type: ignore - assert container_type is not None - return container_type(out) - - return sequence_instantiator, InstantiatorMetadata( - nargs="*", - metavar=_strings.multi_metavar_from_single(inner_meta.metavar), - choices=inner_meta.choices, - action=None, - ) - - -def _instantiator_from_literal( - typ: TypeForm, - markers: Set[_markers.Marker], -) -> Tuple[_StandardInstantiator, InstantiatorMetadata]: - choices = get_args(typ) - str_choices = tuple( - (x.value if _markers.EnumChoicesFromValues in markers else x.name) - if isinstance(x, enum.Enum) - else str(x) - for x in choices - ) - return ( - # Note that if string is not in str_choices, it will be caught from setting - # `choices` below. - lambda strings: choices[str_choices.index(strings[0])], - InstantiatorMetadata( - nargs=1, - metavar="{" + ",".join(str_choices) + "}", - choices=str_choices, - action=None, - ), - ) diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 228021132..0e5f28ce9 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -25,13 +25,13 @@ _arguments, _docstrings, _fields, - _instantiators, _resolver, _strings, _subcommand_matching, ) from ._typing import TypeForm from .conf import _confstruct, _markers +from .constructors._primitive_spec import UnsupportedTypeAnnotationError T = TypeVar("T") @@ -92,7 +92,7 @@ def from_callable_or_type( # Note that 'parent' here refers to in the nesting hierarchy, not the # superclass. if f in parent_classes and f is not dict: - raise _instantiators.UnsupportedTypeAnnotationError( + raise UnsupportedTypeAnnotationError( f"Found a cyclic dependency with type {f}." ) @@ -310,12 +310,11 @@ def handle_field( ]: """Determine what to do with a single field definition.""" - if isinstance(field.type_or_callable, TypeVar): - raise _instantiators.UnsupportedTypeAnnotationError( - f"Field {field.intern_name} has an unbound TypeVar: {field.type_or_callable}." - ) - - if _markers.Fixed not in field.markers and _markers.Suppress not in field.markers: + if ( + field.primitive_spec is None + and _markers.Fixed not in field.markers + and _markers.Suppress not in field.markers + ): # (1) Handle Unions over callables; these result in subparsers. subparsers_attempt = SubparsersSpecification.from_field( field, @@ -334,7 +333,7 @@ def handle_field( return subparsers_attempt # (2) Handle nested callables. - if _fields.is_nested_type(field.type_or_callable, field.default): + if _fields.is_struct_type(field.type_or_callable, field.default): field = dataclasses.replace( field, type_or_callable=_resolver.narrow_subtypes( @@ -351,11 +350,11 @@ def handle_field( intern_prefix=_strings.make_field_name( [intern_prefix, field.intern_name] ), - extern_prefix=_strings.make_field_name( - [extern_prefix, field.extern_name] - ) - if field.argconf.prefix_name in (True, None) - else field.extern_name, + extern_prefix=( + _strings.make_field_name([extern_prefix, field.extern_name]) + if field.argconf.prefix_name in (True, None) + else field.extern_name + ), subcommand_prefix=subcommand_prefix, support_single_arg_types=False, ) @@ -407,7 +406,7 @@ def from_field( # If specified, swap types using tyro.conf.subcommand(constructor=...). for i, option in enumerate(options): _, found_subcommand_configs = _resolver.unwrap_annotated( - option, _confstruct._SubcommandConfiguration + option, _confstruct._SubcommandConfig ) if ( len(found_subcommand_configs) > 0 @@ -424,26 +423,26 @@ def from_field( if not any( [ o is not none_proxy - and _fields.is_nested_type(cast(type, o), _fields.MISSING_NONPROP) + and _fields.is_struct_type(cast(type, o), _fields.MISSING_NONPROP) for o in options ] ): return None # Get subcommand configurations from `tyro.conf.subcommand()`. - subcommand_config_from_name: Dict[ - str, _confstruct._SubcommandConfiguration - ] = {} + subcommand_config_from_name: Dict[str, _confstruct._SubcommandConfig] = {} subcommand_type_from_name: Dict[str, type] = {} for option in options: subcommand_name = _strings.subparser_name_from_type( - "" - if _markers.OmitSubcommandPrefixes in field.markers - else extern_prefix, + ( + "" + if _markers.OmitSubcommandPrefixes in field.markers + else extern_prefix + ), type(None) if option is none_proxy else cast(type, option), ) option_unwrapped, found_subcommand_configs = _resolver.unwrap_annotated( - option, _confstruct._SubcommandConfiguration + option, _confstruct._SubcommandConfig ) if len(found_subcommand_configs) != 0: # Explicitly annotated default. @@ -479,9 +478,11 @@ def from_field( parser_from_name: Dict[str, ParserSpecification] = {} for option in options: subcommand_name = _strings.subparser_name_from_type( - "" - if _markers.OmitSubcommandPrefixes in field.markers - else extern_prefix, + ( + "" + if _markers.OmitSubcommandPrefixes in field.markers + else extern_prefix + ), type(None) if option is none_proxy else cast(type, option), ) @@ -490,7 +491,7 @@ def from_field( if subcommand_name in subcommand_config_from_name: subcommand_config = subcommand_config_from_name[subcommand_name] else: - subcommand_config = _confstruct._SubcommandConfiguration( + subcommand_config = _confstruct._SubcommandConfig( "unused", description=None, default=_fields.MISSING_NONPROP, @@ -513,7 +514,7 @@ def from_field( annotations = tuple( a for a in annotations - if not isinstance(a, _confstruct._SubcommandConfiguration) + if not isinstance(a, _confstruct._SubcommandConfig) ) if len(annotations) == 0: option = option_origin diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 74ca303c5..f1588b058 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -215,8 +215,8 @@ def swap_type_using_confstruct(typ: TypeOrCallable) -> TypeOrCallable: isinstance( anno, ( - conf._confstruct._ArgConfiguration, - conf._confstruct._SubcommandConfiguration, + conf._confstruct._ArgConfig, + conf._confstruct._SubcommandConfig, ), ) and anno.constructor_factory is not None diff --git a/src/tyro/_strings.py b/src/tyro/_strings.py index 2882a1986..2e32c7962 100644 --- a/src/tyro/_strings.py +++ b/src/tyro/_strings.py @@ -4,7 +4,7 @@ import functools import re import textwrap -from typing import List, Sequence, Tuple, Type +from typing import Iterable, List, Sequence, Tuple, Type from typing_extensions import Literal, get_args, get_origin @@ -84,7 +84,7 @@ def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: cls, _resolver.TyroTypeAliasBreadCrumb ) cls, found_subcommand_configs = _resolver.unwrap_annotated( - cls, _confstruct._SubcommandConfiguration + cls, _confstruct._SubcommandConfig ) # Subparser name from `tyro.conf.subcommand()`. @@ -100,9 +100,10 @@ def _subparser_name_from_type(cls: Type) -> Tuple[str, bool]: # Subparser name from type alias. This is lower priority thant he name from # `tyro.conf.subcommand()`. if len(type_alias_breadcrumbs) > 0: - return hyphen_separated_from_camel_case( - type_alias_breadcrumbs[-1].name - ), prefix_name + return ( + hyphen_separated_from_camel_case(type_alias_breadcrumbs[-1].name), + prefix_name, + ) # Subparser name from class name. def get_name(cls: Type) -> str: @@ -166,6 +167,42 @@ def multi_metavar_from_single(single: str) -> str: return f"[{single} [{single} ...]]" +def join_union_metavars(metavars: Iterable[str]) -> str: + """Metavar generation helper for unions. Could be revisited. + + Examples: + None, INT => NONE|INT + {0,1,2}, {3,4} => {0,1,2,3,4} + {0,1,2}, {3,4}, STR => {0,1,2,3,4}|STR + {None}, INT [INT ...] => {None}|{INT [INT ...]} + STR, INT [INT ...] => STR|{INT [INT ...]} + STR, INT INT => STR|{INT INT} + + The curly brackets are unfortunately overloaded but alternatives all interfere with + argparse internals. + """ + metavars = tuple(metavars) + merged_metavars = [metavars[0]] + for i in range(1, len(metavars)): + prev = merged_metavars[-1] + curr = metavars[i] + if ( + prev.startswith("{") + and prev.endswith("}") + and curr.startswith("{") + and curr.endswith("}") + ): + merged_metavars[-1] = prev[:-1] + "," + curr[1:] + else: + merged_metavars.append(curr) + + for i, m in enumerate(merged_metavars): + if " " in m: + merged_metavars[i] = "{" + m + "}" + + return "|".join(merged_metavars) + + def remove_single_line_breaks(helptext: str) -> str: lines = helptext.split("\n") output_parts: List[str] = [] diff --git a/src/tyro/_subcommand_matching.py b/src/tyro/_subcommand_matching.py index 2e6f523bb..4cb07f1fb 100644 --- a/src/tyro/_subcommand_matching.py +++ b/src/tyro/_subcommand_matching.py @@ -5,13 +5,14 @@ from typing_extensions import get_args, get_origin -from . import _fields, _instantiators, _resolver, _typing +from . import _fields, _resolver, _typing from .conf import _confstruct +from .constructors._primitive_spec import UnsupportedTypeAnnotationError def match_subcommand( default: Any, - subcommand_config_from_name: Dict[str, _confstruct._SubcommandConfiguration], + subcommand_config_from_name: Dict[str, _confstruct._SubcommandConfig], subcommand_type_from_name: Dict[str, type], ) -> Optional[str]: """Given a subcommand mapping and a default, return which subcommand the default @@ -82,7 +83,7 @@ def make( typ, field_list = _fields.field_list_from_callable( typ, default_instance=default_instance, support_single_arg_types=False ) - except _instantiators.UnsupportedTypeAnnotationError: + except UnsupportedTypeAnnotationError: return _TypeTree(typ_unwrap, {}) return _TypeTree( diff --git a/src/tyro/conf/_confstruct.py b/src/tyro/conf/_confstruct.py index ad4f57b10..08fa21ce5 100644 --- a/src/tyro/conf/_confstruct.py +++ b/src/tyro/conf/_confstruct.py @@ -1,13 +1,15 @@ from __future__ import annotations import dataclasses -from typing import Any, Callable, Sequence, overload +from typing import Any, Callable, Sequence, TypeVar, overload from .._fields import MISSING_NONPROP +T = TypeVar("T") + @dataclasses.dataclass(frozen=True) -class _SubcommandConfiguration: +class _SubcommandConfig: name: str | None default: Any description: str | None @@ -99,19 +101,19 @@ def subcommand( assert not ( constructor is not None and constructor_factory is not None ), "`constructor` and `constructor_factory` cannot both be set." - return _SubcommandConfiguration( + return _SubcommandConfig( name, default, description, prefix_name, - constructor_factory=constructor_factory - if constructor is None - else lambda: constructor, + constructor_factory=( + constructor_factory if constructor is None else lambda: constructor + ), ) @dataclasses.dataclass(frozen=True) -class _ArgConfiguration: +class _ArgConfig: name: str | None metavar: str | None help: str | None @@ -205,14 +207,14 @@ def arg( for alias in aliases: assert alias.startswith("-"), "Argument alias needs to start with a hyphen!" - return _ArgConfiguration( + return _ArgConfig( name=name, metavar=metavar, help=help, help_behavior_hint=help_behavior_hint, aliases=tuple(aliases) if aliases is not None else None, prefix_name=prefix_name, - constructor_factory=constructor_factory - if constructor is None - else lambda: constructor, + constructor_factory=( + constructor_factory if constructor is None else lambda: constructor + ), ) diff --git a/src/tyro/constructors/__init__.py b/src/tyro/constructors/__init__.py new file mode 100644 index 000000000..34d3a7f52 --- /dev/null +++ b/src/tyro/constructors/__init__.py @@ -0,0 +1,5 @@ +from ._primitive_spec import ( + PrimitiveConstructorRegistry as PrimitiveConstructorRegistry, +) +from ._primitive_spec import PrimitiveConstructorSpec as PrimitiveConstructorSpec +from ._primitive_spec import PrimitiveTypeInfo as PrimitiveTypeInfo diff --git a/src/tyro/constructors/_primitive_spec.py b/src/tyro/constructors/_primitive_spec.py new file mode 100644 index 000000000..12d2ef371 --- /dev/null +++ b/src/tyro/constructors/_primitive_spec.py @@ -0,0 +1,705 @@ +from __future__ import annotations + +import collections +import collections.abc +import dataclasses +import datetime +import enum +import inspect +import json +import os +import pathlib +import sys +from typing import ( + Any, + Callable, + ClassVar, + Dict, + Generic, + List, + Sequence, + Set, + Tuple, + Type, + TypeVar, + Union, + cast, +) + +from typing_extensions import Literal, get_args, get_origin + +# There are cases where typing.Literal doesn't match typing_extensions.Literal: +# https://github.com/python/typing_extensions/pull/148 +try: + from typing import Literal as LiteralAlternate +except ImportError: + LiteralAlternate = Literal # type: ignore + + +from .. import _resolver, _strings +from .._typing import TypeForm +from ..conf import _markers + + +class UnsupportedTypeAnnotationError(Exception): + """Exception raised when an unsupported type annotation is detected.""" + + +T = TypeVar("T") + + +@dataclasses.dataclass(frozen=True) +class PrimitiveTypeInfo: + """Information used to generate constructors for primitive types.""" + + type: TypeForm + """Field type, with runtime annotations (typing.Annotated) stripped.""" + type_origin: TypeForm | None + """The output of get_origin() on the static type.""" + markers: set[_markers.Marker] + """Set of tyro markers used to configure this field.""" + constructor_registry: PrimitiveConstructorRegistry + """The registry used to look up constructor specifications for this type.""" + + @staticmethod + def make( + raw_annotation: TypeForm, + parent_markers: set[_markers.Marker], + source_registry: PrimitiveConstructorRegistry, + ) -> PrimitiveTypeInfo: + typ, extra_markers = _resolver.unwrap_annotated( + raw_annotation, search_type=_markers._Marker + ) + return PrimitiveTypeInfo( + type=typ, + type_origin=get_origin(typ), + markers=parent_markers | set(extra_markers), + constructor_registry=source_registry, + ) + + +@dataclasses.dataclass(frozen=True) +class PrimitiveConstructorSpec(Generic[T]): + """Specification for constructing a primitive type from a string. + + There are two ways to use this class: + + First, we can include it in a type signature via `typing.Annotated`. + This is the simplest, and allows for per-field customization of + instantiation behavior. + + Alternatively, it can be returned by a rule in a `PrimitiveConstructorRegistry`. + """ + + nargs: int | Literal["*"] + """Number of arguments required to construct an instance. If nargs is "*", then + the number of arguments is variable.""" + metavar: str + """Metavar to display in help messages.""" + instance_from_str: Callable[[list[str]], T] + """Given a list of string arguments, construct an instance of the type. The + length of the list will match the value of nargs.""" + is_instance: Callable[[Any], bool] + """Given an object instance, does it match this primitive type? This is + used for help messages when a default is provided.""" + str_from_instance: Callable[[T], list[str]] + """Convert an instance to a list of string arguments that would construct + the instance. This is used for help messages when a default is provided.""" + choices: tuple[str, ...] | None = None + """Finite set of choices for arguments.""" + + _action: Literal["append"] | None = None + """Internal action to use. Not part of the public API.""" + + +SpecFactory = Callable[[PrimitiveTypeInfo], PrimitiveConstructorSpec] + +current_registry: PrimitiveConstructorRegistry | None = None + + +class PrimitiveConstructorRegistry: + """Registry for rules that define how primitive types that can be + constructed from a single command-line argument.""" + + _active_registry: ClassVar[PrimitiveConstructorRegistry | None] = None + _old_registry: PrimitiveConstructorRegistry | None = None + + def __init__(self) -> None: + self._rules: list[ + tuple[ + # Matching function. + Callable[[PrimitiveTypeInfo], bool], + # Spec factory. + SpecFactory, + ] + ] = [] + + # Apply the default primitive-handling rules. + _apply_default_rules(self) + + def define_rule( + self, matcher_fn: Callable[[PrimitiveTypeInfo], bool] + ) -> Callable[[SpecFactory], SpecFactory]: + """Define a rule for constructing a primitive type from a string. The + most recently added rule will be applied first.""" + + def decorator(spec_factory: SpecFactory) -> SpecFactory: + self._rules.append((matcher_fn, spec_factory)) + return spec_factory + + return decorator + + def get_spec(self, type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + """Get a constructor specification for a given type.""" + for matcher_fn, spec_factory in self._rules[::-1]: + if matcher_fn(type_info): + return spec_factory(type_info) + + raise UnsupportedTypeAnnotationError( + f"Unsupported type annotation: {type_info.type}" + ) + + @classmethod + def _get_active_registry(cls) -> PrimitiveConstructorRegistry: + """Get the active registry. Can be changed by using a + PrimitiveConstructorRegistry object as a context.""" + if cls._active_registry is None: + cls._active_registry = PrimitiveConstructorRegistry() + return cls._active_registry + + def __enter__(self) -> None: + cls = self.__class__ + self._old_registry = cls._active_registry + cls._active_registry = self + + def __exit__(self, *args: Any) -> None: + cls = self.__class__ + cls._active_registry = self._old_registry + + +def _apply_default_rules(registry: PrimitiveConstructorRegistry) -> None: + """Apply default rules to the registry.""" + + @registry.define_rule(lambda type_info: type_info.type is Any) + def any_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + raise UnsupportedTypeAnnotationError("`Any` is not a parsable type.") + + # HACK: this is for code that uses `tyro.conf.arg(constructor=json.loads)`. + # We're going to deprecate this syntax (the constructor= argument in + # tyro.conf.arg), but there is code that lives in the wild that relies + # on the behavior so we'll do our best not to break it. + vanilla_types = (int, str, float, bytes, json.loads) + + @registry.define_rule(lambda type_info: type_info.type in vanilla_types) + def basics_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + return PrimitiveConstructorSpec( + nargs=1, + metavar=type_info.type.__name__.upper(), + instance_from_str=lambda args: ( + bytes(args[0], encoding="ascii") + if type_info.type is bytes + else type_info.type(args[0]) + ), + is_instance=lambda x: isinstance(x, type_info.type), + str_from_instance=lambda instance: [str(instance)], + ) + + if "torch" in sys.modules.keys(): + import torch + + @registry.define_rule(lambda type_info: type_info.type is torch.device) + def basics_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + return PrimitiveConstructorSpec( + nargs=1, + metavar=type_info.type.__name__.upper(), + instance_from_str=lambda args: torch.device(args[0]), + is_instance=lambda x: isinstance(x, type_info.type), + str_from_instance=lambda instance: [str(instance)], + ) + + @registry.define_rule(lambda type_info: type_info.type is bool) + def bool_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + del type_info + return PrimitiveConstructorSpec( + nargs=1, + metavar="{True,False}", + instance_from_str=lambda args: args[0] == "True", + choices=("True", "False"), + is_instance=lambda x: isinstance(x, bool), + str_from_instance=lambda instance: ["True" if instance else "False"], + ) + + @registry.define_rule(lambda type_info: type_info.type is type(None)) + def nonetype_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + del type_info + return PrimitiveConstructorSpec( + nargs=1, + metavar="{None}", + choices=("None",), + instance_from_str=lambda args: None, + is_instance=lambda x: x is None, + str_from_instance=lambda instance: ["None"], + ) + + @registry.define_rule( + lambda type_info: type_info.type in (os.PathLike, pathlib.Path) + or ( + inspect.isclass(type_info.type) + and issubclass(type_info.type, pathlib.PurePath) + ) + ) + def path_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + return PrimitiveConstructorSpec( + nargs=1, + metavar=type_info.type.__name__.upper(), + instance_from_str=lambda args: pathlib.Path(args[0]), + is_instance=lambda x: hasattr(x, "__fspath__"), + str_from_instance=lambda instance: [str(instance)], + ) + + @registry.define_rule( + lambda type_info: inspect.isclass(type_info.type) + and issubclass(type_info.type, enum.Enum) + ) + def enum_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + cast_type = cast(Type[enum.Enum], type_info.type) + if _markers.EnumChoicesFromValues in type_info.markers: + choices = tuple(str(m.value) for m in cast_type) + else: + choices = tuple(type_info.type.__members__.keys()) + + return PrimitiveConstructorSpec( + nargs=1, + metavar="{" + ",".join(choices) + "}", + instance_from_str=lambda args: ( + next( + iter(member for member in cast_type if str(member.value) == args[0]) + ) + if _markers.EnumChoicesFromValues in type_info.markers + else cast_type[args[0]] + ), + is_instance=lambda x: isinstance(x, cast_type), + str_from_instance=lambda instance: [ + ( + str(instance.value) + if _markers.EnumChoicesFromValues in type_info.markers + else instance.name + ) + ], + choices=choices, + ) + + @registry.define_rule( + lambda type_info: type_info.type + in (datetime.datetime, datetime.date, datetime.time) + ) + def datetime_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + return PrimitiveConstructorSpec( + nargs=1, + metavar={ + datetime.datetime: "YYYY-MM-DD[THH:MM:[SS[…]]]", + datetime.date: "YYYY-MM-DD", + datetime.time: "HH:MM[:SS[…]]", + }[type_info.type], + instance_from_str=lambda args: cast( + Union[ + Type[datetime.datetime], Type[datetime.date], Type[datetime.time] + ], + type_info.type, + ).fromisoformat(args[0]), + is_instance=lambda x: isinstance(x, type_info.type), + str_from_instance=lambda instance: [instance.isoformat()], + ) + + @registry.define_rule(lambda type_info: type_info.type_origin is tuple) + def tuple_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + types = get_args(type_info.type) + typeset = set(types) # Note that sets are unordered. + typeset_no_ellipsis = typeset - {Ellipsis} # type: ignore + + if typeset_no_ellipsis != typeset: + # Ellipsis: variable argument counts. When an ellipsis is used, tuples must + # contain only one type. + assert len(typeset_no_ellipsis) == 1 + return sequence_rule(type_info) + + inner_specs = [] + total_nargs = 0 + for contained_type in types: + spec = type_info.constructor_registry.get_spec( + PrimitiveTypeInfo.make( + contained_type, type_info.markers, type_info.constructor_registry + ) + ) + if isinstance(spec.nargs, int): + total_nargs += spec.nargs + else: + raise UnsupportedTypeAnnotationError( + f"Tuples containing a variable-length sequences ({contained_type}) are not supported." + ) + + inner_specs.append(spec) + + def instance_from_str(args: list[str]) -> tuple: + assert len(args) == total_nargs + + out = [] + i = 0 + for member_spec in inner_specs: + assert isinstance(member_spec.nargs, int) + member_strings = args[i : i + member_spec.nargs] + if member_spec.choices is not None and any( + s not in member_spec.choices for s in member_strings + ): + raise ValueError( + f"invalid choice: {member_strings} (choose from {member_spec.choices}))" + ) + out.append(member_spec.instance_from_str(member_strings)) + i += member_spec.nargs + return tuple(out) + + def str_from_instance(instance: tuple) -> list[str]: + out = [] + for member, spec in zip(instance, inner_specs): + out.extend(spec.str_from_instance(member)) + return out + + return PrimitiveConstructorSpec( + nargs=total_nargs, + metavar=" ".join(spec.metavar for spec in inner_specs), + instance_from_str=instance_from_str, + str_from_instance=str_from_instance, + is_instance=lambda x: isinstance(x, tuple) + and len(x) == total_nargs + and all(spec.is_instance(member) for member, spec in zip(x, inner_specs)), + ) + + @registry.define_rule( + lambda type_info: type_info.type + in ( + dict, + Dict, + tuple, + Tuple, + list, + List, + collections.abc.Sequence, + Sequence, + set, + Set, + ) + ) + def vague_container_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + typ = type_info.type + if typ in (dict, Dict): + typ = Dict[str, str] + elif typ in (tuple, Tuple): + typ = Tuple[str, ...] # type: ignore + elif typ in (list, List, collections.abc.Sequence, Sequence): + typ = List[str] + elif typ in (set, Set): + typ = Set[str] + return type_info.constructor_registry.get_spec( + PrimitiveTypeInfo.make( + typ, + parent_markers=type_info.markers, + source_registry=type_info.constructor_registry, + ) + ) + + @registry.define_rule( + lambda type_info: type_info.type_origin + in ( + collections.abc.Sequence, + frozenset, + list, + set, + collections.deque, + ) + ) + def sequence_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + container_type = type_info.type_origin + assert container_type is not None + if container_type is collections.abc.Sequence: + container_type = list + + if container_type is tuple: + (contained_type, ell) = get_args(type_info.type) + assert ell == Ellipsis + else: + (contained_type,) = get_args(type_info.type) + + inner_spec = type_info.constructor_registry.get_spec( + PrimitiveTypeInfo.make( + raw_annotation=contained_type, + parent_markers=type_info.markers - {_markers.UseAppendAction}, + source_registry=type_info.constructor_registry, + ) + ) + + if _markers.UseAppendAction not in type_info.markers and not isinstance( + inner_spec.nargs, int + ): + raise UnsupportedTypeAnnotationError( + f"{container_type} and {contained_type} are both variable-length sequences." + " This causes ambiguity." + " For nesting variable-length sequences (example: List[List[int]])," + " `tyro.conf.UseAppendAction` can help resolve ambiguities." + ) + + def instance_from_str(args: list[str]) -> Any: + # Validate nargs. + assert isinstance(inner_spec.nargs, int) + if isinstance(inner_spec.nargs, int) and len(args) % inner_spec.nargs != 0: + raise ValueError( + f"input {args} is of length {len(args)}, which is not" + f" divisible by {inner_spec.nargs}." + ) + + # Instantiate. + out = [] + step = inner_spec.nargs if isinstance(inner_spec.nargs, int) else 1 + for i in range(0, len(args), step): + out.append(inner_spec.instance_from_str(args[i : i + inner_spec.nargs])) + assert container_type is not None + return container_type(out) + + def str_from_instance(instance: Sequence) -> list[str]: + out = [] + for i in instance: + out.extend(inner_spec.str_from_instance(i)) + return out + + if _markers.UseAppendAction in type_info.markers: + return PrimitiveConstructorSpec( + nargs=inner_spec.nargs, + metavar=inner_spec.metavar, + instance_from_str=inner_spec.instance_from_str, + is_instance=lambda x: isinstance(x, container_type) + and all(inner_spec.is_instance(i) for i in x), + str_from_instance=str_from_instance, + choices=inner_spec.choices, + _action="append", + ) + else: + return PrimitiveConstructorSpec( + nargs="*", + metavar=_strings.multi_metavar_from_single(inner_spec.metavar), + instance_from_str=instance_from_str, + is_instance=lambda x: isinstance(x, container_type) + and all(inner_spec.is_instance(i) for i in x), + str_from_instance=str_from_instance, + choices=inner_spec.choices, + ) + + @registry.define_rule( + lambda type_info: type_info.type_origin in (dict, collections.abc.Mapping) + ) + def dict_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + key_type, val_type = get_args(type_info.type) + key_spec = type_info.constructor_registry.get_spec( + PrimitiveTypeInfo.make( + raw_annotation=key_type, + parent_markers=type_info.markers, + source_registry=type_info.constructor_registry, + ) + ) + val_spec = type_info.constructor_registry.get_spec( + PrimitiveTypeInfo.make( + raw_annotation=val_type, + parent_markers=type_info.markers - {_markers.UseAppendAction}, + source_registry=type_info.constructor_registry, + ) + ) + pair_metavar = f"{key_spec.metavar} {val_spec.metavar}" + + if not isinstance(key_spec.nargs, int): + raise UnsupportedTypeAnnotationError( + "Dictionary keys must have a fixed number of arguments." + ) + + if _markers.UseAppendAction not in type_info.markers and not isinstance( + val_spec.nargs, int + ): + raise UnsupportedTypeAnnotationError( + "Dictionary values must have a fixed number of arguments." + ) + + def instance_from_str(args: list[str]) -> dict: + out = {} + key_nargs = key_spec.nargs + assert isinstance(key_nargs, int) + val_nargs = ( + val_spec.nargs + if _markers.UseAppendAction not in type_info.markers + else len(args) - key_nargs + ) + assert isinstance(val_nargs, int) + + pair_nargs = key_nargs + val_nargs + if len(args) % pair_nargs != 0: + raise ValueError("Incomplete set of key-value pairs!") + + for i in range(0, len(args), pair_nargs): + key = key_spec.instance_from_str(args[i : i + key_nargs]) + value = val_spec.instance_from_str(args[i + key_nargs : i + pair_nargs]) + out[key] = value + return out + + def str_from_instance(instance: dict) -> list[str]: + out: list[str] = [] + assert ( + len(instance) == 0 + ), "When parsed as a primitive, we currrently assume all defaults are length=0. Dictionaries with non-zero-length defaults are interpreted as struct types." + # for key, value in instance.items(): + # out.extend(key_spec.str_from_instance(key)) + # out.extend(val_spec.str_from_instance(value)) + return out + + if _markers.UseAppendAction in type_info.markers: + return PrimitiveConstructorSpec( + nargs=( + key_spec.nargs + val_spec.nargs + if isinstance(val_spec.nargs, int) + else "*" + ), + metavar=pair_metavar, + instance_from_str=instance_from_str, + is_instance=lambda x: isinstance(x, dict) + and all( + key_spec.is_instance(k) and val_spec.is_instance(v) + for k, v in x.items() + ), + str_from_instance=str_from_instance, + _action="append", + ) + else: + return PrimitiveConstructorSpec( + nargs="*", + metavar=_strings.multi_metavar_from_single(pair_metavar), + instance_from_str=instance_from_str, + is_instance=lambda x: isinstance(x, dict) + and all( + key_spec.is_instance(k) and val_spec.is_instance(v) + for k, v in x.items() + ), + str_from_instance=str_from_instance, + ) + + @registry.define_rule( + lambda type_info: type_info.type_origin in (Literal, LiteralAlternate) + ) + def literal_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + choices = get_args(type_info.type) + str_choices = tuple( + ( + ( + x.value + if _markers.EnumChoicesFromValues in type_info.markers + else x.name + ) + if isinstance(x, enum.Enum) + else str(x) + ) + for x in choices + ) + return PrimitiveConstructorSpec( + nargs=1, + metavar="{" + ",".join(str_choices) + "}", + instance_from_str=lambda args: choices[str_choices.index(args[0])], + is_instance=lambda x: x in choices, + str_from_instance=lambda instance: [str(instance)], + choices=str_choices, + ) + + @registry.define_rule( + lambda type_info: type_info.type_origin in (Union, _resolver.UnionType) + ) + def union_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + options = list(get_args(type_info.type)) + if type(None) in options: + # Move `None` types to the beginning. + # If we have `Optional[str]`, we want this to be parsed as + # `Union[NoneType, str]`. + options.remove(type(None)) + options.insert(0, type(None)) + + # General unions, eg Union[int, bool]. We'll try to convert these from left to + # right. + option_specs = [] + choices: tuple[str, ...] | None = () + nargs: int | Literal["*"] = 1 + first = True + for t in options: + option_spec = type_info.constructor_registry.get_spec( + PrimitiveTypeInfo.make( + raw_annotation=t, + parent_markers=type_info.markers, + source_registry=type_info.constructor_registry, + ) + ) + if option_spec.choices is None: + choices = None + elif choices is not None: + choices = choices + option_spec.choices + + option_specs.append(option_spec) + + if t is not type(None): + # Enforce that `nargs` is the same for all child types, except for + # NoneType. + if first: + nargs = option_spec.nargs + first = False + elif nargs != option_spec.nargs: + # Just be as general as possible if we see inconsistencies. + nargs = "*" + + metavar: str + metavar = _strings.join_union_metavars( + [option_spec.metavar for option_spec in option_specs], + ) + + def union_instantiator(strings: List[str]) -> Any: + errors = [] + for i, option_spec in enumerate(option_specs): + # Check choices. + if option_spec.choices is not None and any( + x not in option_spec.choices for x in strings + ): + errors.append( + f"{options[i]}: {strings} does not match choices {option_spec.choices}" + ) + continue + + # Try passing input into instantiator. + if len(strings) == option_spec.nargs or option_spec.nargs == "*": + try: + return option_spec.instance_from_str(strings) + except ValueError as e: + # Failed, try next instantiator. + errors.append(f"{options[i]}: {e.args[0]}") + else: + errors.append( + f"{options[i]}: input length {len(strings)} did not match expected" + f" argument count {option_spec.nargs}" + ) + raise ValueError( + f"no type in {options} could be instantiated from" + f" {strings}.\n\nGot errors: \n- " + "\n- ".join(errors) + ) + + def str_from_instance(instance: Any) -> List[str]: + for option_spec in option_specs: + if option_spec.is_instance(instance): + return option_spec.str_from_instance(instance) + assert False, f"could not match default value {instance} with any types in union {options}" + + return PrimitiveConstructorSpec( + nargs=nargs, + metavar=metavar, + instance_from_str=union_instantiator, + is_instance=lambda x: any(spec.is_instance(x) for spec in option_specs), + str_from_instance=str_from_instance, + choices=None if choices is None else tuple(set(choices)), + ) diff --git a/src/tyro/extras/__init__.py b/src/tyro/extras/__init__.py index 39878d58e..5341bc3df 100644 --- a/src/tyro/extras/__init__.py +++ b/src/tyro/extras/__init__.py @@ -1,6 +1,7 @@ """The :mod:`tyro.extras` submodule contains helpers that complement :func:`tyro.cli()`. -Compared to the core interface, APIs here are more likely to be changed or deprecated.""" +Compared to the core interface, APIs here are more likely to be changed or deprecated. +""" from .._argparse_formatter import set_accent_color as set_accent_color from .._cli import get_parser as get_parser diff --git a/tests/helptext_utils.py b/tests/helptext_utils.py index 73dd1fc99..e6d34899f 100644 --- a/tests/helptext_utils.py +++ b/tests/helptext_utils.py @@ -40,6 +40,10 @@ def get_helptext_with_checks( unformatted_helptext = parser.format_help() assert ( tyro._strings.strip_ansi_sequences(unformatted_helptext) == unformatted_helptext + ), ( + tyro._strings.strip_ansi_sequences(unformatted_helptext) + + "\n|\n" + + unformatted_helptext ) unformatted_usage = parser.format_usage() assert tyro._strings.strip_ansi_sequences(unformatted_usage) == unformatted_usage diff --git a/tests/test_conf.py b/tests/test_conf.py index 54dd9aa35..9d5068986 100644 --- a/tests/test_conf.py +++ b/tests/test_conf.py @@ -970,6 +970,16 @@ class A: tyro.cli(A, args=["--x", "1", "2", "3"]) +def test_append_dict_vague() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[dict] + + assert tyro.cli(A, args="--x 1 1 --x 2 2 --x 3 3".split(" ")) == A( + {"1": "1", "2": "2", "3": "3"} + ) + + def test_append_dict_with_default() -> None: """Append has no impact when a dictionary has a default value.""" @@ -1040,6 +1050,19 @@ class A: assert tyro.cli(A, args=[]) == A(x={}) +def test_append_nested_dict_double_with_default() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Dict[str, Dict[str, int]]] = dataclasses.field( + default_factory=dict + ) + + assert tyro.cli(A, args="--x 0 1 2 3 4 --x 4 5 6".split(" ")) == A( + x={"0": {"1": 2, "3": 4}, "4": {"5": 6}} + ) + assert tyro.cli(A, args=[]) == A(x={}) + + def test_duplicated_arg() -> None: # Loosely inspired by: https://github.com/brentyi/tyro/issues/49 @dataclasses.dataclass diff --git a/tests/test_custom_primitive.py b/tests/test_custom_primitive.py new file mode 100644 index 000000000..c9c9d1f6d --- /dev/null +++ b/tests/test_custom_primitive.py @@ -0,0 +1,44 @@ +import json +from typing import Any, Dict + +from typing_extensions import Annotated, get_args + +import tyro + +json_constructor_spec = tyro.constructors.PrimitiveConstructorSpec( + nargs=1, + metavar="JSON", + instance_from_str=lambda args: json.loads(args[0]), + is_instance=lambda x: isinstance(x, dict), + str_from_instance=lambda x: [json.dumps(x)], +) + + +def test_custom_primitive_registry(): + """Test that we can use a custom primitive registry to parse a custom type.""" + primitive_registry = tyro.constructors.PrimitiveConstructorRegistry() + + @primitive_registry.define_rule( + matcher_fn=lambda type_info: type_info.type_origin is dict + and get_args(type_info.type) == (str, Any) + ) + def json_dict_spec( + type_info: tyro.constructors.PrimitiveTypeInfo, + ) -> tyro.constructors.PrimitiveConstructorSpec: + del type_info # We won't use type_info. + return json_constructor_spec + + def main(x: Dict[str, Any]) -> Dict[str, Any]: + return x + + with primitive_registry: + assert tyro.cli(main, args=["--x", '{"a": 1}']) == {"a": 1} + + +def test_custom_primitive_annotated(): + """Test that we can use typing.Annotated to specify custom constructors.""" + + def main(x: Annotated[Dict[str, Any], json_constructor_spec]) -> Dict[str, Any]: + return x + + assert tyro.cli(main, args=["--x", '{"a": 1}']) == {"a": 1} diff --git a/tests/test_errors.py b/tests/test_errors.py index 3a7abcc7d..85c5a479f 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -52,6 +52,24 @@ def test_ambiguous_collection_4() -> None: assert "Unsupported type annotation for the field" not in e.value.args[0] +def test_ambiguous_collection_5() -> None: + @dataclasses.dataclass + class A: + x: Dict[List[int], str] + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(A, args=["--x", "0", "1"]) + + +def test_ambiguous_collection_6() -> None: + @dataclasses.dataclass + class A: + x: Dict[str, List[int]] + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(A, args=["--x", "0", "1"]) + + # Must be global. @dataclasses.dataclass class _CycleDataclass: diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 75c79d178..20e65909b 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -687,7 +687,7 @@ def main(x: os.PathLike) -> None: pass helptext = get_helptext_with_checks(main) - assert "--x PATH " in helptext + assert "--x PATHLIKE " in helptext def test_nested_bool() -> None: @@ -902,10 +902,16 @@ class A: """Help message.""" x: Annotated[ - Tuple[str, str], tyro.conf.arg(constructor=json.loads, metavar="JSON") + # NOTE: this style of JSON construction is/will be deprecated. + # Avoid. + Tuple[str, str], + tyro.conf.arg(constructor=json.loads, metavar="JSON"), ] = ("hello", "world") y: Annotated[ - Tuple[int, int], tyro.conf.arg(constructor=json.loads, metavar="JSON") + # NOTE: this style of JSON construction is/will be deprecated. + # Avoid. + Tuple[int, int], + tyro.conf.arg(constructor=json.loads, metavar="JSON"), ] = (3, 5) help = get_helptext_with_checks(A) @@ -918,7 +924,10 @@ class A: def test_optional_group() -> None: def f( x: Annotated[ - Tuple[str, str], tyro.conf.arg(constructor=json.loads, metavar="JSON") + # NOTE: this style of JSON construction is/will be deprecated. + # Avoid. + Tuple[str, str], + tyro.conf.arg(constructor=json.loads, metavar="JSON"), ] = ("hello", "world"), y: int = 3, ) -> int: diff --git a/tests/test_is_nested_type.py b/tests/test_is_nested_type.py deleted file mode 100644 index a586756f4..000000000 --- a/tests/test_is_nested_type.py +++ /dev/null @@ -1,52 +0,0 @@ -import dataclasses -import pathlib -from typing import Any, Dict, List, Tuple - -from tyro._fields import MISSING_NONPROP, is_nested_type - - -def test_is_nested_type_simple(): - assert not is_nested_type(int, MISSING_NONPROP) - assert not is_nested_type(bool, MISSING_NONPROP) - assert not is_nested_type(str, MISSING_NONPROP) - assert not is_nested_type(pathlib.Path, MISSING_NONPROP) - - -def test_is_nested_type_containers(): - assert not is_nested_type(List[int], MISSING_NONPROP) - assert not is_nested_type(List[bool], MISSING_NONPROP) - assert not is_nested_type(List[str], MISSING_NONPROP) - assert not is_nested_type(List[pathlib.Path], MISSING_NONPROP) - - -@dataclasses.dataclass -class Color: - r: int - g: int - b: int - - -def test_is_nested_type_actually_nested(): - assert is_nested_type(Color, Color(255, 0, 0)) - - -def test_is_nested_type_actually_nested_narrowing(): - assert is_nested_type(Any, Color(255, 0, 0)) - assert is_nested_type(object, Color(255, 0, 0)) - assert not is_nested_type(int, Color(255, 0, 0)) - - -def test_is_nested_type_actually_nested_in_container(): - assert is_nested_type(Tuple[Color, Color], MISSING_NONPROP) - assert is_nested_type(Tuple[object, ...], (Color(255, 0, 0),)) - assert is_nested_type(Tuple[Any, ...], (Color(255, 0, 0),)) - assert is_nested_type(tuple, (Color(255, 0, 0),)) - assert not is_nested_type(tuple, (1, 2, 3)) - assert is_nested_type(tuple, (1, Color(255, 0, 0), 3)) - assert is_nested_type(List[Any], [Color(255, 0, 0)]) - - -def test_nested_dict(): - assert is_nested_type(Dict[str, int], {"x": 5}) - assert is_nested_type(dict, {"x": 5}) - assert is_nested_type(Any, {"x": 5}) diff --git a/tests/test_is_struct_type.py b/tests/test_is_struct_type.py new file mode 100644 index 000000000..5b0736004 --- /dev/null +++ b/tests/test_is_struct_type.py @@ -0,0 +1,52 @@ +import dataclasses +import pathlib +from typing import Any, Dict, List, Tuple + +from tyro._fields import MISSING_NONPROP, is_struct_type + + +def test_is_struct_type_simple(): + assert not is_struct_type(int, MISSING_NONPROP) + assert not is_struct_type(bool, MISSING_NONPROP) + assert not is_struct_type(str, MISSING_NONPROP) + assert not is_struct_type(pathlib.Path, MISSING_NONPROP) + + +def test_is_struct_type_containers(): + assert not is_struct_type(List[int], MISSING_NONPROP) + assert not is_struct_type(List[bool], MISSING_NONPROP) + assert not is_struct_type(List[str], MISSING_NONPROP) + assert not is_struct_type(List[pathlib.Path], MISSING_NONPROP) + + +@dataclasses.dataclass +class Color: + r: int + g: int + b: int + + +def test_is_struct_type_actually_struct(): + assert is_struct_type(Color, Color(255, 0, 0)) + + +def test_is_struct_type_actually_struct_narrowing(): + assert is_struct_type(Any, Color(255, 0, 0)) + assert is_struct_type(object, Color(255, 0, 0)) + assert not is_struct_type(int, Color(255, 0, 0)) + + +def test_is_struct_type_actually_struct_in_container(): + assert is_struct_type(Tuple[Color, Color], MISSING_NONPROP) + assert is_struct_type(Tuple[object, ...], (Color(255, 0, 0),)) + assert is_struct_type(Tuple[Any, ...], (Color(255, 0, 0),)) + assert is_struct_type(tuple, (Color(255, 0, 0),)) + assert not is_struct_type(tuple, (1, 2, 3)) + assert is_struct_type(tuple, (1, Color(255, 0, 0), 3)) + assert is_struct_type(List[Any], [Color(255, 0, 0)]) + + +def test_struct_dict(): + assert is_struct_type(Dict[str, int], {"x": 5}) + assert is_struct_type(dict, {"x": 5}) + assert is_struct_type(Any, {"x": 5}) diff --git a/tests/test_new_style_annotations_min_py310.py b/tests/test_new_style_annotations_min_py310.py index f509d3cbc..3e2e90a55 100644 --- a/tests/test_new_style_annotations_min_py310.py +++ b/tests/test_new_style_annotations_min_py310.py @@ -46,14 +46,16 @@ def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: def test_super_nested() -> None: def main( - x: None - | list[ - tuple[ - None | int, - Literal[3, 4], - tuple[int, int] | tuple[str, str], + x: ( + None + | list[ + tuple[ + None | int, + Literal[3, 4], + tuple[int, int] | tuple[str, str], + ] ] - ] = None, + ) = None, ) -> Any: return x diff --git a/tests/test_new_style_annotations_unions.py b/tests/test_new_style_annotations_unions.py index 22b38ab72..7927af133 100644 --- a/tests/test_new_style_annotations_unions.py +++ b/tests/test_new_style_annotations_unions.py @@ -39,14 +39,16 @@ def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: def test_super_nested(): def main( - x: None - | list[ - tuple[ - None | int, - Literal[3, 4], - tuple[int, int] | tuple[str, str], + x: ( + None + | list[ + tuple[ + None | int, + Literal[3, 4], + tuple[int, int] | tuple[str, str], + ] ] - ] = None, + ) = None, ) -> Any: return x diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py index 13997ecbd..2572a7c64 100644 --- a/tests/test_py311_generated/test_conf_generated.py +++ b/tests/test_py311_generated/test_conf_generated.py @@ -967,6 +967,16 @@ class A: tyro.cli(A, args=["--x", "1", "2", "3"]) +def test_append_dict_vague() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[dict] + + assert tyro.cli(A, args="--x 1 1 --x 2 2 --x 3 3".split(" ")) == A( + {"1": "1", "2": "2", "3": "3"} + ) + + def test_append_dict_with_default() -> None: """Append has no impact when a dictionary has a default value.""" @@ -1037,6 +1047,19 @@ class A: assert tyro.cli(A, args=[]) == A(x={}) +def test_append_nested_dict_double_with_default() -> None: + @dataclasses.dataclass + class A: + x: tyro.conf.UseAppendAction[Dict[str, Dict[str, int]]] = dataclasses.field( + default_factory=dict + ) + + assert tyro.cli(A, args="--x 0 1 2 3 4 --x 4 5 6".split(" ")) == A( + x={"0": {"1": 2, "3": 4}, "4": {"5": 6}} + ) + assert tyro.cli(A, args=[]) == A(x={}) + + def test_duplicated_arg() -> None: # Loosely inspired by: https://github.com/brentyi/tyro/issues/49 @dataclasses.dataclass diff --git a/tests/test_py311_generated/test_custom_primitive_generated.py b/tests/test_py311_generated/test_custom_primitive_generated.py new file mode 100644 index 000000000..f36124df9 --- /dev/null +++ b/tests/test_py311_generated/test_custom_primitive_generated.py @@ -0,0 +1,42 @@ +import json +from typing import Annotated, Any, Dict, get_args + +import tyro + +json_constructor_spec = tyro.constructors.PrimitiveConstructorSpec( + nargs=1, + metavar="JSON", + instance_from_str=lambda args: json.loads(args[0]), + is_instance=lambda x: isinstance(x, dict), + str_from_instance=lambda x: [json.dumps(x)], +) + + +def test_custom_primitive_registry(): + """Test that we can use a custom primitive registry to parse a custom type.""" + primitive_registry = tyro.constructors.PrimitiveConstructorRegistry() + + @primitive_registry.define_rule( + matcher_fn=lambda type_info: type_info.type_origin is dict + and get_args(type_info.type) == (str, Any) + ) + def json_dict_spec( + type_info: tyro.constructors.PrimitiveTypeInfo, + ) -> tyro.constructors.PrimitiveConstructorSpec: + del type_info # We won't use type_info. + return json_constructor_spec + + def main(x: Dict[str, Any]) -> Dict[str, Any]: + return x + + with primitive_registry: + assert tyro.cli(main, args=["--x", '{"a": 1}']) == {"a": 1} + + +def test_custom_primitive_annotated(): + """Test that we can use typing.Annotated to specify custom constructors.""" + + def main(x: Annotated[Dict[str, Any], json_constructor_spec]) -> Dict[str, Any]: + return x + + assert tyro.cli(main, args=["--x", '{"a": 1}']) == {"a": 1} diff --git a/tests/test_py311_generated/test_errors_generated.py b/tests/test_py311_generated/test_errors_generated.py index cd5f7b866..398131ff4 100644 --- a/tests/test_py311_generated/test_errors_generated.py +++ b/tests/test_py311_generated/test_errors_generated.py @@ -51,6 +51,24 @@ def test_ambiguous_collection_4() -> None: assert "Unsupported type annotation for the field" not in e.value.args[0] +def test_ambiguous_collection_5() -> None: + @dataclasses.dataclass + class A: + x: Dict[List[int], str] + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(A, args=["--x", "0", "1"]) + + +def test_ambiguous_collection_6() -> None: + @dataclasses.dataclass + class A: + x: Dict[str, List[int]] + + with pytest.raises(tyro.UnsupportedTypeAnnotationError): + tyro.cli(A, args=["--x", "0", "1"]) + + # Must be global. @dataclasses.dataclass class _CycleDataclass: diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py index 075fdac46..4039e42eb 100644 --- a/tests/test_py311_generated/test_helptext_generated.py +++ b/tests/test_py311_generated/test_helptext_generated.py @@ -688,7 +688,7 @@ def main(x: os.PathLike) -> None: pass helptext = get_helptext_with_checks(main) - assert "--x PATH " in helptext + assert "--x PATHLIKE " in helptext def test_nested_bool() -> None: @@ -903,10 +903,16 @@ class A: """Help message.""" x: Annotated[ - Tuple[str, str], tyro.conf.arg(constructor=json.loads, metavar="JSON") + # NOTE: this style of JSON construction is/will be deprecated. + # Avoid. + Tuple[str, str], + tyro.conf.arg(constructor=json.loads, metavar="JSON"), ] = ("hello", "world") y: Annotated[ - Tuple[int, int], tyro.conf.arg(constructor=json.loads, metavar="JSON") + # NOTE: this style of JSON construction is/will be deprecated. + # Avoid. + Tuple[int, int], + tyro.conf.arg(constructor=json.loads, metavar="JSON"), ] = (3, 5) help = get_helptext_with_checks(A) @@ -919,7 +925,10 @@ class A: def test_optional_group() -> None: def f( x: Annotated[ - Tuple[str, str], tyro.conf.arg(constructor=json.loads, metavar="JSON") + # NOTE: this style of JSON construction is/will be deprecated. + # Avoid. + Tuple[str, str], + tyro.conf.arg(constructor=json.loads, metavar="JSON"), ] = ("hello", "world"), y: int = 3, ) -> int: diff --git a/tests/test_py311_generated/test_is_nested_type_generated.py b/tests/test_py311_generated/test_is_nested_type_generated.py index a586756f4..37bedb6c6 100644 --- a/tests/test_py311_generated/test_is_nested_type_generated.py +++ b/tests/test_py311_generated/test_is_nested_type_generated.py @@ -2,21 +2,21 @@ import pathlib from typing import Any, Dict, List, Tuple -from tyro._fields import MISSING_NONPROP, is_nested_type +from tyro._fields import MISSING_NONPROP, is_struct_type def test_is_nested_type_simple(): - assert not is_nested_type(int, MISSING_NONPROP) - assert not is_nested_type(bool, MISSING_NONPROP) - assert not is_nested_type(str, MISSING_NONPROP) - assert not is_nested_type(pathlib.Path, MISSING_NONPROP) + assert not is_struct_type(int, MISSING_NONPROP) + assert not is_struct_type(bool, MISSING_NONPROP) + assert not is_struct_type(str, MISSING_NONPROP) + assert not is_struct_type(pathlib.Path, MISSING_NONPROP) def test_is_nested_type_containers(): - assert not is_nested_type(List[int], MISSING_NONPROP) - assert not is_nested_type(List[bool], MISSING_NONPROP) - assert not is_nested_type(List[str], MISSING_NONPROP) - assert not is_nested_type(List[pathlib.Path], MISSING_NONPROP) + assert not is_struct_type(List[int], MISSING_NONPROP) + assert not is_struct_type(List[bool], MISSING_NONPROP) + assert not is_struct_type(List[str], MISSING_NONPROP) + assert not is_struct_type(List[pathlib.Path], MISSING_NONPROP) @dataclasses.dataclass @@ -27,26 +27,26 @@ class Color: def test_is_nested_type_actually_nested(): - assert is_nested_type(Color, Color(255, 0, 0)) + assert is_struct_type(Color, Color(255, 0, 0)) def test_is_nested_type_actually_nested_narrowing(): - assert is_nested_type(Any, Color(255, 0, 0)) - assert is_nested_type(object, Color(255, 0, 0)) - assert not is_nested_type(int, Color(255, 0, 0)) + assert is_struct_type(Any, Color(255, 0, 0)) + assert is_struct_type(object, Color(255, 0, 0)) + assert not is_struct_type(int, Color(255, 0, 0)) def test_is_nested_type_actually_nested_in_container(): - assert is_nested_type(Tuple[Color, Color], MISSING_NONPROP) - assert is_nested_type(Tuple[object, ...], (Color(255, 0, 0),)) - assert is_nested_type(Tuple[Any, ...], (Color(255, 0, 0),)) - assert is_nested_type(tuple, (Color(255, 0, 0),)) - assert not is_nested_type(tuple, (1, 2, 3)) - assert is_nested_type(tuple, (1, Color(255, 0, 0), 3)) - assert is_nested_type(List[Any], [Color(255, 0, 0)]) + assert is_struct_type(Tuple[Color, Color], MISSING_NONPROP) + assert is_struct_type(Tuple[object, ...], (Color(255, 0, 0),)) + assert is_struct_type(Tuple[Any, ...], (Color(255, 0, 0),)) + assert is_struct_type(tuple, (Color(255, 0, 0),)) + assert not is_struct_type(tuple, (1, 2, 3)) + assert is_struct_type(tuple, (1, Color(255, 0, 0), 3)) + assert is_struct_type(List[Any], [Color(255, 0, 0)]) def test_nested_dict(): - assert is_nested_type(Dict[str, int], {"x": 5}) - assert is_nested_type(dict, {"x": 5}) - assert is_nested_type(Any, {"x": 5}) + assert is_struct_type(Dict[str, int], {"x": 5}) + assert is_struct_type(dict, {"x": 5}) + assert is_struct_type(Any, {"x": 5}) diff --git a/tests/test_py311_generated/test_is_struct_type_generated.py b/tests/test_py311_generated/test_is_struct_type_generated.py new file mode 100644 index 000000000..5b0736004 --- /dev/null +++ b/tests/test_py311_generated/test_is_struct_type_generated.py @@ -0,0 +1,52 @@ +import dataclasses +import pathlib +from typing import Any, Dict, List, Tuple + +from tyro._fields import MISSING_NONPROP, is_struct_type + + +def test_is_struct_type_simple(): + assert not is_struct_type(int, MISSING_NONPROP) + assert not is_struct_type(bool, MISSING_NONPROP) + assert not is_struct_type(str, MISSING_NONPROP) + assert not is_struct_type(pathlib.Path, MISSING_NONPROP) + + +def test_is_struct_type_containers(): + assert not is_struct_type(List[int], MISSING_NONPROP) + assert not is_struct_type(List[bool], MISSING_NONPROP) + assert not is_struct_type(List[str], MISSING_NONPROP) + assert not is_struct_type(List[pathlib.Path], MISSING_NONPROP) + + +@dataclasses.dataclass +class Color: + r: int + g: int + b: int + + +def test_is_struct_type_actually_struct(): + assert is_struct_type(Color, Color(255, 0, 0)) + + +def test_is_struct_type_actually_struct_narrowing(): + assert is_struct_type(Any, Color(255, 0, 0)) + assert is_struct_type(object, Color(255, 0, 0)) + assert not is_struct_type(int, Color(255, 0, 0)) + + +def test_is_struct_type_actually_struct_in_container(): + assert is_struct_type(Tuple[Color, Color], MISSING_NONPROP) + assert is_struct_type(Tuple[object, ...], (Color(255, 0, 0),)) + assert is_struct_type(Tuple[Any, ...], (Color(255, 0, 0),)) + assert is_struct_type(tuple, (Color(255, 0, 0),)) + assert not is_struct_type(tuple, (1, 2, 3)) + assert is_struct_type(tuple, (1, Color(255, 0, 0), 3)) + assert is_struct_type(List[Any], [Color(255, 0, 0)]) + + +def test_struct_dict(): + assert is_struct_type(Dict[str, int], {"x": 5}) + assert is_struct_type(dict, {"x": 5}) + assert is_struct_type(Any, {"x": 5}) diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py index f509d3cbc..3e2e90a55 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py @@ -46,14 +46,16 @@ def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: def test_super_nested() -> None: def main( - x: None - | list[ - tuple[ - None | int, - Literal[3, 4], - tuple[int, int] | tuple[str, str], + x: ( + None + | list[ + tuple[ + None | int, + Literal[3, 4], + tuple[int, int] | tuple[str, str], + ] ] - ] = None, + ) = None, ) -> Any: return x diff --git a/tests/test_py311_generated/test_new_style_annotations_unions_generated.py b/tests/test_py311_generated/test_new_style_annotations_unions_generated.py index ccb1920ea..19cf034fe 100644 --- a/tests/test_py311_generated/test_new_style_annotations_unions_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_unions_generated.py @@ -38,14 +38,16 @@ def main(x: Literal[1, 2] | Literal[3, 4, 5] | str) -> int | str: def test_super_nested(): def main( - x: None - | list[ - tuple[ - None | int, - Literal[3, 4], - tuple[int, int] | tuple[str, str], + x: ( + None + | list[ + tuple[ + None | int, + Literal[3, 4], + tuple[int, int] | tuple[str, str], + ] ] - ] = None, + ) = None, ) -> Any: return x diff --git a/tests/test_py311_generated/test_pydantic_generated.py b/tests/test_py311_generated/test_pydantic_generated.py index c0892be15..02320270a 100644 --- a/tests/test_py311_generated/test_pydantic_generated.py +++ b/tests/test_py311_generated/test_pydantic_generated.py @@ -3,6 +3,7 @@ import contextlib import io import pathlib +import sys from typing import Annotated, cast import pytest @@ -75,7 +76,12 @@ class ManyTypesB(ManyTypesA): helptext = get_helptext_with_checks(ManyTypesB) assert "--s" not in helptext assert "--i" in helptext - assert "This is a docstring" in helptext + + # This doesn't work when combining older versions of Python with older + # versions of pydantic. The root cause may be in the docstring_parser + # dependency. + if sys.version_info >= (3, 10): + assert "This is a docstring" in helptext def test_pydantic_helptext() -> None: From 6552337995085b3de588c0dc82a3ffbf40591bde Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sun, 3 Nov 2024 23:55:20 -0800 Subject: [PATCH 469/491] Start struct handling rewrite (#191) * Custom constructors + container type resolution cleanup, docs * Simpler rule API * Global namespace cleanup * fix for older Python versions * Ignore docs for pyright * Bump typing-extensions for typing.ReadOnly * Docs sync * Fix 3.7 typing-extension dep * Fix issubclass error for new pydantic * Add pydantic + generics test * Start refactor * More refactor * Passing tests * pyright * ruff * exports * fix frozendict for Python 3.8 * dict fix * mypy * coverage --- .github/workflows/pyright.yml | 1 - README.md | 4 +- docs/requirements.txt | 4 +- .../04_additional/11_custom_constructors.rst | 71 +- .../12_custom_constructors_registry.rst | 92 ++ .../{12_aliases.rst => 17_aliases.rst} | 28 +- .../04_additional/11_custom_constructors.py | 58 +- .../12_custom_constructors_registry.py | 51 + .../{12_aliases.py => 17_aliases.py} | 0 pyproject.toml | 5 +- src/tyro/__init__.py | 8 +- src/tyro/_arguments.py | 36 +- src/tyro/_calling.py | 8 +- src/tyro/_cli.py | 5 +- src/tyro/_fields.py | 1025 ++--------------- src/tyro/_parsers.py | 17 +- src/tyro/_resolver.py | 52 +- src/tyro/_singleton.py | 59 + src/tyro/_subcommand_matching.py | 25 +- src/tyro/conf/_confstruct.py | 26 +- src/tyro/constructors/__init__.py | 10 +- src/tyro/constructors/_primitive_spec.py | 367 +++--- src/tyro/constructors/_registry.py | 122 ++ src/tyro/constructors/_struct_spec.py | 655 +++++++++++ src/tyro/extras/_serialization.py | 9 +- tests/test_custom_primitive.py | 16 +- tests/test_errors.py | 39 +- tests/test_helptext.py | 2 +- tests/test_is_struct_type.py | 3 +- tests/test_nested_in_containers.py | 9 +- tests/test_new_style_annotations_min_py312.py | 3 +- .../test_custom_primitive_generated.py | 16 +- .../test_errors_generated.py | 39 +- .../test_errors_new_annotations_generated.py | 33 - .../test_helptext_generated.py | 2 +- .../test_is_nested_type_generated.py | 52 - .../test_is_struct_type_generated.py | 3 +- .../test_nested_in_containers_generated.py | 9 +- ...w_style_annotations_min_py312_generated.py | 3 +- .../test_pydantic_generic_generated.py | 18 + tests/test_pydantic_generic.py | 18 + 41 files changed, 1516 insertions(+), 1487 deletions(-) create mode 100644 docs/source/examples/04_additional/12_custom_constructors_registry.rst rename docs/source/examples/04_additional/{12_aliases.rst => 17_aliases.rst} (61%) create mode 100644 examples/04_additional/12_custom_constructors_registry.py rename examples/04_additional/{12_aliases.py => 17_aliases.py} (100%) create mode 100644 src/tyro/constructors/_registry.py create mode 100644 src/tyro/constructors/_struct_spec.py delete mode 100644 tests/test_py311_generated/test_errors_new_annotations_generated.py delete mode 100644 tests/test_py311_generated/test_is_nested_type_generated.py create mode 100644 tests/test_py311_generated/test_pydantic_generic_generated.py create mode 100644 tests/test_pydantic_generic.py diff --git a/.github/workflows/pyright.yml b/.github/workflows/pyright.yml index 7da8fd2ce..9c09b5d06 100644 --- a/.github/workflows/pyright.yml +++ b/.github/workflows/pyright.yml @@ -23,7 +23,6 @@ jobs: run: | pip install uv uv pip install --system -e ".[dev]" - uv pip install --system -r docs/requirements.txt - name: Run pyright run: | pyright . diff --git a/README.md b/README.md index ae307195a..2efb09f4e 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,10 @@

- build + mypy pyright - ruff + codecov diff --git a/docs/requirements.txt b/docs/requirements.txt index 241369d0c..5e86e9dc9 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ -sphinx==7.2.6 -furo==2023.9.10 +sphinx==8.0.2 +furo==2024.8.6 docutils==0.20.1 sphinx-autoapi==3.0.0 m2r2==0.3.3.post2 diff --git a/docs/source/examples/04_additional/11_custom_constructors.rst b/docs/source/examples/04_additional/11_custom_constructors.rst index 27af528ee..24cea3871 100644 --- a/docs/source/examples/04_additional/11_custom_constructors.rst +++ b/docs/source/examples/04_additional/11_custom_constructors.rst @@ -4,59 +4,32 @@ Custom Constructors ========================================== -For additional flexibility, :func:`tyro.conf.arg()` accepts a -:code:`constructor` argument, which makes it easier to load complex objects. - -.. warning:: - Custom constructors are permitted wherever :func:`tyro.conf.arg()` is. This - annotation corresponds to a single argument, so we expect that it is placed - placed at the root of the field corresponding to that argument. For - example: - - .. code-block:: python - - # Great! - x: Annotated[int, tyro.conf.arg(custom_constructor=...)] - - However, nesting :func:`tyro.conf.arg()` within other types is generally - not supported. For example: - - .. code-block:: python - - # Not supported. - x: tuple[Annotated[int, tyro.conf.arg(custom_constructor=...)], ...] - - This example can be rewritten with a custom constructor that directly - returns a tuple of integers: - - .. code-block:: python - - # Workaround for above. - x: Annotated[tuple[int, ...], tyro.conf.arg(custom_constructor=...)] +For additional flexibility, :module:`tyro.constructors` exposes +tyro's API for defining behavior for different types. This is the same +API that tyro relies on for the built-in types. .. code-block:: python :linenos: - import json as json_ + import json from typing_extensions import Annotated import tyro - - def dict_json_constructor(json: str) -> dict: - """Construct a dictionary from a JSON string. Raises a ValueError if the result is - not a dictionary.""" - out = json_.loads(json) - if not isinstance(out, dict): - raise ValueError(f"{json} is not a dictionary!") - return out - - # A dictionary type, but `tyro` will expect a JSON string from the CLI. - JsonDict = Annotated[dict, tyro.conf.arg(constructor=dict_json_constructor)] + JsonDict = Annotated[ + dict, + tyro.constructors.PrimitiveConstructorSpec( + nargs=1, + metavar="JSON", + instance_from_str=lambda args: json.loads(args[0]), + is_instance=lambda instance: isinstance(instance, dict), + str_from_instance=lambda instance: [json.dumps(instance)], + ), + ] def main( @@ -88,6 +61,22 @@ For additional flexibility, :func:`tyro.conf.arg()` accepts a ------------ +.. raw:: html + + python 04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' + +.. program-output:: python ../../examples/04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' + +------------ + +.. raw:: html + + python 04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' + +.. program-output:: python ../../examples/04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' + +------------ + .. raw:: html python 04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' diff --git a/docs/source/examples/04_additional/12_custom_constructors_registry.rst b/docs/source/examples/04_additional/12_custom_constructors_registry.rst new file mode 100644 index 000000000..942ec2b98 --- /dev/null +++ b/docs/source/examples/04_additional/12_custom_constructors_registry.rst @@ -0,0 +1,92 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Custom Constructors (Registry) +========================================== + +For additional flexibility, :module:`tyro.constructors` exposes +tyro's API for defining behavior for different types. This is the same +API that tyro relies on for the built-in types. + + +.. code-block:: python + :linenos: + + + import json + from typing import Any + + import tyro + + custom_registry = tyro.constructors.PrimitiveConstructorRegistry() + + + @custom_registry.define_rule + def _( + type_info: tyro.constructors.PrimitiveTypeInfo, + ) -> tyro.constructors.PrimitiveConstructorSpec | None: + # We return `None` if the rule does not apply. + if type_info.type != dict[str, Any]: + return None + + # If the rule applies, we return the constructor spec. + return tyro.constructors.PrimitiveConstructorSpec( + nargs=1, + metavar="JSON", + instance_from_str=lambda args: json.loads(args[0]), + is_instance=lambda instance: isinstance(instance, dict), + str_from_instance=lambda instance: [json.dumps(instance)], + ) + + + def main( + dict1: dict[str, Any], + dict2: dict[str, Any] = {"default": None}, + ) -> None: + print(f"{dict1=}") + print(f"{dict2=}") + + + if __name__ == "__main__": + with custom_registry: + tyro.cli(main) + +------------ + +.. raw:: html + + python 04_additional/12_custom_constructors_registry.py --help + +.. program-output:: python ../../examples/04_additional/12_custom_constructors_registry.py --help + +------------ + +.. raw:: html + + python 04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' + +.. program-output:: python ../../examples/04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' + +------------ + +.. raw:: html + + python 04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' + +.. program-output:: python ../../examples/04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' + +------------ + +.. raw:: html + + python 04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' + +.. program-output:: python ../../examples/04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' + +------------ + +.. raw:: html + + python 04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' + +.. program-output:: python ../../examples/04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' diff --git a/docs/source/examples/04_additional/12_aliases.rst b/docs/source/examples/04_additional/17_aliases.rst similarity index 61% rename from docs/source/examples/04_additional/12_aliases.rst rename to docs/source/examples/04_additional/17_aliases.rst index 1a8c5aa32..e53f28421 100644 --- a/docs/source/examples/04_additional/12_aliases.rst +++ b/docs/source/examples/04_additional/17_aliases.rst @@ -43,54 +43,54 @@ Argument Aliases .. raw:: html - python 04_additional/12_aliases.py --help + python 04_additional/17_aliases.py --help -.. program-output:: python ../../examples/04_additional/12_aliases.py --help +.. program-output:: python ../../examples/04_additional/17_aliases.py --help ------------ .. raw:: html - python 04_additional/12_aliases.py commit --help + python 04_additional/17_aliases.py commit --help -.. program-output:: python ../../examples/04_additional/12_aliases.py commit --help +.. program-output:: python ../../examples/04_additional/17_aliases.py commit --help ------------ .. raw:: html - python 04_additional/12_aliases.py commit --message hello --all + python 04_additional/17_aliases.py commit --message hello --all -.. program-output:: python ../../examples/04_additional/12_aliases.py commit --message hello --all +.. program-output:: python ../../examples/04_additional/17_aliases.py commit --message hello --all ------------ .. raw:: html - python 04_additional/12_aliases.py commit -m hello -a + python 04_additional/17_aliases.py commit -m hello -a -.. program-output:: python ../../examples/04_additional/12_aliases.py commit -m hello -a +.. program-output:: python ../../examples/04_additional/17_aliases.py commit -m hello -a ------------ .. raw:: html - python 04_additional/12_aliases.py checkout --help + python 04_additional/17_aliases.py checkout --help -.. program-output:: python ../../examples/04_additional/12_aliases.py checkout --help +.. program-output:: python ../../examples/04_additional/17_aliases.py checkout --help ------------ .. raw:: html - python 04_additional/12_aliases.py checkout --branch main + python 04_additional/17_aliases.py checkout --branch main -.. program-output:: python ../../examples/04_additional/12_aliases.py checkout --branch main +.. program-output:: python ../../examples/04_additional/17_aliases.py checkout --branch main ------------ .. raw:: html - python 04_additional/12_aliases.py checkout -b main + python 04_additional/17_aliases.py checkout -b main -.. program-output:: python ../../examples/04_additional/12_aliases.py checkout -b main +.. program-output:: python ../../examples/04_additional/17_aliases.py checkout -b main diff --git a/examples/04_additional/11_custom_constructors.py b/examples/04_additional/11_custom_constructors.py index 2c314b9d2..4d8cff568 100644 --- a/examples/04_additional/11_custom_constructors.py +++ b/examples/04_additional/11_custom_constructors.py @@ -1,60 +1,34 @@ """Custom Constructors -For additional flexibility, :func:`tyro.conf.arg()` accepts a -:code:`constructor` argument, which makes it easier to load complex objects. - -.. warning:: - Custom constructors are permitted wherever :func:`tyro.conf.arg()` is. This - annotation corresponds to a single argument, so we expect that it is placed - placed at the root of the field corresponding to that argument. For - example: - - .. code-block:: python - - # Great! - x: Annotated[int, tyro.conf.arg(custom_constructor=...)] - - However, nesting :func:`tyro.conf.arg()` within other types is generally - not supported. For example: - - .. code-block:: python - - # Not supported. - x: tuple[Annotated[int, tyro.conf.arg(custom_constructor=...)], ...] - - This example can be rewritten with a custom constructor that directly - returns a tuple of integers: - - .. code-block:: python - - # Workaround for above. - x: Annotated[tuple[int, ...], tyro.conf.arg(custom_constructor=...)] - +For additional flexibility, :module:`tyro.constructors` exposes +tyro's API for defining behavior for different types. This is the same +API that tyro relies on for the built-in types. Usage: `python ./10_custom_constructors.py --help` +`python ./10_custom_constructors.py --dict1.json '{"hello": "world"}'` `python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}"` +`python ./10_custom_constructors.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}'` `python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}" --dict2.json "{\"hello\": \"world\"}"` """ -import json as json_ +import json from typing_extensions import Annotated import tyro - -def dict_json_constructor(json: str) -> dict: - """Construct a dictionary from a JSON string. Raises a ValueError if the result is - not a dictionary.""" - out = json_.loads(json) - if not isinstance(out, dict): - raise ValueError(f"{json} is not a dictionary!") - return out - - # A dictionary type, but `tyro` will expect a JSON string from the CLI. -JsonDict = Annotated[dict, tyro.conf.arg(constructor=dict_json_constructor)] +JsonDict = Annotated[ + dict, + tyro.constructors.PrimitiveConstructorSpec( + nargs=1, + metavar="JSON", + instance_from_str=lambda args: json.loads(args[0]), + is_instance=lambda instance: isinstance(instance, dict), + str_from_instance=lambda instance: [json.dumps(instance)], + ), +] def main( diff --git a/examples/04_additional/12_custom_constructors_registry.py b/examples/04_additional/12_custom_constructors_registry.py new file mode 100644 index 000000000..88eb13828 --- /dev/null +++ b/examples/04_additional/12_custom_constructors_registry.py @@ -0,0 +1,51 @@ +"""Custom Constructors (Registry) + +For additional flexibility, :module:`tyro.constructors` exposes +tyro's API for defining behavior for different types. This is the same +API that tyro relies on for the built-in types. + +Usage: +`python ./10_custom_constructors.py --help` +`python ./10_custom_constructors.py --dict1.json '{"hello": "world"}'` +`python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}"` +`python ./10_custom_constructors.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}'` +`python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}" --dict2.json "{\"hello\": \"world\"}"` +""" + +import json +from typing import Any + +import tyro + +custom_registry = tyro.constructors.ConstructorRegistry() + + +@custom_registry.primitive_rule +def _( + type_info: tyro.constructors.PrimitiveTypeInfo, +) -> tyro.constructors.PrimitiveConstructorSpec | None: + # We return `None` if the rule does not apply. + if type_info.type != dict[str, Any]: + return None + + # If the rule applies, we return the constructor spec. + return tyro.constructors.PrimitiveConstructorSpec( + nargs=1, + metavar="JSON", + instance_from_str=lambda args: json.loads(args[0]), + is_instance=lambda instance: isinstance(instance, dict), + str_from_instance=lambda instance: [json.dumps(instance)], + ) + + +def main( + dict1: dict[str, Any], + dict2: dict[str, Any] = {"default": None}, +) -> None: + print(f"{dict1=}") + print(f"{dict2=}") + + +if __name__ == "__main__": + with custom_registry: + tyro.cli(main) diff --git a/examples/04_additional/12_aliases.py b/examples/04_additional/17_aliases.py similarity index 100% rename from examples/04_additional/12_aliases.py rename to examples/04_additional/17_aliases.py diff --git a/pyproject.toml b/pyproject.toml index f1978a113..fe3ef57aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,8 @@ classifiers = [ ] dependencies = [ "docstring-parser>=0.16", - "typing-extensions>=4.7.0", + "typing-extensions>=4.9.0; python_version>='3.8'", + "typing-extensions>=4.7.0; python_version<'3.8'", "backports.cached-property>=1.0.2; python_version<'3.8'", "colorama>=0.4.0; platform_system=='Windows'", "rich>=11.1.0", @@ -135,4 +136,4 @@ extend-exclude = ["**/_argparse.py"] [tool.pyright] pythonVersion = "3.12" -ignore = ["**/_argparse.py"] +ignore = ["**/_argparse.py", "./docs/**/*"] diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py index 49f5e40f2..7be6652a5 100644 --- a/src/tyro/__init__.py +++ b/src/tyro/__init__.py @@ -4,14 +4,14 @@ from . import constructors as constructors from . import extras as extras from ._cli import cli as cli -from ._fields import MISSING as MISSING -from .constructors._primitive_spec import ( - UnsupportedTypeAnnotationError as UnsupportedTypeAnnotationError, -) +from ._singleton import MISSING as MISSING # Deprecated interface. if not TYPE_CHECKING: from ._deprecated import * # noqa + from .constructors._primitive_spec import ( + UnsupportedTypeAnnotationError as UnsupportedTypeAnnotationError, + ) # TODO: this should be synchronized automatically with the pyproject.toml. diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index 44e36dcdb..a9f6489d6 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -24,10 +24,10 @@ from typing_extensions import get_origin from . import _argparse as argparse -from . import _fields, _strings +from . import _fields, _singleton, _strings from .conf import _markers -from .constructors._primitive_spec import ( - PrimitiveConstructorRegistry, +from .constructors import ( + ConstructorRegistry, PrimitiveTypeInfo, UnsupportedTypeAnnotationError, ) @@ -129,7 +129,7 @@ def add_argument( # the field default to a string format, then back to the desired type. action = kwargs.get("action", None) if action not in {"append", "count"}: - kwargs["default"] = _fields.MISSING_NONPROP + kwargs["default"] = _singleton.MISSING_NONPROP elif action in {BooleanOptionalAction, "count"}: pass else: @@ -237,7 +237,7 @@ def _rule_handle_boolean_flags( return if ( - arg.field.default in _fields.MISSING_SINGLETONS + arg.field.default in _singleton.MISSING_SINGLETONS or arg.field.is_positional() or _markers.FlagConversionOff in arg.field.markers or _markers.Fixed in arg.field.markers @@ -275,7 +275,7 @@ def _rule_apply_primitive_specs( lowered.instance_from_str = None lowered.metavar = "{fixed}" lowered.required = False - lowered.default = _fields.MISSING_PROP + lowered.default = _singleton.MISSING_PROP return if lowered.instance_from_str is not None: return @@ -284,16 +284,14 @@ def _rule_apply_primitive_specs( if arg.field.primitive_spec is not None: spec = arg.field.primitive_spec else: - registry = PrimitiveConstructorRegistry._get_active_registry() - spec = registry.get_spec( + spec = ConstructorRegistry._get_active_registry().get_primitive_spec( PrimitiveTypeInfo.make( cast(type, arg.field.type_or_callable), arg.field.markers, - source_registry=registry, ) ) except UnsupportedTypeAnnotationError as e: - if arg.field.default in _fields.MISSING_SINGLETONS: + if arg.field.default in _singleton.MISSING_SINGLETONS: field_name = _strings.make_field_name( [arg.extern_prefix, arg.field.intern_name] ) @@ -313,19 +311,19 @@ def _rule_apply_primitive_specs( # available. lowered.metavar = "{fixed}" lowered.required = False - lowered.default = _fields.MISSING_PROP + lowered.default = _singleton.MISSING_PROP return # Mark lowered as required if a default is missing. if ( - arg.field.default in _fields.MISSING_SINGLETONS + arg.field.default in _singleton.MISSING_SINGLETONS and _markers._OPTIONAL_GROUP not in arg.field.markers ): lowered.default = None lowered.required = True elif ( - arg.field.default is not _fields.EXCLUDE_FROM_CALL - and arg.field.default not in _fields.MISSING_SINGLETONS + arg.field.default is not _singleton.EXCLUDE_FROM_CALL + and arg.field.default not in _singleton.MISSING_SINGLETONS ): # Set default. lowered.default = spec.str_from_instance(arg.field.default) @@ -346,7 +344,7 @@ def append_instantiator(x: list[list[str]]) -> Any: # Instantiate initial output. out = ( arg.field.default - if arg.field.default not in _fields.MISSING_SINGLETONS + if arg.field.default not in _singleton.MISSING_SINGLETONS else None ) if out is None: @@ -454,7 +452,7 @@ def _rule_generate_helptext( default = lowered.default if lowered.is_fixed() or lowered.action == "append": # Cases where we'll be missing the lowered default. Use field default instead. - assert default in _fields.MISSING_SINGLETONS or default is None + assert default in _singleton.MISSING_SINGLETONS or default is None default = arg.field.default # Get the default value label. @@ -492,18 +490,18 @@ def _rule_generate_helptext( # Repeatable argument. behavior_hint = "(repeatable)" elif lowered.action == "append" and ( - default in _fields.MISSING_SINGLETONS or len(cast(tuple, default)) == 0 + default in _singleton.MISSING_SINGLETONS or len(cast(tuple, default)) == 0 ): behavior_hint = "(repeatable)" elif lowered.action == "append" and len(cast(tuple, default)) > 0: assert default is not None # Just for type checker. behavior_hint = f"(repeatable, appends to: {default_label})" - elif arg.field.default is _fields.EXCLUDE_FROM_CALL: + elif arg.field.default is _singleton.EXCLUDE_FROM_CALL: # ^important to use arg.field.default and not the stringified default variable. behavior_hint = "(unset by default)" elif ( _markers._OPTIONAL_GROUP in arg.field.markers - and default in _fields.MISSING_SINGLETONS + and default in _singleton.MISSING_SINGLETONS ): # Argument in an optional group, but with no default. This is typically used # when general (non-argument, non-dataclass) object arguments are given a diff --git a/src/tyro/_calling.py b/src/tyro/_calling.py index 76c16d5a3..b714c20ec 100644 --- a/src/tyro/_calling.py +++ b/src/tyro/_calling.py @@ -10,7 +10,7 @@ from typing_extensions import get_args -from . import _arguments, _fields, _parsers, _resolver, _strings +from . import _arguments, _fields, _parsers, _resolver, _singleton, _strings from .conf import _markers @@ -29,7 +29,7 @@ class InstantiationError(Exception): def callable_with_args( f: Callable[..., T], parser_definition: _parsers.ParserSpecification, - default_instance: Union[T, _fields.NonpropagatingMissingType], + default_instance: Union[T, _singleton.NonpropagatingMissingType], value_from_prefixed_field_name: Dict[str, Any], field_name_prefix: str, ) -> Tuple[Callable[[], T], Set[str]]: @@ -167,7 +167,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: ( field.default if type(field.default) is chosen_f - else _fields.MISSING_NONPROP + else _singleton.MISSING_NONPROP ), value_from_prefixed_field_name, field_name_prefix=prefixed_field_name, @@ -176,7 +176,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: del get_value consumed_keywords |= consumed_keywords_child - if value is _fields.EXCLUDE_FROM_CALL: + if value is _singleton.EXCLUDE_FROM_CALL: continue if _markers._UnpackArgsCall in field.markers: diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index 6703d65b1..d24ec32e8 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -18,6 +18,7 @@ _calling, _fields, _parsers, + _singleton, _strings, _unsafe_cache, conf, @@ -316,8 +317,8 @@ def _cli_impl( # one or many arguments, depending on various factors). # # This could be revisited. - default_instance_internal: _fields.NonpropagatingMissingType | OutT = ( - _fields.MISSING_NONPROP if default is None else default + default_instance_internal: _singleton.NonpropagatingMissingType | OutT = ( + _singleton.MISSING_NONPROP if default is None else default ) # We wrap our type with a dummy dataclass if it can't be treated as a nested type. diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index e4f2789ed..b1f50680d 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -3,53 +3,28 @@ from __future__ import annotations -import builtins -import collections -import collections.abc import contextlib import dataclasses -import enum import functools import inspect -import itertools import numbers -import os -import sys -import typing import warnings -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Hashable, - Iterable, - List, - Optional, - Set, - Tuple, - Union, - cast, -) +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union import docstring_parser -import typing_extensions -from typing_extensions import ( - Annotated, - NotRequired, - Required, - get_args, - get_origin, - is_typeddict, -) +from typing_extensions import Annotated -from . import _docstrings, _resolver, _singleton, _strings, _unsafe_cache +from . import _docstrings, _resolver, _strings, _unsafe_cache +from ._singleton import DEFAULT_SENTINEL_SINGLETONS, MISSING_SINGLETONS from ._typing import TypeForm from .conf import _confstruct, _markers from .constructors._primitive_spec import ( PrimitiveConstructorSpec, + PrimitiveTypeInfo, UnsupportedTypeAnnotationError, ) +from .constructors._registry import ConstructorRegistry +from .constructors._struct_spec import StructTypeInfo, UnsupportedStructTypeMessage global_context_markers: List[Tuple[_markers.Marker, ...]] = [] @@ -95,28 +70,28 @@ def marker_context(markers: Tuple[_markers.Marker, ...]): @staticmethod def make( name: str, - static_type: Union[TypeForm[Any], Callable], + typ: Union[TypeForm[Any], Callable], default: Any, is_default_from_default_instance: bool, helptext: Optional[str], call_argname_override: Optional[Any] = None, ): # Resolve generics. - static_type = _resolver.TypeParamResolver.concretize_type_params(static_type) + typ = _resolver.TypeParamResolver.concretize_type_params(typ) # Narrow types. - if static_type is Any and default not in MISSING_SINGLETONS: - static_type = type(default) + if typ is Any and default not in MISSING_SINGLETONS: + typ = type(default) else: # TypeVar constraints are already applied in # TypeParamResolver.concretize_type_params(), but that won't be # called for functions. - static_type = _resolver.type_from_typevar_constraints(static_type) - static_type = _resolver.narrow_collection_types(static_type, default) - static_type = _resolver.narrow_union_type(static_type, default) + typ = _resolver.type_from_typevar_constraints(typ) + typ = _resolver.narrow_collection_types(typ, default) + typ = _resolver.narrow_union_type(typ, default) # Try to extract argconf overrides from type. - _, argconfs = _resolver.unwrap_annotated(static_type, _confstruct._ArgConfig) + _, argconfs = _resolver.unwrap_annotated(typ, _confstruct._ArgConfig) argconf = _confstruct._ArgConfig( None, None, @@ -139,15 +114,13 @@ def make( if argconf.help is not None: helptext = argconf.help - _, primitive_specs = _resolver.unwrap_annotated( - static_type, PrimitiveConstructorSpec - ) + _, primitive_specs = _resolver.unwrap_annotated(typ, PrimitiveConstructorSpec) if len(primitive_specs) > 0: primitive_spec = primitive_specs[0] else: primitive_spec = None - static_type, markers = _resolver.unwrap_annotated(static_type, _markers._Marker) + typ, markers = _resolver.unwrap_annotated(typ, _markers._Marker) # Include markers set via context manager. for context_markers in global_context_markers: @@ -160,8 +133,8 @@ def make( # Be relatively conservative: isinstance() can be checked on non-type # types (like unions in Python >=3.10), but we'll only consider single types # for now. - type(static_type) is type - and not isinstance(default, static_type) # type: ignore + type(typ) is type + and not isinstance(default, typ) # type: ignore # If a custom constructor is set, static_type may not be # matched to the annotated type. and argconf.constructor_factory is None @@ -173,17 +146,17 @@ def make( # If the default value doesn't match the resolved type, we expand the # type. This is inspired by https://github.com/brentyi/tyro/issues/88. warnings.warn( - f"The field {name} is annotated with type {static_type}, " + f"The field {name} is annotated with type {typ}, " f"but the default value {default} has type {type(default)}. " f"We'll try to handle this gracefully, but it may cause unexpected behavior." ) - static_type = Union[static_type, type(default)] # type: ignore + typ = Union[typ, type(default)] # type: ignore out = FieldDefinition( intern_name=name, extern_name=name if argconf.name is None else argconf.name, type_or_callable=( - static_type + typ if argconf.constructor_factory is None else argconf.constructor_factory() ), @@ -224,71 +197,8 @@ def is_positional_call(self) -> bool: ) -class PropagatingMissingType(_singleton.Singleton): - pass - - -class NonpropagatingMissingType(_singleton.Singleton): - pass - - -class ExcludeFromCallType(_singleton.Singleton): - pass - - -class NotRequiredButWeDontKnowTheValueType(_singleton.Singleton): - pass - - -# We have two types of missing sentinels: a propagating missing value, which when set as -# a default will set all child values of nested structures as missing as well, and a -# nonpropagating missing sentinel, which does not override child defaults. -MISSING_PROP = PropagatingMissingType() -MISSING_NONPROP = NonpropagatingMissingType() - -# When total=False in a TypedDict, we exclude fields from the constructor by default. -NOT_REQUIRED_BUT_WE_DONT_KNOW_THE_VALUE = NotRequiredButWeDontKnowTheValueType() - - -EXCLUDE_FROM_CALL = ExcludeFromCallType() - -# Note that our "public" missing API will always be the propagating missing sentinel. -MISSING: Any = MISSING_PROP -"""Sentinel value to mark fields as missing. Can be used to mark fields passed -in via `default=` for `tyro.cli()` as required.""" - - -MISSING_SINGLETONS = [ - dataclasses.MISSING, - MISSING_PROP, - MISSING_NONPROP, - inspect.Parameter.empty, -] -try: - # Undocumented feature: support omegaconf dataclasses out of the box. - import omegaconf - - MISSING_SINGLETONS.append(omegaconf.MISSING) -except ImportError: - pass - -DEFAULT_SENTINEL_SINGLETONS = MISSING_SINGLETONS + [ - NOT_REQUIRED_BUT_WE_DONT_KNOW_THE_VALUE, - EXCLUDE_FROM_CALL, -] - - -@dataclasses.dataclass(frozen=True) -class UnsupportedNestedTypeMessage: - """Reason why a callable cannot be treated as a struct type.""" - - message: str - - @_unsafe_cache.unsafe_cache(maxsize=1024) -def is_struct_type( - typ: Union[TypeForm[Any], Callable], default_instance: DefaultInstance -) -> bool: +def is_struct_type(typ: Union[TypeForm[Any], Callable], default_instance: Any) -> bool: """Determine whether a type should be treated as a 'struct type', where a single type can be broken down into multiple fields (eg for nested dataclasses or classes). @@ -296,690 +206,92 @@ def is_struct_type( TODO: we should come up with a better name than 'struct type', which is a little bit misleading.""" - list_or_error = _try_field_list_from_callable(typ, default_instance) + list_or_error = field_list_from_type_or_callable( + typ, default_instance, support_single_arg_types=False + ) return not isinstance( list_or_error, - UnsupportedNestedTypeMessage, + UnsupportedStructTypeMessage, ) -def field_list_from_callable( +def field_list_from_type_or_callable( f: Union[Callable, TypeForm[Any]], - default_instance: DefaultInstance, + default_instance: Any, support_single_arg_types: bool, -) -> Tuple[Union[Callable, TypeForm[Any]], List[FieldDefinition]]: +) -> ( + UnsupportedStructTypeMessage + | tuple[Callable | TypeForm[Any], list[FieldDefinition]] +): """Generate a list of generic 'field' objects corresponding to the inputs of some annotated callable. Returns: The type that `f` is resolved as. - A type_from_typevar dict. A list of field definitions. """ - # Resolve generic types. - f = _resolver.narrow_subtypes(f, default_instance) - - # Try to generate field list. - # We recursively apply markers. - _, parent_markers = _resolver.unwrap_annotated(f, _markers._Marker) - with FieldDefinition.marker_context(parent_markers): - field_list = _try_field_list_from_callable(f, default_instance) - - if isinstance(field_list, UnsupportedNestedTypeMessage): - if support_single_arg_types: - with FieldDefinition.marker_context( - (_markers.Positional, _markers._PositionalCall) - ): - return ( - f, - [ - FieldDefinition.make( - name="value", - static_type=f, - default=default_instance, - is_default_from_default_instance=True, - helptext="", - ) - ], - ) - else: - raise UnsupportedTypeAnnotationError(field_list.message) - - return f, field_list - - -# Implementation details below. - - -DefaultInstance = Union[ - Any, PropagatingMissingType, NonpropagatingMissingType, ExcludeFromCallType -] -_known_parsable_types: Set[type] = set( - filter( - lambda x: isinstance(x, Hashable), # type: ignore - itertools.chain( - vars(builtins).values(), # type: ignore - vars(typing).values(), - vars(typing_extensions).values(), - vars(collections.abc).values(), - ), - ) -) - - -def _try_field_list_from_callable( - f: Union[Callable, TypeForm[Any]], - default_instance: DefaultInstance, -) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - # Apply constructor_factory override for type. f = _resolver.swap_type_using_confstruct(f) - typevar_context = _resolver.TypeParamResolver.get_assignment_context(f) - f = typevar_context.origin_type - with typevar_context: - # Check for default instances in subcommand configs. This is needed for - # is_nested_type() when arguments are not valid without a default, and this - # default is specified in the subcommand config. - from . import conf # Avoid circular import. - - f, found_subcommand_configs = _resolver.unwrap_annotated( - f, conf._confstruct._SubcommandConfig - ) - if len(found_subcommand_configs) > 0: - default_instance = found_subcommand_configs[0].default - - # Unwrap generics. - f = _resolver.narrow_subtypes(f, default_instance) - f = _resolver.narrow_collection_types(f, default_instance) - f_origin = _resolver.unwrap_origin_strip_extras(cast(TypeForm, f)) - - # If `f` is a type: - # 1. Set cls to the type. - # 2. Consider `f` to be `cls.__init__`. - cls: Optional[TypeForm[Any]] = None - if inspect.isclass(f): - cls = f - if hasattr(cls, "__init__") and cls.__init__ is not object.__init__: - f = cls.__init__ # type: ignore - elif hasattr(cls, "__new__") and cls.__new__ is not object.__new__: - f = cls.__new__ - else: - return UnsupportedNestedTypeMessage( - f"Cannot instantiate class {cls} with no unique __init__ or __new__" - " method." - ) - f_origin = cls # type: ignore - - # Try field generation from class inputs. - if cls is not None: - if is_typeddict(cls): - return _field_list_from_typeddict(cls, default_instance) - if _resolver.is_namedtuple(cls): - return _field_list_from_namedtuple(cls, default_instance) - if _resolver.is_dataclass(cls): - return _field_list_from_dataclass(cls, default_instance) - if _is_attrs(cls): - return _field_list_from_attrs(cls, default_instance) - if _is_pydantic(cls): - return _field_list_from_pydantic(cls, default_instance) - - # Standard container types. These are different because they can be nested structures - # if they contain other struct types (eg Tuple[Struct, Struct]), or treated as - # single arguments otherwise (eg Tuple[int, int]). - # - # Note that f_origin will be populated if we annotate as `Tuple[..]`, and cls will - # be populated if we annotate as just `tuple`. - if f_origin is tuple or cls is tuple: - return _field_list_from_tuple(f, default_instance) - elif f_origin in (collections.abc.Mapping, dict) or cls in ( - collections.abc.Mapping, - dict, - ): - return _field_list_from_dict(f, default_instance) - elif f_origin in ( - list, - set, - typing.Sequence, - collections.abc.Sequence, - ) or cls in ( - list, - set, - typing.Sequence, - collections.abc.Sequence, - ): - return _field_list_from_nontuple_sequence_checked(f, default_instance) - - # General cases. - if ( - cls is not None and cls in _known_parsable_types - ) or _resolver.unwrap_origin_strip_extras(f) in _known_parsable_types: - return UnsupportedNestedTypeMessage(f"{f} should be parsed directly!") - elif ( - cls is not None - and issubclass(_resolver.unwrap_origin_strip_extras(cls), os.PathLike) - # and _instantiators.is_type_string_converter(cls) - ): - return UnsupportedNestedTypeMessage( - f"PathLike {cls} should be parsed directly!" - ) - else: - return _try_field_list_from_general_callable(f, cls, default_instance) - - -def _field_list_from_typeddict( - cls: TypeForm[Any], default_instance: DefaultInstance -) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - field_list = [] - valid_default_instance = ( - default_instance not in MISSING_SINGLETONS - and default_instance is not EXCLUDE_FROM_CALL - ) - assert not valid_default_instance or isinstance(default_instance, dict) - total = getattr(cls, "__total__", True) - assert isinstance(total, bool) - assert not valid_default_instance or isinstance(default_instance, dict) - for name, typ in _resolver.get_type_hints_with_backported_syntax( - cls, include_extras=True - ).items(): - typ_origin = get_origin(typ) - is_default_from_default_instance = False - if valid_default_instance and name in cast(dict, default_instance): - default = cast(dict, default_instance)[name] - is_default_from_default_instance = True - elif typ_origin is Required and total is False: - # Support total=False. - default = MISSING_PROP - elif total is False: - # Support total=False. - default = EXCLUDE_FROM_CALL - if is_struct_type(typ, MISSING_NONPROP): - # total=False behavior is unideal for nested structures. - pass - # raise _instantiators.UnsupportedTypeAnnotationError( - # "`total=False` not supported for nested structures." - # ) - elif typ_origin is NotRequired: - # Support typing.NotRequired[]. - default = EXCLUDE_FROM_CALL - else: - default = MISSING_PROP - - # Nested types need to be populated / can't be excluded from the call. - if default is EXCLUDE_FROM_CALL and is_struct_type(typ, MISSING_NONPROP): - default = MISSING_NONPROP - - if typ_origin in (Required, NotRequired): - args = get_args(typ) - assert ( - len(args) == 1 - ), "typing.Required[] and typing.NotRequired[T] require a concrete type T." - typ = args[0] - del args - - field_list.append( - FieldDefinition.make( - name=name, - static_type=typ, - default=default, - is_default_from_default_instance=is_default_from_default_instance, - helptext=_docstrings.get_field_docstring(cls, name), - ) - ) - return field_list - - -def _field_list_from_namedtuple( - cls: TypeForm[Any], default_instance: DefaultInstance -) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - # Handle NamedTuples. - # - # TODO: in terms of helptext, we currently do display the default NamedTuple - # helptext. But we (intentionally) don't for dataclasses; this is somewhat - # inconsistent. - field_list = [] - field_defaults = getattr(cls, "_field_defaults") - - # Note that _field_types is removed in Python 3.9. - for name, typ in _resolver.get_type_hints_with_backported_syntax( - cls, include_extras=True - ).items(): - # Get default, with priority for `default_instance`. - default = field_defaults.get(name, MISSING_NONPROP) - if hasattr(default_instance, name): - default = getattr(default_instance, name) - if default_instance is MISSING_PROP: - default = MISSING_PROP - - field_list.append( - FieldDefinition.make( - name=name, - static_type=typ, - default=default, - is_default_from_default_instance=True, - helptext=_docstrings.get_field_docstring(cls, name), - ) - ) - return field_list - - -def _field_list_from_dataclass( - cls: TypeForm[Any], default_instance: DefaultInstance -) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - is_flax_module = False - try: - # Check if dataclass is a flax module. This is only possible if flax is already - # loaded. - # - # We generally want to avoid importing flax, since it requires a lot of heavy - # imports. - if "flax.linen" in sys.modules.keys(): - import flax.linen - - if issubclass(cls, flax.linen.Module): - is_flax_module = True - except ImportError: - pass - - # Handle dataclasses. - field_list = [] - for dc_field in filter(lambda field: field.init, _resolver.resolved_fields(cls)): - # For flax modules, we ignore the built-in "name" and "parent" fields. - if is_flax_module and dc_field.name in ("name", "parent"): - continue - - default, is_default_from_default_instance = _get_dataclass_field_default( - dc_field, default_instance - ) - - # Try to get helptext from field metadata. This is also intended to be - # compatible with HuggingFace-style config objects. - helptext = dc_field.metadata.get("help", None) - assert isinstance(helptext, (str, type(None))) - - # Try to get helptext from docstrings. Note that this can't be generated - # dynamically. - if helptext is None: - helptext = _docstrings.get_field_docstring(cls, dc_field.name) - - assert not isinstance(dc_field.type, str) - field_list.append( - FieldDefinition.make( - name=dc_field.name, - static_type=dc_field.type, - default=default, - is_default_from_default_instance=is_default_from_default_instance, - helptext=helptext, - ) - ) - return field_list - - -def _is_pydantic(cls: TypeForm[Any]) -> bool: - # pydantic will already be imported if it's used. - if "pydantic" not in sys.modules.keys(): # pragma: no cover - # This is needed for the mock import test in - # test_missing_optional_packages.py to pass. - return False - - try: - import pydantic - except ImportError: - # This is needed for the mock import test in - # test_missing_optional_packages.py to pass. - return False - - try: - if "pydantic.v1" in sys.modules.keys(): - from pydantic import v1 as pydantic_v1 - else: - pydantic_v1 = None # type: ignore - except ImportError: - if TYPE_CHECKING: - from pydantic import v1 as pydantic_v1 - else: - pydantic_v1 = None - - if issubclass(cls, pydantic.BaseModel): - return True - if pydantic_v1 is not None and issubclass(cls, pydantic_v1.BaseModel): - return True - return False - - -def _field_list_from_pydantic( - cls: TypeForm[Any], default_instance: DefaultInstance -) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - import pydantic - - try: - if "pydantic.v1" in sys.modules.keys(): - from pydantic import v1 as pydantic_v1 - else: - pydantic_v1 = None # type: ignore - except ImportError: - if TYPE_CHECKING: - from pydantic import v1 as pydantic_v1 - else: - pydantic_v1 = None - - # Handle pydantic models. - field_list = [] - pydantic_version = int(getattr(pydantic, "__version__", "1.0.0").partition(".")[0]) - if pydantic_version < 2 or ( - pydantic_v1 is not None and issubclass(cls, pydantic_v1.BaseModel) - ): - # Pydantic 1.xx. - # We do a conditional cast because the pydantic.v1 module won't - # actually exist in legacy versions of pydantic. - if TYPE_CHECKING: - cls_cast = cast(pydantic_v1.BaseModel, cls) # type: ignore - else: - cls_cast = cls - - hints = _resolver.get_type_hints_with_backported_syntax( - cls, include_extras=True - ) - for pd1_field in cls_cast.__fields__.values(): - helptext = pd1_field.field_info.description - if helptext is None: - helptext = _docstrings.get_field_docstring(cls, pd1_field.name) - - default, is_default_from_default_instance = _get_pydantic_v1_field_default( - pd1_field.name, pd1_field, default_instance - ) - - field_list.append( - FieldDefinition.make( - name=pd1_field.name, - static_type=hints[pd1_field.name], - default=default, - is_default_from_default_instance=is_default_from_default_instance, - helptext=helptext, - ) - ) - else: - # Pydantic 2.xx. - for name, pd2_field in cast(pydantic.BaseModel, cls).model_fields.items(): - helptext = pd2_field.description - if helptext is None: - helptext = _docstrings.get_field_docstring(cls, name) - - default, is_default_from_default_instance = _get_pydantic_v2_field_default( - name, pd2_field, default_instance - ) - - field_list.append( - FieldDefinition.make( - name=name, - static_type=( - Annotated.__class_getitem__( # type: ignore - (pd2_field.annotation,) + tuple(pd2_field.metadata) - ) - if len(pd2_field.metadata) > 0 - else pd2_field.annotation - ), - default=default, - is_default_from_default_instance=is_default_from_default_instance, - helptext=helptext, + registry = ConstructorRegistry._get_active_registry() + type_info = StructTypeInfo.make(f, default_instance) + + with type_info._typevar_context: + spec = registry.get_struct_spec(type_info) + + with FieldDefinition.marker_context(type_info.markers): + if spec is not None: + return f, [ + FieldDefinition.make( + f.name, + f.type, + f.default, + f.is_default_overridden, + f.helptext, + call_argname_override=f._call_argname, + ) + for f in spec.fields + ] + + try: + registry.get_primitive_spec(PrimitiveTypeInfo.make(f, set())) + is_primitive = True + except UnsupportedTypeAnnotationError: + is_primitive = False + + if is_primitive and support_single_arg_types: + with FieldDefinition.marker_context( + (_markers.Positional, _markers._PositionalCall) + ): + return ( + f, + [ + FieldDefinition.make( + name="value", + typ=f, + default=default_instance, + is_default_from_default_instance=True, + helptext="", + ) + ], + ) + elif not is_primitive and callable(f): + return _field_list_from_function( + type_info.type, # This will have typing.Annotated metadata stripped. + default_instance, ) - ) - return field_list + return UnsupportedStructTypeMessage(f"{f} is not a valid struct type!") -def _is_attrs(cls: TypeForm[Any]) -> bool: - # attr will already be imported if it's used. - if "attr" not in sys.modules.keys(): # pragma: no cover - return False +def _field_list_from_function( + f: Callable, default_instance: Any +) -> UnsupportedStructTypeMessage | tuple[Callable, list[FieldDefinition]]: + """Generate field lists from non-class callables.""" try: - import attr - except ImportError: - # This is needed for the mock import test in - # test_missing_optional_packages.py to pass. - return False - - return attr.has(cls) - - -def _field_list_from_attrs( - cls: TypeForm[Any], default_instance: DefaultInstance -) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - import attr - - # Resolve forward references in-place, if any exist. - attr.resolve_types(cls) - - # Handle attr classes. - field_list = [] - for attr_field in attr.fields(cls): - # Skip fields with init=False. - if not attr_field.init: - continue - - # Default handling. - name = attr_field.name - default = attr_field.default - is_default_from_default_instance = False - if default_instance not in MISSING_SINGLETONS: - if hasattr(default_instance, name): - default = getattr(default_instance, name) - is_default_from_default_instance = True - else: - warnings.warn( - f"Could not find field {name} in default instance" - f" {default_instance}, which has" - f" type {type(default_instance)},", - stacklevel=2, - ) - elif default is attr.NOTHING: - default = MISSING_NONPROP - elif isinstance(default, attr.Factory): # type: ignore - default = default.factory() # type: ignore - - assert attr_field.type is not None, attr_field - field_list.append( - FieldDefinition.make( - name=name, - static_type=attr_field.type, - default=default, - is_default_from_default_instance=is_default_from_default_instance, - helptext=_docstrings.get_field_docstring(cls, name), - ) - ) - return field_list - - -def _field_list_from_tuple( - f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance -) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - # Fixed-length tuples. - field_list: List[FieldDefinition] = [] - children = get_args(f) - if Ellipsis in children: - return _try_field_list_from_sequence_inner( - next(iter(set(children) - {Ellipsis})), default_instance - ) - - # Infer more specific type when tuple annotation isn't subscripted. This generally - # doesn't happen - if len(children) == 0: - if default_instance in MISSING_SINGLETONS: - return UnsupportedNestedTypeMessage( - "If contained types of a tuple are not specified in the annotation, a" - " default instance must be specified." - ) - else: - assert isinstance(default_instance, tuple) - children = tuple(type(x) for x in default_instance) - - if ( - default_instance in MISSING_SINGLETONS - # EXCLUDE_FROM_CALL indicates we're inside a TypedDict, with total=False. - or default_instance is EXCLUDE_FROM_CALL - ): - default_instance = (default_instance,) * len(children) - - for i, child in enumerate(children): - default_i = default_instance[i] # type: ignore - field_list.append( - FieldDefinition.make( - # Ideally we'd have --tuple[0] instead of --tuple.0 as the command-line - # argument, but in practice the brackets are annoying because they - # require escaping. - name=str(i), - static_type=child, - default=default_i, - is_default_from_default_instance=True, - helptext="", - # This should really set the positional marker, but the CLI is more - # intuitive for mixed nested/non-struct types in tuples when we stick - # with kwargs. Tuples are special-cased in _calling.py. - ) - ) - - contains_nested = False - for field in field_list: - if get_origin(field.type_or_callable) is Union: - for option in get_args(field.type_or_callable): - # The second argument here is the default value, which can help with - # narrowing but is generall not necessary. - contains_nested |= is_struct_type(option, MISSING_NONPROP) - - contains_nested |= is_struct_type(field.type_or_callable, field.default) - - if contains_nested: - break - if not contains_nested: - # We could also check for variable length children, which can be populated when - # the tuple is interpreted as a nested field but not a directly parsed one. - return UnsupportedNestedTypeMessage( - "Tuple does not contain any nested structures." - ) - - return field_list + params = list(inspect.signature(f).parameters.values()) + except ValueError: + return UnsupportedStructTypeMessage(f"Could not get signature for {f}!") - -def _field_list_from_nontuple_sequence_checked( - f: Union[Callable, TypeForm[Any]], default_instance: DefaultInstance -) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - """Tuples are handled differently due to variadics (tuple[T1, T2, T3, ..., Tn]), - while list[], sequence[], set[], etc only have one typevar.""" - contained_type: Any - if len(get_args(f)) == 0: - # A raw collection type (list, tuple, etc) was passed in, but narrowing also failed. - assert ( - default_instance in MISSING_SINGLETONS - or len(cast(typing.Sequence, default_instance)) == 0 - ), f"Default instance {default_instance} for type {f} was unexpected!" - return UnsupportedNestedTypeMessage( - f"Sequence type {f} needs either an explicit type or a" - " default to infer from." - ) - else: - (contained_type,) = get_args(f) - return _try_field_list_from_sequence_inner(contained_type, default_instance) - - -def _try_field_list_from_sequence_inner( - contained_type: TypeForm[Any], - default_instance: DefaultInstance, -) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - # When no default instance is specified: - # If we have List[int] => this can be parsed as a single field. - # If we have List[SomeStruct] => OK. - if default_instance in MISSING_SINGLETONS and not is_struct_type( - contained_type, MISSING_NONPROP - ): - return UnsupportedNestedTypeMessage( - f"Sequence containing type {contained_type} should be parsed directly!" - ) - - # If we have a default instance: - # [int, int, int] => this can be parsed as a single field. - # [SomeStruct, int, int] => OK. - if isinstance(default_instance, Iterable) and all( - [not is_struct_type(type(x), x) for x in default_instance] - ): - return UnsupportedNestedTypeMessage( - f"Sequence with default {default_instance} should be parsed directly!" - ) - if default_instance in MISSING_SINGLETONS: - # We use the broader error type to prevent it from being caught by - # is_possibly_nested_type(). This is for sure a bad annotation! - raise UnsupportedTypeAnnotationError( - "tyro currently only supports fixed-length sequences of struct types. For " - "variable-length sequences over struct types, we need a default value to " - "infer length from. You can also consider a custom constructor, see here " - "for an example: https://github.com/brentyi/tyro/issues/151" - ) - - field_list = [] - for i, default_i in enumerate(default_instance): # type: ignore - field_list.append( - FieldDefinition.make( - name=str(i), - static_type=contained_type, - default=default_i, - is_default_from_default_instance=True, - helptext="", - ) - ) - return field_list - - -def _field_list_from_dict( - f: Union[Callable, TypeForm[Any]], - default_instance: DefaultInstance, -) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - if default_instance in MISSING_SINGLETONS or len(cast(dict, default_instance)) == 0: - return UnsupportedNestedTypeMessage( - "Nested dictionary structures must have non-empty default instance specified." - ) - field_list = [] - for k, v in cast(dict, default_instance).items(): - field_list.append( - FieldDefinition.make( - name=str(k) if not isinstance(k, enum.Enum) else k.name, - static_type=type(v), - default=v, - is_default_from_default_instance=True, - helptext=None, - # Dictionary specific key: - call_argname_override=k, - ) - ) - return field_list - - -def _try_field_list_from_general_callable( - f: Union[Callable, TypeForm[Any]], - cls: Optional[TypeForm[Any]], - default_instance: DefaultInstance, -) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: - # Generate field list from function signature. - if not callable(f): - return UnsupportedNestedTypeMessage( - f"Cannot extract annotations from {f}, which is not a callable type." - ) - params = list(inspect.signature(f).parameters.values()) - if cls is not None: - # Ignore self parameter. - params = params[1:] - - # If a default is provided: either all or none of the arguments must be passed in. - markers: Tuple[Any, ...] = () - if default_instance not in MISSING_SINGLETONS: - markers = (_markers._OPTIONAL_GROUP,) - - with FieldDefinition.marker_context(markers): - return _field_list_from_params(f, cls, params) - - -def _field_list_from_params( - f: Union[Callable, TypeForm[Any]], - cls: Optional[TypeForm[Any]], - params: List[inspect.Parameter], -) -> Union[List[FieldDefinition], UnsupportedNestedTypeMessage]: # Unwrap functools.wraps and functools.partial. done = False while not done: @@ -991,10 +303,18 @@ def _field_list_from_params( f = f.func done = False - # Sometime functools.* is applied to a class. + # Check for abstract classes. + if inspect.isabstract(f): + return UnsupportedStructTypeMessage("Abstract classes cannot be instantiated!") + + # `f` that is called (output) may be different from what we want to + # inspect. + f_out = f if inspect.isclass(f): - cls = f - f = f.__init__ # type: ignore + if hasattr(f, "__init__") and f.__init__ is not object.__init__: + f = f.__init__ # type: ignore + elif hasattr(f, "__new__") and f.__new__ is not object.__new__: + f = f.__new__ # Get type annotations, docstrings. docstring = inspect.getdoc(f) @@ -1008,7 +328,7 @@ def _field_list_from_params( try: hints = _resolver.get_type_hints_with_backported_syntax(f, include_extras=True) except TypeError: - return UnsupportedNestedTypeMessage(f"Could not get hints for {f}!") + return UnsupportedStructTypeMessage(f"Could not get hints for {f}!") field_list = [] for param in params: @@ -1017,11 +337,13 @@ def _field_list_from_params( # Get helptext from docstring. helptext = docstring_from_arg_name.get(param.name) - if helptext is None and cls is not None: - helptext = _docstrings.get_field_docstring(cls, param.name) + + # TODO: re-add. + if helptext is None and inspect.isclass(f_out): + helptext = _docstrings.get_field_docstring(f_out, param.name) if param.name not in hints: - out = UnsupportedNestedTypeMessage( + out = UnsupportedStructTypeMessage( f"Expected fully type-annotated callable, but {f} with arguments" f" {tuple(map(lambda p: p.name, params))} has no annotation for" f" '{param.name}'." @@ -1062,144 +384,13 @@ def _field_list_from_params( FieldDefinition.make( name=param.name, # Note that param.annotation doesn't resolve forward references. - static_type=typ, + typ=typ + if default_instance in MISSING_SINGLETONS + else Annotated.__class_getitem__((typ, _markers._OPTIONAL_GROUP)), # type: ignore default=default, is_default_from_default_instance=False, helptext=helptext, ) ) - return field_list - - -def _ensure_dataclass_instance_used_as_default_is_frozen( - field: dataclasses.Field, default_instance: Any -) -> None: - """Ensure that a dataclass type used directly as a default value is marked as - frozen.""" - assert dataclasses.is_dataclass(default_instance) - cls = type(default_instance) - if not cls.__dataclass_params__.frozen: # type: ignore - warnings.warn( - f"Mutable type {cls} is used as a default value for `{field.name}`. This is" - " dangerous! Consider using `dataclasses.field(default_factory=...)` or" - f" marking {cls} as frozen." - ) - - -def _get_dataclass_field_default( - field: dataclasses.Field, parent_default_instance: Any -) -> Tuple[Any, bool]: - """Helper for getting the default instance for a dataclass field.""" - # If the dataclass's parent is explicitly marked MISSING, mark this field as missing - # as well. - if parent_default_instance is MISSING_PROP: - return MISSING_PROP, False - - # Try grabbing default from parent instance. - if ( - parent_default_instance not in MISSING_SINGLETONS - and parent_default_instance is not None - ): - # Populate default from some parent, eg `default=` in `tyro.cli()`. - if hasattr(parent_default_instance, field.name): - return getattr(parent_default_instance, field.name), True - else: - warnings.warn( - f"Could not find field {field.name} in default instance" - f" {parent_default_instance}, which has" - f" type {type(parent_default_instance)},", - stacklevel=2, - ) - - # Try grabbing default from dataclass field. - if field.default not in MISSING_SINGLETONS: - default = field.default - # Note that dataclasses.is_dataclass() will also return true for dataclass - # _types_, not just instances. - if type(default) is not type and dataclasses.is_dataclass(default): - _ensure_dataclass_instance_used_as_default_is_frozen(field, default) - return default, False - - # Populate default from `dataclasses.field(default_factory=...)`. - if field.default_factory is not dataclasses.MISSING and not ( - # Special case to ignore default_factory if we write: - # `field: Dataclass = dataclasses.field(default_factory=Dataclass)`. - # - # In other words, treat it the same way as: `field: Dataclass`. - # - # The only time this matters is when we our dataclass has a `__post_init__` - # function that mutates the dataclass. We choose here to use the default values - # before this method is called. - dataclasses.is_dataclass(field.type) and field.default_factory is field.type - ): - return field.default_factory(), False - - # Otherwise, no default. This is different from MISSING, because MISSING propagates - # to children. We could revisit this design to make it clearer. - return MISSING_NONPROP, False - - -if TYPE_CHECKING: - import pydantic as pydantic - import pydantic.v1.fields as pydantic_v1_fields - - -def _get_pydantic_v1_field_default( - name: str, - field: pydantic_v1_fields.ModelField, - parent_default_instance: DefaultInstance, -) -> Tuple[Any, bool]: - """Helper for getting the default instance for a Pydantic field.""" - - # Try grabbing default from parent instance. - if ( - parent_default_instance not in MISSING_SINGLETONS - and parent_default_instance is not None - ): - # Populate default from some parent, eg `default=` in `tyro.cli()`. - if hasattr(parent_default_instance, name): - return getattr(parent_default_instance, name), True - else: - warnings.warn( - f"Could not find field {name} in default instance" - f" {parent_default_instance}, which has" - f" type {type(parent_default_instance)},", - stacklevel=2, - ) - - if not field.required: - return field.get_default(), False - - # Otherwise, no default. - return MISSING_NONPROP, False - - -def _get_pydantic_v2_field_default( - name: str, - field: pydantic.fields.FieldInfo, - parent_default_instance: DefaultInstance, -) -> Tuple[Any, bool]: - """Helper for getting the default instance for a Pydantic field.""" - - # Try grabbing default from parent instance. - if ( - parent_default_instance not in MISSING_SINGLETONS - and parent_default_instance is not None - ): - # Populate default from some parent, eg `default=` in `tyro.cli()`. - if hasattr(parent_default_instance, name): - return getattr(parent_default_instance, name), True - else: - warnings.warn( - f"Could not find field {name} in default instance" - f" {parent_default_instance}, which has" - f" type {type(parent_default_instance)},", - stacklevel=2, - ) - - if not field.is_required(): - return field.get_default(call_default_factory=True), False - - # Otherwise, no default. - return MISSING_NONPROP, False + return f_out, field_list diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 0e5f28ce9..068b1ac8e 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -19,6 +19,8 @@ from typing_extensions import Annotated, get_args, get_origin +from tyro.constructors._struct_spec import UnsupportedStructTypeMessage + from . import _argparse as argparse from . import ( _argparse_formatter, @@ -26,6 +28,7 @@ _docstrings, _fields, _resolver, + _singleton, _strings, _subcommand_matching, ) @@ -66,7 +69,7 @@ def from_callable_or_type( description: Optional[str], parent_classes: Set[Type[Any]], default_instance: Union[ - T, _fields.PropagatingMissingType, _fields.NonpropagatingMissingType + T, _singleton.PropagatingMissingType, _singleton.NonpropagatingMissingType ], intern_prefix: str, extern_prefix: str, @@ -81,11 +84,13 @@ def from_callable_or_type( # Resolve the type of `f`, generate a field list. with _fields.FieldDefinition.marker_context(tuple(markers)): - f, field_list = _fields.field_list_from_callable( + out = _fields.field_list_from_type_or_callable( f=f, default_instance=default_instance, support_single_arg_types=support_single_arg_types, ) + assert not isinstance(out, UnsupportedStructTypeMessage) + f, field_list = out # Cycle detection. # @@ -423,7 +428,7 @@ def from_field( if not any( [ o is not none_proxy - and _fields.is_struct_type(cast(type, o), _fields.MISSING_NONPROP) + and _fields.is_struct_type(cast(type, o), _singleton.MISSING_NONPROP) for o in options ] ): @@ -456,7 +461,7 @@ def from_field( subcommand_type_from_name[subcommand_name] = cast(type, option) # If a field default is provided, try to find a matching subcommand name. - if field.default is None or field.default in _fields.MISSING_SINGLETONS: + if field.default is None or field.default in _singleton.MISSING_SINGLETONS: default_name = None else: default_name = _subcommand_matching.match_subcommand( @@ -494,7 +499,7 @@ def from_field( subcommand_config = _confstruct._SubcommandConfig( "unused", description=None, - default=_fields.MISSING_NONPROP, + default=_singleton.MISSING_NONPROP, prefix_name=True, constructor_factory=None, ) @@ -502,7 +507,7 @@ def from_field( # If names match, borrow subcommand default from field default. if default_name == subcommand_name and ( field.is_default_from_default_instance - or subcommand_config.default in _fields.MISSING_SINGLETONS + or subcommand_config.default in _singleton.MISSING_SINGLETONS ): subcommand_config = dataclasses.replace( subcommand_config, default=field.default diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index f1588b058..3fa2ea065 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -14,12 +14,10 @@ Callable, ClassVar, Dict, - FrozenSet, List, Sequence, Set, Tuple, - Type, TypeVar, Union, cast, @@ -38,7 +36,8 @@ get_type_hints, ) -from . import _fields, _unsafe_cache, conf +from . import _unsafe_cache, conf +from ._singleton import MISSING_SINGLETONS from ._typing import TypeForm UnionType = getattr(types, "UnionType", Union) @@ -425,49 +424,20 @@ def _concretize_type_params(typ: TypeOrCallable, seen: set[Any]) -> TypeOrCallab for x in new_args_list ) - # Apply shims to convert from types.UnionType to typing.Union, list to typing.List, etc. - # This will let us call `copy_with()` next. - typ = standardize_builtin_generics(typ) - # Standard generic aliases have a `copy_with()`! - if hasattr(typ, "copy_with"): + if origin is UnionType: + return Union.__getitem__(new_args) # type: ignore + elif hasattr(typ, "copy_with"): + # typing.List, typing.Dict, etc. return typ.copy_with(new_args) # type: ignore else: - # `collections` types, like collections.abc.Sequence. - assert hasattr(origin, "__class_getitem__") - return origin.__class_getitem__(new_args) # type: ignore + # list[], dict[], etc. + assert origin is not None + return origin[new_args] return typ # type: ignore -def standardize_builtin_generics(typ: TypeOrCallable) -> TypeOrCallable: - """Apply shims to convert `types.UnionType` to `typing.Union`, `list` to - `typing.List`, etc.""" - origin = get_origin(typ) - args = get_args(typ) - - if origin is None or len(args) == 0: - return typ - - # Convert Python 3.9 and 3.10 types to their typing library equivalents, which - # support `.copy_with()`. This is not really the right place for this logic... - if sys.version_info[:2] >= (3, 9): - shim_table = { - # PEP 585. Requires Python 3.9. - tuple: Tuple, - list: List, - dict: Dict, - set: Set, - frozenset: FrozenSet, - type: Type, - UnionType: Union, - } - - if origin in shim_table: # type: ignore - typ = shim_table[origin].__getitem__(args) # type: ignore - return typ - - class TypeParamAssignmentContext: def __init__( self, @@ -501,9 +471,8 @@ def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCalla options = get_args(typ) options_unwrapped = [unwrap_origin_strip_extras(o) for o in options] - # (A) try: - if default_instance not in _fields.MISSING_SINGLETONS and not any( + if default_instance not in MISSING_SINGLETONS and not any( isinstance(default_instance, o) for o in options_unwrapped ): warnings.warn( @@ -535,7 +504,6 @@ def resolve_generic_types( # Apply shims to convert from types.UnionType to typing.Union, list to # typing.List, etc. - typ = standardize_builtin_generics(typ) typ = resolve_newtype_and_aliases(typ) # We'll ignore NewType when getting the origin + args for generics. diff --git a/src/tyro/_singleton.py b/src/tyro/_singleton.py index f52efefd1..c947966ad 100644 --- a/src/tyro/_singleton.py +++ b/src/tyro/_singleton.py @@ -1,3 +1,8 @@ +import dataclasses +import inspect +from typing import Any + + class Singleton: # Singleton pattern. # https://www.python.org/download/releases/2.2/descrintro/#__new__ @@ -11,3 +16,57 @@ def __new__(cls, *args, **kwds): def init(self, *args, **kwds): pass + + +class PropagatingMissingType(Singleton): + pass + + +class NonpropagatingMissingType(Singleton): + pass + + +class ExcludeFromCallType(Singleton): + pass + + +class NotRequiredButWeDontKnowTheValueType(Singleton): + pass + + +# We have two types of missing sentinels: a propagating missing value, which when set as +# a default will set all child values of nested structures as missing as well, and a +# nonpropagating missing sentinel, which does not override child defaults. +MISSING_PROP = PropagatingMissingType() +MISSING_NONPROP = NonpropagatingMissingType() + +# When total=False in a TypedDict, we exclude fields from the constructor by default. +NOT_REQUIRED_BUT_WE_DONT_KNOW_THE_VALUE = NotRequiredButWeDontKnowTheValueType() + + +EXCLUDE_FROM_CALL = ExcludeFromCallType() + +# Note that our "public" missing API will always be the propagating missing sentinel. +MISSING: Any = MISSING_PROP +"""Sentinel value to mark fields as missing. Can be used to mark fields passed +in via `default=` for `tyro.cli()` as required.""" + + +MISSING_SINGLETONS = [ + dataclasses.MISSING, + MISSING_PROP, + MISSING_NONPROP, + inspect.Parameter.empty, +] +try: + # Undocumented feature: support omegaconf dataclasses out of the box. + import omegaconf + + MISSING_SINGLETONS.append(omegaconf.MISSING) +except ImportError: + pass + +DEFAULT_SENTINEL_SINGLETONS = MISSING_SINGLETONS + [ + NOT_REQUIRED_BUT_WE_DONT_KNOW_THE_VALUE, + EXCLUDE_FROM_CALL, +] diff --git a/src/tyro/_subcommand_matching.py b/src/tyro/_subcommand_matching.py index 4cb07f1fb..82e8d20bb 100644 --- a/src/tyro/_subcommand_matching.py +++ b/src/tyro/_subcommand_matching.py @@ -5,9 +5,10 @@ from typing_extensions import get_args, get_origin -from . import _fields, _resolver, _typing +from tyro.constructors._struct_spec import UnsupportedStructTypeMessage + +from . import _fields, _resolver, _singleton, _typing from .conf import _confstruct -from .constructors._primitive_spec import UnsupportedTypeAnnotationError def match_subcommand( @@ -25,14 +26,14 @@ def match_subcommand( # Get default subcommand name: by default hash. default_hash = object.__hash__(default) for subcommand_name, conf in subcommand_config_from_name.items(): - if conf.default in _fields.MISSING_SINGLETONS: + if conf.default in _singleton.MISSING_SINGLETONS: continue if default_hash == object.__hash__(conf.default): return subcommand_name # Get default subcommand name: by default value. for subcommand_name, conf in subcommand_config_from_name.items(): - if conf.default in _fields.MISSING_SINGLETONS: + if conf.default in _singleton.MISSING_SINGLETONS: continue equal = default == conf.default if isinstance(equal, bool) and equal: @@ -41,14 +42,14 @@ def match_subcommand( # Get default subcommand name: by concrete type tree. typetree = _TypeTree.make(type(default), default) for subcommand_name, conf in subcommand_config_from_name.items(): - if conf.default in _fields.MISSING_SINGLETONS: + if conf.default in _singleton.MISSING_SINGLETONS: continue if typetree == _TypeTree.make(type(conf.default), conf.default): return subcommand_name # Get default subcommand name: by annotated type tree. typetree_from_name = { - subcommand_name: _TypeTree.make(subcommand_type, _fields.MISSING_NONPROP) + subcommand_name: _TypeTree.make(subcommand_type, _singleton.MISSING_NONPROP) for subcommand_name, subcommand_type in subcommand_type_from_name.items() } for subcommand_name in subcommand_type_from_name.keys(): @@ -73,19 +74,19 @@ class _TypeTree: @staticmethod def make( typ: Union[Callable, _typing.TypeForm], - default_instance: _fields.DefaultInstance, + default_instance: Any, ) -> _TypeTree: """From an object instance, return a data structure representing the types in the object.""" typ_unwrap = _resolver.resolve_generic_types(typ)[0] - try: - typ, field_list = _fields.field_list_from_callable( - typ, default_instance=default_instance, support_single_arg_types=False - ) - except UnsupportedTypeAnnotationError: + field_list_out = _fields.field_list_from_type_or_callable( + typ, default_instance=default_instance, support_single_arg_types=False + ) + if isinstance(field_list_out, UnsupportedStructTypeMessage): return _TypeTree(typ_unwrap, {}) + typ, field_list = field_list_out return _TypeTree( typ_unwrap, { diff --git a/src/tyro/conf/_confstruct.py b/src/tyro/conf/_confstruct.py index 08fa21ce5..0104c4d01 100644 --- a/src/tyro/conf/_confstruct.py +++ b/src/tyro/conf/_confstruct.py @@ -3,7 +3,7 @@ import dataclasses from typing import Any, Callable, Sequence, TypeVar, overload -from .._fields import MISSING_NONPROP +from .._singleton import MISSING_NONPROP T = TypeVar("T") @@ -91,12 +91,12 @@ def subcommand( docstring. prefix_name: Whether to prefix the name of the subcommand based on where it is in a nested structure. - constructor: A constructor type or function. This should either be (a) a subtype - of an argument's annotated type, or (b) a function with type-annotated - inputs that returns an instance of the annotated type. This will be used in - place of the argument's type for parsing arguments. No validation is done. - constructor_factory: A function that returns a constructor type or function. - Useful when the constructor isn't immediately available. + constructor: A constructor type or function. This will be used in + place of the argument's type for parsing arguments. For more + configurability, see :module:`tyro.constructors`. + constructor_factory: A function that returns a constructor type. This + will be used in place of the argument's type for parsing arguments. + For more configurability, see :module:`tyro.constructors`. """ assert not ( constructor is not None and constructor_factory is not None @@ -189,12 +189,12 @@ def arg( structure, and are not supported for positional arguments. prefix_name: Whether or not to prefix the name of the argument based on where it is in a nested structure. Arguments are prefixed by default. - constructor: A constructor type or function. This should either be (a) a subtype - of an argument's annotated type, or (b) a function with type-annotated - inputs that returns an instance of the annotated type. This will be used in - place of the argument's type for parsing arguments. No validation is done. - constructor_factory: A function that returns a constructor type or function. - Useful when the constructor isn't immediately available. + constructor: A constructor type or function. This will be used in + place of the argument's type for parsing arguments. For more + configurability, see :module:`tyro.constructors`. + constructor_factory: A function that returns a constructor type. This + will be used in place of the argument's type for parsing arguments. + For more configurability, see :module:`tyro.constructors`. Returns: Object to attach via `typing.Annotated[]`. diff --git a/src/tyro/constructors/__init__.py b/src/tyro/constructors/__init__.py index 34d3a7f52..d93bfe0ac 100644 --- a/src/tyro/constructors/__init__.py +++ b/src/tyro/constructors/__init__.py @@ -1,5 +1,9 @@ -from ._primitive_spec import ( - PrimitiveConstructorRegistry as PrimitiveConstructorRegistry, -) from ._primitive_spec import PrimitiveConstructorSpec as PrimitiveConstructorSpec from ._primitive_spec import PrimitiveTypeInfo as PrimitiveTypeInfo +from ._primitive_spec import ( + UnsupportedTypeAnnotationError as UnsupportedTypeAnnotationError, +) +from ._registry import ConstructorRegistry as ConstructorRegistry +from ._struct_spec import StructConstructorSpec as StructConstructorSpec +from ._struct_spec import StructFieldSpec as StructFieldSpec +from ._struct_spec import StructTypeInfo as StructTypeInfo diff --git a/src/tyro/constructors/_primitive_spec.py b/src/tyro/constructors/_primitive_spec.py index 12d2ef371..80575e374 100644 --- a/src/tyro/constructors/_primitive_spec.py +++ b/src/tyro/constructors/_primitive_spec.py @@ -13,7 +13,6 @@ from typing import ( Any, Callable, - ClassVar, Dict, Generic, List, @@ -26,7 +25,10 @@ cast, ) -from typing_extensions import Literal, get_args, get_origin +from typing_extensions import TYPE_CHECKING, Literal, get_args, get_origin + +if TYPE_CHECKING: + from ._registry import ConstructorRegistry # There are cases where typing.Literal doesn't match typing_extensions.Literal: # https://github.com/python/typing_extensions/pull/148 @@ -53,28 +55,26 @@ class PrimitiveTypeInfo: """Information used to generate constructors for primitive types.""" type: TypeForm - """Field type, with runtime annotations (typing.Annotated) stripped.""" + """Annotated field type. Forward references, aliases, and type + variables/parameters will have been resolved and runtime annotations + (typing.Annotated) will have been stripped.""" type_origin: TypeForm | None """The output of get_origin() on the static type.""" markers: set[_markers.Marker] """Set of tyro markers used to configure this field.""" - constructor_registry: PrimitiveConstructorRegistry - """The registry used to look up constructor specifications for this type.""" @staticmethod def make( - raw_annotation: TypeForm, + raw_annotation: TypeForm | Callable, parent_markers: set[_markers.Marker], - source_registry: PrimitiveConstructorRegistry, ) -> PrimitiveTypeInfo: typ, extra_markers = _resolver.unwrap_annotated( raw_annotation, search_type=_markers._Marker ) return PrimitiveTypeInfo( - type=typ, + type=cast(TypeForm, typ), type_origin=get_origin(typ), markers=parent_markers | set(extra_markers), - constructor_registry=source_registry, ) @@ -85,8 +85,8 @@ class PrimitiveConstructorSpec(Generic[T]): There are two ways to use this class: First, we can include it in a type signature via `typing.Annotated`. - This is the simplest, and allows for per-field customization of - instantiation behavior. + This is the simplest for making local modifications to parsing behavior for + individual fields. Alternatively, it can be returned by a rule in a `PrimitiveConstructorRegistry`. """ @@ -101,7 +101,8 @@ class PrimitiveConstructorSpec(Generic[T]): length of the list will match the value of nargs.""" is_instance: Callable[[Any], bool] """Given an object instance, does it match this primitive type? This is - used for help messages when a default is provided.""" + used for specific help messages when both a union type is present and a + default is provided.""" str_from_instance: Callable[[T], list[str]] """Convert an instance to a list of string arguments that would construct the instance. This is used for help messages when a default is provided.""" @@ -112,76 +113,15 @@ class PrimitiveConstructorSpec(Generic[T]): """Internal action to use. Not part of the public API.""" -SpecFactory = Callable[[PrimitiveTypeInfo], PrimitiveConstructorSpec] - -current_registry: PrimitiveConstructorRegistry | None = None - - -class PrimitiveConstructorRegistry: - """Registry for rules that define how primitive types that can be - constructed from a single command-line argument.""" - - _active_registry: ClassVar[PrimitiveConstructorRegistry | None] = None - _old_registry: PrimitiveConstructorRegistry | None = None - - def __init__(self) -> None: - self._rules: list[ - tuple[ - # Matching function. - Callable[[PrimitiveTypeInfo], bool], - # Spec factory. - SpecFactory, - ] - ] = [] - - # Apply the default primitive-handling rules. - _apply_default_rules(self) - - def define_rule( - self, matcher_fn: Callable[[PrimitiveTypeInfo], bool] - ) -> Callable[[SpecFactory], SpecFactory]: - """Define a rule for constructing a primitive type from a string. The - most recently added rule will be applied first.""" - - def decorator(spec_factory: SpecFactory) -> SpecFactory: - self._rules.append((matcher_fn, spec_factory)) - return spec_factory - - return decorator - - def get_spec(self, type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: - """Get a constructor specification for a given type.""" - for matcher_fn, spec_factory in self._rules[::-1]: - if matcher_fn(type_info): - return spec_factory(type_info) - - raise UnsupportedTypeAnnotationError( - f"Unsupported type annotation: {type_info.type}" - ) - - @classmethod - def _get_active_registry(cls) -> PrimitiveConstructorRegistry: - """Get the active registry. Can be changed by using a - PrimitiveConstructorRegistry object as a context.""" - if cls._active_registry is None: - cls._active_registry = PrimitiveConstructorRegistry() - return cls._active_registry - - def __enter__(self) -> None: - cls = self.__class__ - self._old_registry = cls._active_registry - cls._active_registry = self - - def __exit__(self, *args: Any) -> None: - cls = self.__class__ - cls._active_registry = self._old_registry - - -def _apply_default_rules(registry: PrimitiveConstructorRegistry) -> None: +def apply_default_primitive_rules(registry: ConstructorRegistry) -> None: """Apply default rules to the registry.""" - @registry.define_rule(lambda type_info: type_info.type is Any) - def any_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + from ._registry import ConstructorRegistry + + @registry.primitive_rule + def any_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: + if type_info.type is not Any: + return None raise UnsupportedTypeAnnotationError("`Any` is not a parsable type.") # HACK: this is for code that uses `tyro.conf.arg(constructor=json.loads)`. @@ -190,8 +130,10 @@ def any_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: # on the behavior so we'll do our best not to break it. vanilla_types = (int, str, float, bytes, json.loads) - @registry.define_rule(lambda type_info: type_info.type in vanilla_types) - def basics_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + @registry.primitive_rule + def basics_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: + if type_info.type not in vanilla_types: + return None return PrimitiveConstructorSpec( nargs=1, metavar=type_info.type.__name__.upper(), @@ -207,8 +149,12 @@ def basics_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: if "torch" in sys.modules.keys(): import torch - @registry.define_rule(lambda type_info: type_info.type is torch.device) - def basics_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + @registry.primitive_rule + def torch_device_rule( + type_info: PrimitiveTypeInfo, + ) -> PrimitiveConstructorSpec | None: + if type_info.type is not torch.device: + return None return PrimitiveConstructorSpec( nargs=1, metavar=type_info.type.__name__.upper(), @@ -217,9 +163,10 @@ def basics_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: str_from_instance=lambda instance: [str(instance)], ) - @registry.define_rule(lambda type_info: type_info.type is bool) - def bool_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: - del type_info + @registry.primitive_rule + def bool_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: + if type_info.type is not bool: + return None return PrimitiveConstructorSpec( nargs=1, metavar="{True,False}", @@ -229,9 +176,10 @@ def bool_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: str_from_instance=lambda instance: ["True" if instance else "False"], ) - @registry.define_rule(lambda type_info: type_info.type is type(None)) - def nonetype_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: - del type_info + @registry.primitive_rule + def nonetype_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: + if type_info.type is not type(None): + return None return PrimitiveConstructorSpec( nargs=1, metavar="{None}", @@ -241,14 +189,16 @@ def nonetype_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: str_from_instance=lambda instance: ["None"], ) - @registry.define_rule( - lambda type_info: type_info.type in (os.PathLike, pathlib.Path) - or ( - inspect.isclass(type_info.type) - and issubclass(type_info.type, pathlib.PurePath) - ) - ) - def path_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + @registry.primitive_rule + def path_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: + if not ( + type_info.type in (os.PathLike, pathlib.Path) + or ( + inspect.isclass(type_info.type) + and issubclass(type_info.type, pathlib.PurePath) + ) + ): + return None return PrimitiveConstructorSpec( nargs=1, metavar=type_info.type.__name__.upper(), @@ -257,11 +207,12 @@ def path_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: str_from_instance=lambda instance: [str(instance)], ) - @registry.define_rule( - lambda type_info: inspect.isclass(type_info.type) - and issubclass(type_info.type, enum.Enum) - ) - def enum_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + @registry.primitive_rule + def enum_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: + if not ( + inspect.isclass(type_info.type) and issubclass(type_info.type, enum.Enum) + ): + return None cast_type = cast(Type[enum.Enum], type_info.type) if _markers.EnumChoicesFromValues in type_info.markers: choices = tuple(str(m.value) for m in cast_type) @@ -289,11 +240,10 @@ def enum_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: choices=choices, ) - @registry.define_rule( - lambda type_info: type_info.type - in (datetime.datetime, datetime.date, datetime.time) - ) - def datetime_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + @registry.primitive_rule + def datetime_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: + if type_info.type not in (datetime.datetime, datetime.date, datetime.time): + return None return PrimitiveConstructorSpec( nargs=1, metavar={ @@ -311,72 +261,11 @@ def datetime_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: str_from_instance=lambda instance: [instance.isoformat()], ) - @registry.define_rule(lambda type_info: type_info.type_origin is tuple) - def tuple_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: - types = get_args(type_info.type) - typeset = set(types) # Note that sets are unordered. - typeset_no_ellipsis = typeset - {Ellipsis} # type: ignore - - if typeset_no_ellipsis != typeset: - # Ellipsis: variable argument counts. When an ellipsis is used, tuples must - # contain only one type. - assert len(typeset_no_ellipsis) == 1 - return sequence_rule(type_info) - - inner_specs = [] - total_nargs = 0 - for contained_type in types: - spec = type_info.constructor_registry.get_spec( - PrimitiveTypeInfo.make( - contained_type, type_info.markers, type_info.constructor_registry - ) - ) - if isinstance(spec.nargs, int): - total_nargs += spec.nargs - else: - raise UnsupportedTypeAnnotationError( - f"Tuples containing a variable-length sequences ({contained_type}) are not supported." - ) - - inner_specs.append(spec) - - def instance_from_str(args: list[str]) -> tuple: - assert len(args) == total_nargs - - out = [] - i = 0 - for member_spec in inner_specs: - assert isinstance(member_spec.nargs, int) - member_strings = args[i : i + member_spec.nargs] - if member_spec.choices is not None and any( - s not in member_spec.choices for s in member_strings - ): - raise ValueError( - f"invalid choice: {member_strings} (choose from {member_spec.choices}))" - ) - out.append(member_spec.instance_from_str(member_strings)) - i += member_spec.nargs - return tuple(out) - - def str_from_instance(instance: tuple) -> list[str]: - out = [] - for member, spec in zip(instance, inner_specs): - out.extend(spec.str_from_instance(member)) - return out - - return PrimitiveConstructorSpec( - nargs=total_nargs, - metavar=" ".join(spec.metavar for spec in inner_specs), - instance_from_str=instance_from_str, - str_from_instance=str_from_instance, - is_instance=lambda x: isinstance(x, tuple) - and len(x) == total_nargs - and all(spec.is_instance(member) for member, spec in zip(x, inner_specs)), - ) - - @registry.define_rule( - lambda type_info: type_info.type - in ( + @registry.primitive_rule + def vague_container_rule( + type_info: PrimitiveTypeInfo, + ) -> PrimitiveConstructorSpec | None: + if type_info.type not in ( dict, Dict, tuple, @@ -387,9 +276,8 @@ def str_from_instance(instance: tuple) -> list[str]: Sequence, set, Set, - ) - ) - def vague_container_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + ): + return None typ = type_info.type if typ in (dict, Dict): typ = Dict[str, str] @@ -399,25 +287,26 @@ def vague_container_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSp typ = List[str] elif typ in (set, Set): typ = Set[str] - return type_info.constructor_registry.get_spec( + + registry = ConstructorRegistry._get_active_registry() + return registry.get_primitive_spec( PrimitiveTypeInfo.make( typ, parent_markers=type_info.markers, - source_registry=type_info.constructor_registry, ) ) - @registry.define_rule( - lambda type_info: type_info.type_origin - in ( + @registry.primitive_rule + def sequence_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: + if type_info.type_origin not in ( collections.abc.Sequence, frozenset, list, set, collections.deque, - ) - ) - def sequence_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + tuple, + ): + return None container_type = type_info.type_origin assert container_type is not None if container_type is collections.abc.Sequence: @@ -429,11 +318,11 @@ def sequence_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: else: (contained_type,) = get_args(type_info.type) - inner_spec = type_info.constructor_registry.get_spec( + registry = ConstructorRegistry._get_active_registry() + inner_spec = registry.get_primitive_spec( PrimitiveTypeInfo.make( raw_annotation=contained_type, parent_markers=type_info.markers - {_markers.UseAppendAction}, - source_registry=type_info.constructor_registry, ) ) @@ -462,7 +351,7 @@ def instance_from_str(args: list[str]) -> Any: for i in range(0, len(args), step): out.append(inner_spec.instance_from_str(args[i : i + inner_spec.nargs])) assert container_type is not None - return container_type(out) + return cast(Callable, container_type)(out) def str_from_instance(instance: Sequence) -> list[str]: out = [] @@ -492,23 +381,89 @@ def str_from_instance(instance: Sequence) -> list[str]: choices=inner_spec.choices, ) - @registry.define_rule( - lambda type_info: type_info.type_origin in (dict, collections.abc.Mapping) - ) - def dict_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + @registry.primitive_rule + def tuple_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: + if type_info.type_origin is not tuple: + return None + types = get_args(type_info.type) + typeset = set(types) # Note that sets are unordered. + typeset_no_ellipsis = typeset - {Ellipsis} # type: ignore + + if typeset_no_ellipsis != typeset: + # Ellipsis: variable argument counts. When an ellipsis is used, tuples must + # contain only one type. + assert len(typeset_no_ellipsis) == 1 + return sequence_rule(type_info) + + registry = ConstructorRegistry._get_active_registry() + + inner_specs = [] + total_nargs = 0 + for contained_type in types: + spec = registry.get_primitive_spec( + PrimitiveTypeInfo.make(contained_type, type_info.markers) + ) + if isinstance(spec.nargs, int): + total_nargs += spec.nargs + else: + raise UnsupportedTypeAnnotationError( + f"Tuples containing a variable-length sequences ({contained_type}) are not supported." + ) + + inner_specs.append(spec) + + def instance_from_str(args: list[str]) -> tuple: + assert len(args) == total_nargs + + out = [] + i = 0 + for member_spec in inner_specs: + assert isinstance(member_spec.nargs, int) + member_strings = args[i : i + member_spec.nargs] + if member_spec.choices is not None and any( + s not in member_spec.choices for s in member_strings + ): + raise ValueError( + f"invalid choice: {member_strings} (choose from {member_spec.choices}))" + ) + out.append(member_spec.instance_from_str(member_strings)) + i += member_spec.nargs + return tuple(out) + + def str_from_instance(instance: tuple) -> list[str]: + out = [] + for member, spec in zip(instance, inner_specs): + out.extend(spec.str_from_instance(member)) + return out + + return PrimitiveConstructorSpec( + nargs=total_nargs, + metavar=" ".join(spec.metavar for spec in inner_specs), + instance_from_str=instance_from_str, + str_from_instance=str_from_instance, + is_instance=lambda x: isinstance(x, tuple) + and len(x) == total_nargs + and all(spec.is_instance(member) for member, spec in zip(x, inner_specs)), + ) + + @registry.primitive_rule + def dict_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: + if type_info.type_origin not in (dict, collections.abc.Mapping): + return None + + registry = ConstructorRegistry._get_active_registry() + key_type, val_type = get_args(type_info.type) - key_spec = type_info.constructor_registry.get_spec( + key_spec = registry.get_primitive_spec( PrimitiveTypeInfo.make( raw_annotation=key_type, parent_markers=type_info.markers, - source_registry=type_info.constructor_registry, ) ) - val_spec = type_info.constructor_registry.get_spec( + val_spec = registry.get_primitive_spec( PrimitiveTypeInfo.make( raw_annotation=val_type, parent_markers=type_info.markers - {_markers.UseAppendAction}, - source_registry=type_info.constructor_registry, ) ) pair_metavar = f"{key_spec.metavar} {val_spec.metavar}" @@ -586,10 +541,10 @@ def str_from_instance(instance: dict) -> list[str]: str_from_instance=str_from_instance, ) - @registry.define_rule( - lambda type_info: type_info.type_origin in (Literal, LiteralAlternate) - ) - def literal_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + @registry.primitive_rule + def literal_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: + if type_info.type_origin not in (Literal, LiteralAlternate): + return None choices = get_args(type_info.type) str_choices = tuple( ( @@ -612,10 +567,10 @@ def literal_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: choices=str_choices, ) - @registry.define_rule( - lambda type_info: type_info.type_origin in (Union, _resolver.UnionType) - ) - def union_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: + @registry.primitive_rule + def union_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: + if type_info.type_origin not in (Union, _resolver.UnionType): + return None options = list(get_args(type_info.type)) if type(None) in options: # Move `None` types to the beginning. @@ -630,12 +585,12 @@ def union_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec: choices: tuple[str, ...] | None = () nargs: int | Literal["*"] = 1 first = True + registry = ConstructorRegistry._get_active_registry() for t in options: - option_spec = type_info.constructor_registry.get_spec( + option_spec = registry.get_primitive_spec( PrimitiveTypeInfo.make( raw_annotation=t, parent_markers=type_info.markers, - source_registry=type_info.constructor_registry, ) ) if option_spec.choices is None: diff --git a/src/tyro/constructors/_registry.py b/src/tyro/constructors/_registry.py new file mode 100644 index 000000000..654379fc0 --- /dev/null +++ b/src/tyro/constructors/_registry.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from typing import Any, Callable, ClassVar, Union + +from ._primitive_spec import ( + PrimitiveConstructorSpec, + PrimitiveTypeInfo, + UnsupportedTypeAnnotationError, + apply_default_primitive_rules, +) +from ._struct_spec import ( + StructConstructorSpec, + StructTypeInfo, + apply_default_struct_rules, +) + +current_registry: ConstructorRegistry | None = None + +PrimitiveSpecRule = Callable[[PrimitiveTypeInfo], Union[PrimitiveConstructorSpec, None]] +StructSpecRule = Callable[[StructTypeInfo], Union[StructConstructorSpec, None]] + + +class ConstructorRegistry: + """Registry for rules that define how types are constructed from + command-line arguments. + + The behavior of CLIs generated by tyro are based on two types of rules. + + *Primitive rules* should be a callable with the signature: + ```python + (type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None + ``` + where `None` is returned if the rule doesn't apply. Each primitive rule + defines behavior for a type that can be instantiated from a single + command-line argument. + + + *Struct rules* should be a callable with the signature: + ```python + (type_info: StructTypeInfo) -> StructConstructorSpec | None + ``` + where `None` is returned if the rule doesn't apply. Each struct rule + defines behavior for a type that can be instantiated from multiple + command-line arguments. + + + To activate a registry, use it as a context manager. For example: + + ```python + registry = ConstructorRegistry() + + with registry: + tyro.cli(...) + ``` + """ + + _active_registry: ClassVar[ConstructorRegistry | None] = None + _old_registry: ConstructorRegistry | None = None + + def __init__(self) -> None: + self._primitive_rules: list[PrimitiveSpecRule] = [] + self._struct_rules: list[StructSpecRule] = [] + + # Apply the default primitive-handling rules. + apply_default_primitive_rules(self) + apply_default_struct_rules(self) + + def primitive_rule(self, rule: PrimitiveSpecRule) -> PrimitiveSpecRule: + """Define a rule for constructing a primitive type from a string. The + most recently added rule will be applied first.""" + + self._primitive_rules.append(rule) + return rule + + def struct_rule(self, rule: StructSpecRule) -> StructSpecRule: + """Define a rule for constructing a primitive type from a string. The + most recently added rule will be applied first.""" + + self._struct_rules.append(rule) + return rule + + def get_primitive_spec( + self, type_info: PrimitiveTypeInfo + ) -> PrimitiveConstructorSpec: + """Get a constructor specification for a given type.""" + for spec_factory in self._primitive_rules[::-1]: + maybe_spec = spec_factory(type_info) + if maybe_spec is not None: + return maybe_spec + + raise UnsupportedTypeAnnotationError( + f"Unsupported type annotation: {type_info.type}" + ) + + def get_struct_spec( + self, type_info: StructTypeInfo + ) -> StructConstructorSpec | None: + """Get a constructor specification for a given type. Returns `None` if + unsuccessful.""" + for spec_factory in self._struct_rules[::-1]: + maybe_spec = spec_factory(type_info) + if maybe_spec is not None: + return maybe_spec + + return None + + @classmethod + def _get_active_registry(cls) -> ConstructorRegistry: + """Get the active registry. Can be changed by using a + PrimitiveConstructorRegistry object as a context.""" + if cls._active_registry is None: + cls._active_registry = ConstructorRegistry() + return cls._active_registry + + def __enter__(self) -> None: + cls = self.__class__ + self._old_registry = cls._active_registry + cls._active_registry = self + + def __exit__(self, *args: Any) -> None: + cls = self.__class__ + cls._active_registry = self._old_registry diff --git a/src/tyro/constructors/_struct_spec.py b/src/tyro/constructors/_struct_spec.py new file mode 100644 index 000000000..6a9aeecdb --- /dev/null +++ b/src/tyro/constructors/_struct_spec.py @@ -0,0 +1,655 @@ +from __future__ import annotations + +import collections.abc +import dataclasses +import enum +import sys +import warnings +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, Sequence, Union + +from typing_extensions import ( + Annotated, + NotRequired, + Required, + cast, + get_args, + get_origin, + is_typeddict, +) + +from .. import _docstrings, _resolver +from .._singleton import ( + EXCLUDE_FROM_CALL, + MISSING_NONPROP, + MISSING_PROP, + MISSING_SINGLETONS, +) +from .._typing import TypeForm +from ..conf import _confstruct, _markers + +if TYPE_CHECKING: + from ._registry import ConstructorRegistry + + +@dataclasses.dataclass(frozen=True) +class UnsupportedStructTypeMessage: + """Reason why a callable cannot be treated as a struct type.""" + + message: str + + +@dataclasses.dataclass(frozen=True) +class StructFieldSpec: + """Behavior specification for a single field in our callable.""" + + name: str + type: TypeForm + default: Any + is_default_overridden: bool + helptext: str | None + # TODO: it's theoretically possible to override the argname with `None`. + _call_argname: Any = None + + +@dataclasses.dataclass(frozen=True) +class StructConstructorSpec: + instantiate: Callable[..., Any] + fields: tuple[StructFieldSpec, ...] + + +@dataclasses.dataclass(frozen=True) +class StructTypeInfo: + """Information used to generate constructors for primitive types.""" + + type: TypeForm + markers: tuple[Any, ...] + default: Any + _typevar_context: _resolver.TypeParamAssignmentContext + + @staticmethod + def make(f: TypeForm | Callable, default: Any) -> StructTypeInfo: + _, parent_markers = _resolver.unwrap_annotated(f, _markers._Marker) + f = _resolver.swap_type_using_confstruct(f) + f, found_subcommand_configs = _resolver.unwrap_annotated( + f, _confstruct._SubcommandConfig + ) + if len(found_subcommand_configs) > 0: + default = found_subcommand_configs[0].default + + # Handle generics. + typevar_context = _resolver.TypeParamResolver.get_assignment_context(f) + f = typevar_context.origin_type + f = _resolver.narrow_subtypes(f, default) + f = _resolver.narrow_collection_types(f, default) + return StructTypeInfo( + cast(TypeForm, f), parent_markers, default, typevar_context + ) + + +def apply_default_struct_rules(registry: ConstructorRegistry) -> None: + from .._fields import is_struct_type + + @registry.struct_rule + def dataclass_rule(info: StructTypeInfo) -> StructConstructorSpec | None: + if not dataclasses.is_dataclass(info.type): + return None + + is_flax_module = False + try: + # Check if dataclass is a flax module. This is only possible if flax is already + # loaded. + # + # We generally want to avoid importing flax, since it requires a lot of heavy + # imports. + if "flax.linen" in sys.modules.keys(): + import flax.linen + + if issubclass(info.type, flax.linen.Module): + is_flax_module = True + except ImportError: + pass + + # Handle dataclasses. + field_list = [] + for dc_field in filter( + lambda field: field.init, _resolver.resolved_fields(info.type) + ): + # For flax modules, we ignore the built-in "name" and "parent" fields. + if is_flax_module and dc_field.name in ("name", "parent"): + continue + + default, is_default_from_default_instance = _get_dataclass_field_default( + dc_field, info.default + ) + + # Try to get helptext from field metadata. This is also intended to be + # compatible with HuggingFace-style config objects. + helptext = dc_field.metadata.get("help", None) + assert isinstance(helptext, (str, type(None))) + + # Try to get helptext from docstrings. Note that this can't be generated + # dynamically. + if helptext is None: + helptext = _docstrings.get_field_docstring(info.type, dc_field.name) + + assert not isinstance(dc_field.type, str) + field_list.append( + StructFieldSpec( + name=dc_field.name, + type=dc_field.type, + default=default, + is_default_overridden=is_default_from_default_instance, + helptext=helptext, + ) + ) + return StructConstructorSpec(instantiate=info.type, fields=tuple(field_list)) + + @registry.struct_rule + def typeddict_rule(info: StructTypeInfo) -> StructConstructorSpec | None: + # Is this a TypedDict? + if not is_typeddict(info.type): + return None + + cls = cast(type, info.type) + + # Handle TypedDicts. + field_list = [] + valid_default_instance = ( + info.default not in MISSING_SINGLETONS + and info.default is not EXCLUDE_FROM_CALL + ) + assert not valid_default_instance or isinstance(info.default, dict) + total = getattr(cls, "__total__", True) + assert isinstance(total, bool) + assert not valid_default_instance or isinstance(info.default, dict) + for name, typ in _resolver.get_type_hints_with_backported_syntax( + cls, include_extras=True + ).items(): + typ_origin = get_origin(typ) + is_default_from_default_instance = False + if valid_default_instance and name in cast(dict, info.default): + default = cast(dict, info.default)[name] + is_default_from_default_instance = True + elif typ_origin is Required and total is False: + # Support total=False. + default = MISSING_PROP + elif total is False: + # Support total=False. + default = EXCLUDE_FROM_CALL + if is_struct_type(typ, MISSING_NONPROP): + # total=False behavior is unideal for nested structures. + pass + # raise _instantiators.UnsupportedTypeAnnotationError( + # "`total=False` not supported for nested structures." + # ) + elif typ_origin is NotRequired: + # Support typing.NotRequired[]. + default = EXCLUDE_FROM_CALL + else: + default = MISSING_PROP + + # Nested types need to be populated / can't be excluded from the call. + if default is EXCLUDE_FROM_CALL and is_struct_type(typ, MISSING_NONPROP): + default = MISSING_NONPROP + + if typ_origin in (Required, NotRequired): + args = get_args(typ) + assert ( + len(args) == 1 + ), "typing.Required[] and typing.NotRequired[T] require a concrete type T." + typ = args[0] + del args + + field_list.append( + StructFieldSpec( + name=name, + type=typ, + default=default, + is_default_overridden=is_default_from_default_instance, + helptext=_docstrings.get_field_docstring(cls, name), + ) + ) + return StructConstructorSpec(instantiate=info.type, fields=tuple(field_list)) + + @registry.struct_rule + def attrs_rule(info: StructTypeInfo) -> StructConstructorSpec | None: + # attr will already be imported if it's used. + if "attr" not in sys.modules.keys(): # pragma: no cover + return None + + try: + import attr + except ImportError: + # This is needed for the mock import test in + # test_missing_optional_packages.py to pass. + return None + + if not attr.has(info.type): + return None + + # Resolve forward references in-place, if any exist. + attr.resolve_types(info.type) + + # Handle attr classes. + field_list = [] + for attr_field in attr.fields(info.type): + # Skip fields with init=False. + if not attr_field.init: + continue + + # Default handling. + name = attr_field.name + default = attr_field.default + is_default_from_default_instance = False + if info.default not in MISSING_SINGLETONS: + if hasattr(info.default, name): + default = getattr(info.default, name) + is_default_from_default_instance = True + else: + warnings.warn( + f"Could not find field {name} in default instance" + f" {info.default}, which has" + f" type {type(info.default)},", + stacklevel=2, + ) + elif default is attr.NOTHING: + default = MISSING_NONPROP + elif isinstance(default, attr.Factory): # type: ignore + default = default.factory() # type: ignore + + assert attr_field.type is not None, attr_field + field_list.append( + StructFieldSpec( + name=name, + type=attr_field.type, + default=default, + is_default_overridden=is_default_from_default_instance, + helptext=_docstrings.get_field_docstring(info.type, name), + ) + ) + return StructConstructorSpec(instantiate=info.type, fields=tuple(field_list)) + + @registry.struct_rule + def dict_rule(info: StructTypeInfo) -> StructConstructorSpec | None: + if is_typeddict(info.type) or ( + info.type + not in ( + dict, + collections.abc.Mapping, + ) + and get_origin(info.type) + not in ( + dict, + collections.abc.Mapping, + ) + ): + return None + + if info.default in MISSING_SINGLETONS or len(info.default) == 0: + return None + + field_list = [] + for k, v in info.default.items(): + field_list.append( + StructFieldSpec( + name=str(k) if not isinstance(k, enum.Enum) else k.name, + type=type(v), + default=v, + is_default_overridden=True, + helptext=None, + _call_argname=k, + ) + ) + return StructConstructorSpec(instantiate=dict, fields=tuple(field_list)) + + @registry.struct_rule + def namedtuple_rule(info: StructTypeInfo) -> StructConstructorSpec | None: + if not ( + isinstance(info.type, type) + and issubclass(info.type, tuple) + and hasattr(info.type, "_fields") + ): + return None + + field_list = [] + field_defaults = getattr(info.type, "_field_defaults", {}) + + for name, typ in _resolver.get_type_hints_with_backported_syntax( + info.type, include_extras=True + ).items(): + default = field_defaults.get(name, MISSING_NONPROP) + is_default_from_default_instance = False + + if info.default not in MISSING_SINGLETONS and hasattr(info.default, name): + default = getattr(info.default, name) + is_default_from_default_instance = True + elif info.default is MISSING_PROP: + default = MISSING_PROP + + field_list.append( + StructFieldSpec( + name=name, + type=typ, + default=default, + is_default_overridden=is_default_from_default_instance, + helptext=_docstrings.get_field_docstring(info.type, name), + ) + ) + + return StructConstructorSpec(instantiate=info.type, fields=tuple(field_list)) + + @registry.struct_rule + def sequence_rule(info: StructTypeInfo) -> StructConstructorSpec | None: + if get_origin(info.type) not in ( + list, + set, + tuple, + Sequence, + collections.abc.Sequence, + ) or not isinstance(info.default, Iterable): + return None + + contained_type = get_args(info.type)[0] if get_args(info.type) else Any + + if all(not is_struct_type(type(x), x) for x in info.default): + return None + + field_list = [] + for i, default_i in enumerate(info.default): + field_list.append( + StructFieldSpec( + name=str(i), + type=cast(type, contained_type), + default=default_i, + is_default_overridden=True, + helptext="", + ) + ) + + return StructConstructorSpec( + instantiate=type(info.default), fields=tuple(field_list) + ) + + @registry.struct_rule + def tuple_rule(info: StructTypeInfo) -> StructConstructorSpec | None: + # It's important that this tuple rule is defined *after* the general sequence rule. It should take precedence. + if info.type is not tuple and get_origin(info.type) is not tuple: + return None + + # Fixed-length tuples. + children = get_args(info.type) + if Ellipsis in children: + return None # We don't handle variable-length tuples here + + # Infer more specific type when tuple annotation isn't subscripted. + if len(children) == 0: + if info.default in MISSING_SINGLETONS: + return None + else: + assert isinstance(info.default, tuple) + children = tuple(type(x) for x in info.default) + + if info.default in MISSING_SINGLETONS or info.default is EXCLUDE_FROM_CALL: + default_instance = (info.default,) * len(children) + else: + default_instance = info.default + + field_list: list[StructFieldSpec] = [] + for i, child in enumerate(children): + default_i = default_instance[i] + field_list.append( + StructFieldSpec( + name=str(i), + type=child, + default=default_i, + is_default_overridden=True, + helptext="", + ) + ) + + contains_nested = False + for field in field_list: + # Inefficient, since is_struct_type will compute StructTypeInfo again. + field_info = StructTypeInfo.make(field.type, field.default) + if get_origin(field_info.type) is Union: + for option in get_args(field_info.type): + # The second argument here is the default value, which can help with + # narrowing but is generall not necessary. + contains_nested |= is_struct_type(option, MISSING_NONPROP) + contains_nested |= is_struct_type(field.type, field.default) + if contains_nested: + break + + if not contains_nested: + return None + return StructConstructorSpec(instantiate=tuple, fields=tuple(field_list)) + + @registry.struct_rule + def pydantic_rule(info: StructTypeInfo) -> StructConstructorSpec | None: + # Check if pydantic is imported + if "pydantic" not in sys.modules.keys(): # pragma: no cover + return None + + try: + import pydantic + except ImportError: + # Needed for the mock import test in + # test_missing_optional_packages.py to pass. + return None + + try: + if "pydantic.v1" in sys.modules.keys(): + from pydantic import v1 as pydantic_v1 + else: # pragma: no cover + pydantic_v1 = None # type: ignore + except ImportError: + pydantic_v1 = None # type: ignore + + # Check if the type is a Pydantic model + try: + if not ( + issubclass(info.type, pydantic.BaseModel) + or ( + pydantic_v1 is not None + and issubclass(info.type, pydantic_v1.BaseModel) + ) + ): + return None + except TypeError: + # issubclass failed! + return None + + field_list = [] + pydantic_version = int( + getattr(pydantic, "__version__", "1.0.0").partition(".")[0] + ) + + if pydantic_version < 2 or ( + pydantic_v1 is not None and issubclass(info.type, pydantic_v1.BaseModel) + ): + # Pydantic 1.xx + cls_cast = info.type + hints = _resolver.get_type_hints_with_backported_syntax( + info.type, include_extras=True + ) + for pd1_field in cast(Dict[str, Any], cls_cast.__fields__).values(): + helptext = pd1_field.field_info.description + if helptext is None: + helptext = _docstrings.get_field_docstring( + info.type, pd1_field.name + ) + + default, is_default_from_default_instance = ( + _get_pydantic_v1_field_default( + pd1_field.name, pd1_field, info.default + ) + ) + field_list.append( + StructFieldSpec( + name=pd1_field.name, + type=hints[pd1_field.name], + default=default, + is_default_overridden=is_default_from_default_instance, + helptext=helptext, + ) + ) + else: + # Pydantic 2.xx + for name, pd2_field in cast(Any, info.type).model_fields.items(): + helptext = pd2_field.description + if helptext is None: + helptext = _docstrings.get_field_docstring(info.type, name) + + default, is_default_from_default_instance = ( + _get_pydantic_v2_field_default(name, pd2_field, info.default) + ) + field_list.append( + StructFieldSpec( + name=name, + type=( + Annotated.__class_getitem__( # type: ignore + (pd2_field.annotation,) + tuple(pd2_field.metadata) + ) + if len(pd2_field.metadata) > 0 + else pd2_field.annotation + ), + default=default, + is_default_overridden=is_default_from_default_instance, + helptext=helptext, + ) + ) + + return StructConstructorSpec(instantiate=info.type, fields=tuple(field_list)) + + +def _ensure_dataclass_instance_used_as_default_is_frozen( + field: dataclasses.Field, default_instance: Any +) -> None: + """Ensure that a dataclass type used directly as a default value is marked as + frozen.""" + assert dataclasses.is_dataclass(default_instance) + cls = type(default_instance) + if not cls.__dataclass_params__.frozen: # type: ignore + warnings.warn( + f"Mutable type {cls} is used as a default value for `{field.name}`. This is" + " dangerous! Consider using `dataclasses.field(default_factory=...)` or" + f" marking {cls} as frozen." + ) + + +def _get_dataclass_field_default( + field: dataclasses.Field, parent_default_instance: Any +) -> tuple[Any, bool]: + """Helper for getting the default instance for a dataclass field.""" + # If the dataclass's parent is explicitly marked MISSING, mark this field as missing + # as well. + if parent_default_instance is MISSING_PROP: + return MISSING_PROP, False + + # Try grabbing default from parent instance. + if ( + parent_default_instance not in MISSING_SINGLETONS + and parent_default_instance is not None + ): + # Populate default from some parent, eg `default=` in `tyro.cli()`. + if hasattr(parent_default_instance, field.name): + return getattr(parent_default_instance, field.name), True + else: + warnings.warn( + f"Could not find field {field.name} in default instance" + f" {parent_default_instance}, which has" + f" type {type(parent_default_instance)},", + stacklevel=2, + ) + + # Try grabbing default from dataclass field. + if field.default not in MISSING_SINGLETONS: + default = field.default + # Note that dataclasses.is_dataclass() will also return true for dataclass + # _types_, not just instances. + if type(default) is not type and dataclasses.is_dataclass(default): + _ensure_dataclass_instance_used_as_default_is_frozen(field, default) + return default, False + + # Populate default from `dataclasses.field(default_factory=...)`. + if field.default_factory is not dataclasses.MISSING and not ( + # Special case to ignore default_factory if we write: + # `field: Dataclass = dataclasses.field(default_factory=Dataclass)`. + # + # In other words, treat it the same way as: `field: Dataclass`. + # + # The only time this matters is when we our dataclass has a `__post_init__` + # function that mutates the dataclass. We choose here to use the default values + # before this method is called. + dataclasses.is_dataclass(field.type) and field.default_factory is field.type + ): + return field.default_factory(), False + + # Otherwise, no default. This is different from MISSING, because MISSING propagates + # to children. We could revisit this design to make it clearer. + return MISSING_NONPROP, False + + +if TYPE_CHECKING: + import pydantic as pydantic + import pydantic.v1.fields as pydantic_v1_fields + + +def _get_pydantic_v1_field_default( + name: str, + field: pydantic_v1_fields.ModelField, + parent_default_instance: Any, +) -> tuple[Any, bool]: + """Helper for getting the default instance for a Pydantic field.""" + + # Try grabbing default from parent instance. + if ( + parent_default_instance not in MISSING_SINGLETONS + and parent_default_instance is not None + ): + # Populate default from some parent, eg `default=` in `tyro.cli()`. + if hasattr(parent_default_instance, name): + return getattr(parent_default_instance, name), True + else: + warnings.warn( + f"Could not find field {name} in default instance" + f" {parent_default_instance}, which has" + f" type {type(parent_default_instance)},", + stacklevel=2, + ) + + if not field.required: + return field.get_default(), False + + # Otherwise, no default. + return MISSING_NONPROP, False + + +def _get_pydantic_v2_field_default( + name: str, + field: pydantic.fields.FieldInfo, + parent_default_instance: Any, +) -> tuple[Any, bool]: + """Helper for getting the default instance for a Pydantic field.""" + + # Try grabbing default from parent instance. + if ( + parent_default_instance not in MISSING_SINGLETONS + and parent_default_instance is not None + ): + # Populate default from some parent, eg `default=` in `tyro.cli()`. + if hasattr(parent_default_instance, name): + return getattr(parent_default_instance, name), True + else: + warnings.warn( + f"Could not find field {name} in default instance" + f" {parent_default_instance}, which has" + f" type {type(parent_default_instance)},", + stacklevel=2, + ) + + if not field.is_required(): + return field.get_default(call_default_factory=True), False + + # Otherwise, no default. + return MISSING_NONPROP, False diff --git a/src/tyro/extras/_serialization.py b/src/tyro/extras/_serialization.py index bfb5a28d2..352f974ab 100644 --- a/src/tyro/extras/_serialization.py +++ b/src/tyro/extras/_serialization.py @@ -13,7 +13,8 @@ from typing_extensions import get_args, get_origin -from .. import _fields, _resolver +from .. import _resolver +from ..constructors._struct_spec import MISSING_PROP ENUM_YAML_TAG_PREFIX = "!enum:" DATACLASS_YAML_TAG_PREFIX = "!dataclass:" @@ -118,7 +119,7 @@ def make_enum_constructor(typ: Type[enum.Enum]): DataclassLoader.add_constructor( tag=MISSING_YAML_TAG_PREFIX, - constructor=lambda *_unused: _fields.MISSING_PROP, # type: ignore + constructor=lambda *_unused: MISSING_PROP, # type: ignore ) return DataclassLoader @@ -129,7 +130,7 @@ def _make_dumper(instance: Any) -> Type[yaml.Dumper]: class DataclassDumper(yaml.Dumper): def ignore_aliases(self, data): - return super().ignore_aliases(data) or data is _fields.MISSING_PROP + return super().ignore_aliases(data) or data is MISSING_PROP contained_types = list(_get_contained_special_types_from_type(type(instance))) contained_type_names = list(map(lambda cls: cls.__name__, contained_types)) @@ -163,7 +164,7 @@ def representer(dumper: DataclassDumper, data: Any) -> yaml.Node: DataclassDumper.add_representer(typ, make_representer(name)) DataclassDumper.add_representer( - type(_fields.MISSING_PROP), + type(MISSING_PROP), lambda dumper, data: dumper.represent_scalar( tag=MISSING_YAML_TAG_PREFIX, value="" ), diff --git a/tests/test_custom_primitive.py b/tests/test_custom_primitive.py index c9c9d1f6d..a2758999d 100644 --- a/tests/test_custom_primitive.py +++ b/tests/test_custom_primitive.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json from typing import Any, Dict @@ -16,16 +18,16 @@ def test_custom_primitive_registry(): """Test that we can use a custom primitive registry to parse a custom type.""" - primitive_registry = tyro.constructors.PrimitiveConstructorRegistry() + primitive_registry = tyro.constructors.ConstructorRegistry() - @primitive_registry.define_rule( - matcher_fn=lambda type_info: type_info.type_origin is dict - and get_args(type_info.type) == (str, Any) - ) + @primitive_registry.primitive_rule def json_dict_spec( type_info: tyro.constructors.PrimitiveTypeInfo, - ) -> tyro.constructors.PrimitiveConstructorSpec: - del type_info # We won't use type_info. + ) -> tyro.constructors.PrimitiveConstructorSpec | None: + if not ( + type_info.type_origin is dict and get_args(type_info.type) == (str, Any) + ): + return None return json_constructor_spec def main(x: Dict[str, Any]) -> Dict[str, Any]: diff --git a/tests/test_errors.py b/tests/test_errors.py index 85c5a479f..1e4c9fd5a 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -7,6 +7,7 @@ from typing_extensions import Literal import tyro +from tyro.constructors import UnsupportedTypeAnnotationError def test_ambiguous_collection_0() -> None: @@ -14,7 +15,7 @@ def test_ambiguous_collection_0() -> None: class A: x: Tuple[Tuple[int, ...], ...] - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(A, args=["--x", "0", "1"]) @@ -23,7 +24,7 @@ def test_ambiguous_collection_1() -> None: class A: x: List[List[int]] - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(A, args=["--x", "0", "1"]) @@ -31,7 +32,7 @@ def test_ambiguous_collection_2() -> None: def main(x: Tuple[List[str], List[str]]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError) as e: + with pytest.raises(UnsupportedTypeAnnotationError) as e: tyro.cli(main, args=["--help"]) assert "Unsupported type annotation for the field x" in e.value.args[0] @@ -40,14 +41,14 @@ def test_ambiguous_collection_3() -> None: def main(x: List[Union[Tuple[int, int], Tuple[int, int, int]]]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError) as e: + with pytest.raises(UnsupportedTypeAnnotationError) as e: tyro.cli(main, args=["--help"]) assert "Unsupported type annotation for the field x" in e.value.args[0] def test_ambiguous_collection_4() -> None: X = List[Union[Tuple[int, int], Tuple[int, int, int]]] - with pytest.raises(tyro.UnsupportedTypeAnnotationError) as e: + with pytest.raises(UnsupportedTypeAnnotationError) as e: tyro.cli(X, args=["--help"]) assert "Unsupported type annotation for the field" not in e.value.args[0] @@ -57,7 +58,7 @@ def test_ambiguous_collection_5() -> None: class A: x: Dict[List[int], str] - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(A, args=["--x", "0", "1"]) @@ -66,7 +67,7 @@ def test_ambiguous_collection_6() -> None: class A: x: Dict[str, List[int]] - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(A, args=["--x", "0", "1"]) @@ -77,7 +78,7 @@ class _CycleDataclass: def test_cycle() -> None: - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(_CycleDataclass, args=[]) @@ -85,7 +86,7 @@ def test_uncallable_annotation() -> None: def main(arg: 5) -> None: # type: ignore pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=[]) @@ -97,7 +98,7 @@ class OneIntArg: def main(arg: List[OneIntArg]) -> List[OneIntArg]: # type: ignore return arg - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=[]) @dataclasses.dataclass @@ -107,7 +108,7 @@ class OneStringArg: def main(arg: List[OneStringArg]) -> List[OneStringArg]: # type: ignore return arg - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=["--arg", "0", "1", "2"]) @dataclasses.dataclass @@ -118,7 +119,7 @@ class TwoStringArg: def main2(arg: List[TwoStringArg]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main2, args=[]) @@ -126,7 +127,7 @@ def test_missing_annotation_1() -> None: def main(a, b) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) @@ -134,7 +135,7 @@ def test_missing_annotation_2() -> None: def main(*, a) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) @@ -143,7 +144,7 @@ def main(arg: tuple) -> None: # type: ignore pass # This formerly raised an error, but now defaults to Tuple[str, ...]. - # with pytest.raises(tyro.UnsupportedTypeAnnotationError): + # with pytest.raises(UnsupportedTypeAnnotationError): with pytest.raises(SystemExit): tyro.cli(main, args=["--help"]) @@ -154,7 +155,7 @@ def test_unbound_typevar() -> None: def main(arg: T) -> None: # type: ignore pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) @@ -162,7 +163,7 @@ def test_missing_default_fixed() -> None: def main(value: tyro.conf.SuppressFixed[tyro.conf.Fixed[int]]) -> int: return value - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) @@ -170,7 +171,7 @@ def test_missing_default_suppressed() -> None: def main(value: tyro.conf.Suppress[int]) -> int: return value - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) @@ -179,7 +180,7 @@ def main(value: list) -> None: return None # This formerly raised an error, but now defaults to List[str]. - # with pytest.raises(tyro.UnsupportedTypeAnnotationError): + # with pytest.raises(UnsupportedTypeAnnotationError): with pytest.raises(SystemExit): tyro.cli(main, args=["--help"]) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 20e65909b..588018f29 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -671,7 +671,7 @@ def main(x: Any = Struct()): pass helptext = get_helptext_with_checks(main) - assert "--x {fixed}" in helptext + assert "--x {fixed}" not in helptext def main2(x: Callable = nn.ReLU): pass diff --git a/tests/test_is_struct_type.py b/tests/test_is_struct_type.py index 5b0736004..2ce8c3e49 100644 --- a/tests/test_is_struct_type.py +++ b/tests/test_is_struct_type.py @@ -2,7 +2,8 @@ import pathlib from typing import Any, Dict, List, Tuple -from tyro._fields import MISSING_NONPROP, is_struct_type +from tyro._fields import is_struct_type +from tyro._singleton import MISSING_NONPROP def test_is_struct_type_simple(): diff --git a/tests/test_nested_in_containers.py b/tests/test_nested_in_containers.py index 819998176..21bcbc67b 100644 --- a/tests/test_nested_in_containers.py +++ b/tests/test_nested_in_containers.py @@ -5,6 +5,7 @@ import pytest import tyro +from tyro.constructors import UnsupportedTypeAnnotationError @dataclasses.dataclass(frozen=True) @@ -79,7 +80,7 @@ def test_tuple_bad() -> None: def main(x: Tuple[Color, ...]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=[]) @@ -88,7 +89,7 @@ def test_set_bad() -> None: def main(x: Set[Color]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=[]) @@ -105,7 +106,7 @@ def test_list_bad() -> None: def main(x: List[Color]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=[]) @@ -156,7 +157,7 @@ def test_dict_bad() -> None: def main(x: Dict[str, Color]) -> Any: return x - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=[]) diff --git a/tests/test_new_style_annotations_min_py312.py b/tests/test_new_style_annotations_min_py312.py index 7efee58b8..0e2f4e8c7 100644 --- a/tests/test_new_style_annotations_min_py312.py +++ b/tests/test_new_style_annotations_min_py312.py @@ -8,6 +8,7 @@ import tyro from tyro.conf._markers import OmitArgPrefixes +from tyro.constructors import UnsupportedTypeAnnotationError def test_simple_generic(): @@ -143,7 +144,7 @@ def test_pep695_recursive_types() -> None: class Config: arg: RecursiveList[str] - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(Config, args=["--arg", "True"]) diff --git a/tests/test_py311_generated/test_custom_primitive_generated.py b/tests/test_py311_generated/test_custom_primitive_generated.py index f36124df9..4244759a2 100644 --- a/tests/test_py311_generated/test_custom_primitive_generated.py +++ b/tests/test_py311_generated/test_custom_primitive_generated.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json from typing import Annotated, Any, Dict, get_args @@ -14,16 +16,16 @@ def test_custom_primitive_registry(): """Test that we can use a custom primitive registry to parse a custom type.""" - primitive_registry = tyro.constructors.PrimitiveConstructorRegistry() + primitive_registry = tyro.constructors.ConstructorRegistry() - @primitive_registry.define_rule( - matcher_fn=lambda type_info: type_info.type_origin is dict - and get_args(type_info.type) == (str, Any) - ) + @primitive_registry.primitive_rule def json_dict_spec( type_info: tyro.constructors.PrimitiveTypeInfo, - ) -> tyro.constructors.PrimitiveConstructorSpec: - del type_info # We won't use type_info. + ) -> tyro.constructors.PrimitiveConstructorSpec | None: + if not ( + type_info.type_origin is dict and get_args(type_info.type) == (str, Any) + ): + return None return json_constructor_spec def main(x: Dict[str, Any]) -> Dict[str, Any]: diff --git a/tests/test_py311_generated/test_errors_generated.py b/tests/test_py311_generated/test_errors_generated.py index 398131ff4..81fdd1502 100644 --- a/tests/test_py311_generated/test_errors_generated.py +++ b/tests/test_py311_generated/test_errors_generated.py @@ -6,6 +6,7 @@ import pytest import tyro +from tyro.constructors import UnsupportedTypeAnnotationError def test_ambiguous_collection_0() -> None: @@ -13,7 +14,7 @@ def test_ambiguous_collection_0() -> None: class A: x: Tuple[Tuple[int, ...], ...] - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(A, args=["--x", "0", "1"]) @@ -22,7 +23,7 @@ def test_ambiguous_collection_1() -> None: class A: x: List[List[int]] - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(A, args=["--x", "0", "1"]) @@ -30,7 +31,7 @@ def test_ambiguous_collection_2() -> None: def main(x: Tuple[List[str], List[str]]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError) as e: + with pytest.raises(UnsupportedTypeAnnotationError) as e: tyro.cli(main, args=["--help"]) assert "Unsupported type annotation for the field x" in e.value.args[0] @@ -39,14 +40,14 @@ def test_ambiguous_collection_3() -> None: def main(x: List[Tuple[int, int] | Tuple[int, int, int]]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError) as e: + with pytest.raises(UnsupportedTypeAnnotationError) as e: tyro.cli(main, args=["--help"]) assert "Unsupported type annotation for the field x" in e.value.args[0] def test_ambiguous_collection_4() -> None: X = List[Tuple[int, int] | Tuple[int, int, int]] - with pytest.raises(tyro.UnsupportedTypeAnnotationError) as e: + with pytest.raises(UnsupportedTypeAnnotationError) as e: tyro.cli(X, args=["--help"]) assert "Unsupported type annotation for the field" not in e.value.args[0] @@ -56,7 +57,7 @@ def test_ambiguous_collection_5() -> None: class A: x: Dict[List[int], str] - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(A, args=["--x", "0", "1"]) @@ -65,7 +66,7 @@ def test_ambiguous_collection_6() -> None: class A: x: Dict[str, List[int]] - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(A, args=["--x", "0", "1"]) @@ -76,7 +77,7 @@ class _CycleDataclass: def test_cycle() -> None: - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(_CycleDataclass, args=[]) @@ -84,7 +85,7 @@ def test_uncallable_annotation() -> None: def main(arg: 5) -> None: # type: ignore pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=[]) @@ -96,7 +97,7 @@ class OneIntArg: def main(arg: List[OneIntArg]) -> List[OneIntArg]: # type: ignore return arg - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=[]) @dataclasses.dataclass @@ -106,7 +107,7 @@ class OneStringArg: def main(arg: List[OneStringArg]) -> List[OneStringArg]: # type: ignore return arg - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=["--arg", "0", "1", "2"]) @dataclasses.dataclass @@ -117,7 +118,7 @@ class TwoStringArg: def main2(arg: List[TwoStringArg]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main2, args=[]) @@ -125,7 +126,7 @@ def test_missing_annotation_1() -> None: def main(a, b) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) @@ -133,7 +134,7 @@ def test_missing_annotation_2() -> None: def main(*, a) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) @@ -142,7 +143,7 @@ def main(arg: tuple) -> None: # type: ignore pass # This formerly raised an error, but now defaults to Tuple[str, ...]. - # with pytest.raises(tyro.UnsupportedTypeAnnotationError): + # with pytest.raises(UnsupportedTypeAnnotationError): with pytest.raises(SystemExit): tyro.cli(main, args=["--help"]) @@ -153,7 +154,7 @@ def test_unbound_typevar() -> None: def main(arg: T) -> None: # type: ignore pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) @@ -161,7 +162,7 @@ def test_missing_default_fixed() -> None: def main(value: tyro.conf.SuppressFixed[tyro.conf.Fixed[int]]) -> int: return value - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) @@ -169,7 +170,7 @@ def test_missing_default_suppressed() -> None: def main(value: tyro.conf.Suppress[int]) -> int: return value - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=["--help"]) @@ -178,7 +179,7 @@ def main(value: list) -> None: return None # This formerly raised an error, but now defaults to List[str]. - # with pytest.raises(tyro.UnsupportedTypeAnnotationError): + # with pytest.raises(UnsupportedTypeAnnotationError): with pytest.raises(SystemExit): tyro.cli(main, args=["--help"]) diff --git a/tests/test_py311_generated/test_errors_new_annotations_generated.py b/tests/test_py311_generated/test_errors_new_annotations_generated.py deleted file mode 100644 index da4aee733..000000000 --- a/tests/test_py311_generated/test_errors_new_annotations_generated.py +++ /dev/null @@ -1,33 +0,0 @@ -from __future__ import annotations - -import sys - -import pytest - -import tyro - - -@pytest.mark.skipif( - sys.version_info >= (3, 10), reason="No error for newer versions of Python." -) -def test_new_union_error() -> None: - """PEP 604 allows `|` to be used as a type annotation in Python >=3.10.""" - - def main(x: int | str) -> None: ... - - with pytest.raises(TypeError) as e: - tyro.cli(main) - assert "You may be using a Union in the form of `X | Y`" in e.value.args[0] - - -@pytest.mark.skipif( - sys.version_info >= (3, 9), reason="No error for newer versions of Python." -) -def test_new_collection_error() -> None: - """PEP 585 allows standard collections to be used as generics in Python >=3.9.""" - - def main(x: list[int]) -> None: ... - - with pytest.raises(TypeError) as e: - tyro.cli(main) - assert "You may be using a standard collection as a generic" in e.value.args[0] diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py index 4039e42eb..7ef04eca7 100644 --- a/tests/test_py311_generated/test_helptext_generated.py +++ b/tests/test_py311_generated/test_helptext_generated.py @@ -672,7 +672,7 @@ def main(x: Any = Struct()): pass helptext = get_helptext_with_checks(main) - assert "--x {fixed}" in helptext + assert "--x {fixed}" not in helptext def main2(x: Callable = nn.ReLU): pass diff --git a/tests/test_py311_generated/test_is_nested_type_generated.py b/tests/test_py311_generated/test_is_nested_type_generated.py deleted file mode 100644 index 37bedb6c6..000000000 --- a/tests/test_py311_generated/test_is_nested_type_generated.py +++ /dev/null @@ -1,52 +0,0 @@ -import dataclasses -import pathlib -from typing import Any, Dict, List, Tuple - -from tyro._fields import MISSING_NONPROP, is_struct_type - - -def test_is_nested_type_simple(): - assert not is_struct_type(int, MISSING_NONPROP) - assert not is_struct_type(bool, MISSING_NONPROP) - assert not is_struct_type(str, MISSING_NONPROP) - assert not is_struct_type(pathlib.Path, MISSING_NONPROP) - - -def test_is_nested_type_containers(): - assert not is_struct_type(List[int], MISSING_NONPROP) - assert not is_struct_type(List[bool], MISSING_NONPROP) - assert not is_struct_type(List[str], MISSING_NONPROP) - assert not is_struct_type(List[pathlib.Path], MISSING_NONPROP) - - -@dataclasses.dataclass -class Color: - r: int - g: int - b: int - - -def test_is_nested_type_actually_nested(): - assert is_struct_type(Color, Color(255, 0, 0)) - - -def test_is_nested_type_actually_nested_narrowing(): - assert is_struct_type(Any, Color(255, 0, 0)) - assert is_struct_type(object, Color(255, 0, 0)) - assert not is_struct_type(int, Color(255, 0, 0)) - - -def test_is_nested_type_actually_nested_in_container(): - assert is_struct_type(Tuple[Color, Color], MISSING_NONPROP) - assert is_struct_type(Tuple[object, ...], (Color(255, 0, 0),)) - assert is_struct_type(Tuple[Any, ...], (Color(255, 0, 0),)) - assert is_struct_type(tuple, (Color(255, 0, 0),)) - assert not is_struct_type(tuple, (1, 2, 3)) - assert is_struct_type(tuple, (1, Color(255, 0, 0), 3)) - assert is_struct_type(List[Any], [Color(255, 0, 0)]) - - -def test_nested_dict(): - assert is_struct_type(Dict[str, int], {"x": 5}) - assert is_struct_type(dict, {"x": 5}) - assert is_struct_type(Any, {"x": 5}) diff --git a/tests/test_py311_generated/test_is_struct_type_generated.py b/tests/test_py311_generated/test_is_struct_type_generated.py index 5b0736004..2ce8c3e49 100644 --- a/tests/test_py311_generated/test_is_struct_type_generated.py +++ b/tests/test_py311_generated/test_is_struct_type_generated.py @@ -2,7 +2,8 @@ import pathlib from typing import Any, Dict, List, Tuple -from tyro._fields import MISSING_NONPROP, is_struct_type +from tyro._fields import is_struct_type +from tyro._singleton import MISSING_NONPROP def test_is_struct_type_simple(): diff --git a/tests/test_py311_generated/test_nested_in_containers_generated.py b/tests/test_py311_generated/test_nested_in_containers_generated.py index 819998176..21bcbc67b 100644 --- a/tests/test_py311_generated/test_nested_in_containers_generated.py +++ b/tests/test_py311_generated/test_nested_in_containers_generated.py @@ -5,6 +5,7 @@ import pytest import tyro +from tyro.constructors import UnsupportedTypeAnnotationError @dataclasses.dataclass(frozen=True) @@ -79,7 +80,7 @@ def test_tuple_bad() -> None: def main(x: Tuple[Color, ...]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=[]) @@ -88,7 +89,7 @@ def test_set_bad() -> None: def main(x: Set[Color]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=[]) @@ -105,7 +106,7 @@ def test_list_bad() -> None: def main(x: List[Color]) -> None: pass - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=[]) @@ -156,7 +157,7 @@ def test_dict_bad() -> None: def main(x: Dict[str, Color]) -> Any: return x - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(main, args=[]) diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py index 7efee58b8..0e2f4e8c7 100644 --- a/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py +++ b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py @@ -8,6 +8,7 @@ import tyro from tyro.conf._markers import OmitArgPrefixes +from tyro.constructors import UnsupportedTypeAnnotationError def test_simple_generic(): @@ -143,7 +144,7 @@ def test_pep695_recursive_types() -> None: class Config: arg: RecursiveList[str] - with pytest.raises(tyro.UnsupportedTypeAnnotationError): + with pytest.raises(UnsupportedTypeAnnotationError): tyro.cli(Config, args=["--arg", "True"]) diff --git a/tests/test_py311_generated/test_pydantic_generic_generated.py b/tests/test_py311_generated/test_pydantic_generic_generated.py new file mode 100644 index 000000000..44bae51b8 --- /dev/null +++ b/tests/test_py311_generated/test_pydantic_generic_generated.py @@ -0,0 +1,18 @@ +from typing import Generic, TypeVar + +from pydantic import BaseModel + +import tyro + +T = TypeVar("T") + + +def test_pydantic_generic() -> None: + class ManyTypesA(BaseModel, Generic[T]): + i: T + s: str = "hello" + f: float = 3.0 + + assert tyro.cli(ManyTypesA[int], args=["--i", "5"]) == ManyTypesA( + i=5, s="hello", f=3.0 + ) diff --git a/tests/test_pydantic_generic.py b/tests/test_pydantic_generic.py new file mode 100644 index 000000000..44bae51b8 --- /dev/null +++ b/tests/test_pydantic_generic.py @@ -0,0 +1,18 @@ +from typing import Generic, TypeVar + +from pydantic import BaseModel + +import tyro + +T = TypeVar("T") + + +def test_pydantic_generic() -> None: + class ManyTypesA(BaseModel, Generic[T]): + i: T + s: str = "hello" + f: float = 3.0 + + assert tyro.cli(ManyTypesA[int], args=["--i", "5"]) == ManyTypesA( + i=5, s="hello", f=3.0 + ) From 24bbac0aa01a97cc2609d53b13b8a7c5256a3c4a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Mon, 4 Nov 2024 09:53:35 -0800 Subject: [PATCH 470/491] Fix dictionary + numeric tower edge cases introduced by new primitive API (#192) --- src/tyro/constructors/_primitive_spec.py | 15 +++++++------- src/tyro/constructors/_struct_spec.py | 1 + tests/test_collections.py | 20 +++++++++++++++++++ .../test_collections_generated.py | 20 +++++++++++++++++++ 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/tyro/constructors/_primitive_spec.py b/src/tyro/constructors/_primitive_spec.py index 80575e374..729c29993 100644 --- a/src/tyro/constructors/_primitive_spec.py +++ b/src/tyro/constructors/_primitive_spec.py @@ -142,7 +142,10 @@ def basics_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None if type_info.type is bytes else type_info.type(args[0]) ), - is_instance=lambda x: isinstance(x, type_info.type), + # Numeric tower in Python is weird... + is_instance=lambda x: isinstance(x, (int, float)) + if type_info.type is float + else isinstance(x, type_info.type), str_from_instance=lambda instance: [str(instance)], ) @@ -502,13 +505,11 @@ def instance_from_str(args: list[str]) -> dict: return out def str_from_instance(instance: dict) -> list[str]: + # TODO: this may be strange right now for the append action. out: list[str] = [] - assert ( - len(instance) == 0 - ), "When parsed as a primitive, we currrently assume all defaults are length=0. Dictionaries with non-zero-length defaults are interpreted as struct types." - # for key, value in instance.items(): - # out.extend(key_spec.str_from_instance(key)) - # out.extend(val_spec.str_from_instance(value)) + for key, value in instance.items(): + out.extend(key_spec.str_from_instance(key)) + out.extend(val_spec.str_from_instance(value)) return out if _markers.UseAppendAction in type_info.markers: diff --git a/src/tyro/constructors/_struct_spec.py b/src/tyro/constructors/_struct_spec.py index 6a9aeecdb..06f0776f4 100644 --- a/src/tyro/constructors/_struct_spec.py +++ b/src/tyro/constructors/_struct_spec.py @@ -274,6 +274,7 @@ def dict_rule(info: StructTypeInfo) -> StructConstructorSpec | None: if is_typeddict(info.type) or ( info.type not in ( + Dict, dict, collections.abc.Mapping, ) diff --git a/tests/test_collections.py b/tests/test_collections.py index 2f84ffefb..300d14810 100644 --- a/tests/test_collections.py +++ b/tests/test_collections.py @@ -482,6 +482,26 @@ def main(x: Dict[str, Any] = {"int": 5, "str": "5"}): } +def test_dict_no_annotation_2() -> None: + def main(x: Dict = {"int": 5, "str": "5"}): + return x + + assert tyro.cli(main, args=[]) == {"int": 5, "str": "5"} + assert tyro.cli(main, args="--x.int 3 --x.str 7".split(" ")) == { + "int": 3, + "str": "7", + } + + +def test_dict_optional() -> None: + # In this case, the `None` is ignored. + def main(x: Optional[Dict[str, float]] = {"three": 3, "five": 5}): + return x + + assert tyro.cli(main, args=[]) == {"three": 3, "five": 5} + assert tyro.cli(main, args="--x 3 3 5 5".split(" ")) == {"3": 3, "5": 5} + + def test_double_dict_no_annotation() -> None: def main( x: Dict[str, Any] = { diff --git a/tests/test_py311_generated/test_collections_generated.py b/tests/test_py311_generated/test_collections_generated.py index 5b6e3d811..9aca7e8af 100644 --- a/tests/test_py311_generated/test_collections_generated.py +++ b/tests/test_py311_generated/test_collections_generated.py @@ -481,6 +481,26 @@ def main(x: Dict[str, Any] = {"int": 5, "str": "5"}): } +def test_dict_no_annotation_2() -> None: + def main(x: Dict = {"int": 5, "str": "5"}): + return x + + assert tyro.cli(main, args=[]) == {"int": 5, "str": "5"} + assert tyro.cli(main, args="--x.int 3 --x.str 7".split(" ")) == { + "int": 3, + "str": "7", + } + + +def test_dict_optional() -> None: + # In this case, the `None` is ignored. + def main(x: Optional[Dict[str, int]] = {"three": 3, "five": 5}): + return x + + assert tyro.cli(main, args=[]) == {"three": 3, "five": 5} + assert tyro.cli(main, args="--x 3 3 5 5".split(" ")) == {"3": 3, "5": 5} + + def test_double_dict_no_annotation() -> None: def main( x: Dict[str, Any] = { From 42a7f805830959e5ef34937c44586cd21e9430b7 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 5 Nov 2024 00:26:43 -0800 Subject: [PATCH 471/491] Numeric tower improvements + test (#193) * Improvements for numeric tower edge cases * Fix test for Python <3.10 --- src/tyro/_resolver.py | 38 ++++++++++++++++- src/tyro/constructors/_primitive_spec.py | 42 +++++++++++++------ tests/test_dcargs.py | 21 ++++++++++ .../test_collections_generated.py | 2 +- .../test_dcargs_generated.py | 21 ++++++++++ 5 files changed, 109 insertions(+), 15 deletions(-) diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 3fa2ea065..2473447fe 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -18,6 +18,7 @@ Sequence, Set, Tuple, + Type, TypeVar, Union, cast, @@ -473,7 +474,8 @@ def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCalla try: if default_instance not in MISSING_SINGLETONS and not any( - isinstance(default_instance, o) for o in options_unwrapped + isinstance_with_fuzzy_numeric_tower(default_instance, o) is not False + for o in options_unwrapped ): warnings.warn( f"{type(default_instance)} does not match any type in Union:" @@ -486,6 +488,40 @@ def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCalla return typ +def isinstance_with_fuzzy_numeric_tower( + obj: Any, classinfo: Type +) -> Union[bool, Literal["~"]]: + """ + Enhanced version of isinstance() that returns: + - True: if object is exactly of the specified type + - "~": if object follows numeric tower rules but isn't exact type + - False: if object is not of the specified type or numeric tower rules don't apply + + Examples: + >>> enhanced_isinstance(3, int) # Returns True + >>> enhanced_isinstance(3, float) # Returns "~" + >>> enhanced_isinstance(True, int) # Returns "~" + >>> enhanced_isinstance(3, bool) # Returns False + >>> enhanced_isinstance(True, bool) # Returns True + """ + # Handle exact match first + if isinstance(obj, classinfo): + return True + + # Handle numeric tower cases + if isinstance(obj, bool): + if classinfo in (int, float, complex): + return "~" + elif isinstance(obj, int) and not isinstance(obj, bool): # explicit bool check + if classinfo in (float, complex): + return "~" + elif isinstance(obj, float): + if classinfo is complex: + return "~" + + return False + + NoneType = type(None) diff --git a/src/tyro/constructors/_primitive_spec.py b/src/tyro/constructors/_primitive_spec.py index 729c29993..8c113d6e8 100644 --- a/src/tyro/constructors/_primitive_spec.py +++ b/src/tyro/constructors/_primitive_spec.py @@ -99,10 +99,15 @@ class PrimitiveConstructorSpec(Generic[T]): instance_from_str: Callable[[list[str]], T] """Given a list of string arguments, construct an instance of the type. The length of the list will match the value of nargs.""" - is_instance: Callable[[Any], bool] + is_instance: Callable[[Any], bool | Literal["~"]] """Given an object instance, does it match this primitive type? This is used for specific help messages when both a union type is present and a - default is provided.""" + default is provided. + + Can return "~" to signify that an instance is a "fuzzy" match, and should + only be used if there are no other matches. This is used for numeric tower + support. + """ str_from_instance: Callable[[T], list[str]] """Convert an instance to a list of string arguments that would construct the instance. This is used for help messages when a default is provided.""" @@ -124,11 +129,12 @@ def any_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: return None raise UnsupportedTypeAnnotationError("`Any` is not a parsable type.") - # HACK: this is for code that uses `tyro.conf.arg(constructor=json.loads)`. - # We're going to deprecate this syntax (the constructor= argument in - # tyro.conf.arg), but there is code that lives in the wild that relies - # on the behavior so we'll do our best not to break it. - vanilla_types = (int, str, float, bytes, json.loads) + # HACK (json.loads): this is for code that uses + # `tyro.conf.arg(constructor=json.loads)`. We're going to deprecate this + # syntax (the constructor= argument in tyro.conf.arg), but there is code + # that lives in the wild that relies on the behavior so we'll do our best + # not to break it. + vanilla_types = (int, str, float, complex, bytes, bytearray, json.loads) @registry.primitive_rule def basics_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: @@ -142,10 +148,11 @@ def basics_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None if type_info.type is bytes else type_info.type(args[0]) ), - # Numeric tower in Python is weird... - is_instance=lambda x: isinstance(x, (int, float)) - if type_info.type is float - else isinstance(x, type_info.type), + # issubclass(type(x), y) here is preferable over isinstance(x, y) + # due to quirks in the numeric tower. + is_instance=lambda x: _resolver.isinstance_with_fuzzy_numeric_tower( + x, type_info.type + ), str_from_instance=lambda instance: [str(instance)], ) @@ -582,7 +589,7 @@ def union_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: # General unions, eg Union[int, bool]. We'll try to convert these from left to # right. - option_specs = [] + option_specs: list[PrimitiveConstructorSpec] = [] choices: tuple[str, ...] | None = () nargs: int | Literal["*"] = 1 first = True @@ -646,9 +653,18 @@ def union_instantiator(strings: List[str]) -> Any: ) def str_from_instance(instance: Any) -> List[str]: + fuzzy_match = None for option_spec in option_specs: - if option_spec.is_instance(instance): + is_instance = option_spec.is_instance(instance) + if is_instance is True: return option_spec.str_from_instance(instance) + elif is_instance == "~": + fuzzy_match = option_spec + + # If we get here, we have a fuzzy match. + if fuzzy_match is not None: + return fuzzy_match.str_from_instance(instance) + assert False, f"could not match default value {instance} with any types in union {options}" return PrimitiveConstructorSpec( diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 432140881..f9c604971 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -942,3 +942,24 @@ def main(dt: datetime.time) -> datetime.time: # Invalid hour value. with pytest.raises(SystemExit): tyro.cli(main, args=["--dt", "25:00:00"]) + + +def test_numeric_tower() -> None: + @dataclasses.dataclass(frozen=True) + class NumericTower: + a: Union[complex, str] = 3.0 + b: Union[bytearray, str] = dataclasses.field( + default_factory=lambda: bytearray(b"123") + ) + c: Union[complex, str] = True + d: Union[int, complex] = False + e: Union[float, str] = 3 + + assert tyro.cli(NumericTower, args=[]) == NumericTower(3.0) + assert tyro.cli(NumericTower, args="--a 1+3j".split(" ")) == NumericTower(1 + 3j) + assert tyro.cli(NumericTower, args="--c False".split(" ")) == NumericTower( + c="False" + ) + assert tyro.cli(NumericTower, args="--e 3.2".split(" ")) == NumericTower(e=3.2) + with pytest.raises(SystemExit): + tyro.cli(NumericTower, args="--d False".split(" ")) diff --git a/tests/test_py311_generated/test_collections_generated.py b/tests/test_py311_generated/test_collections_generated.py index 9aca7e8af..5810a0216 100644 --- a/tests/test_py311_generated/test_collections_generated.py +++ b/tests/test_py311_generated/test_collections_generated.py @@ -494,7 +494,7 @@ def main(x: Dict = {"int": 5, "str": "5"}): def test_dict_optional() -> None: # In this case, the `None` is ignored. - def main(x: Optional[Dict[str, int]] = {"three": 3, "five": 5}): + def main(x: Optional[Dict[str, float]] = {"three": 3, "five": 5}): return x assert tyro.cli(main, args=[]) == {"three": 3, "five": 5} diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py index 8952be3e2..7f45093be 100644 --- a/tests/test_py311_generated/test_dcargs_generated.py +++ b/tests/test_py311_generated/test_dcargs_generated.py @@ -944,3 +944,24 @@ def main(dt: datetime.time) -> datetime.time: # Invalid hour value. with pytest.raises(SystemExit): tyro.cli(main, args=["--dt", "25:00:00"]) + + +def test_numeric_tower() -> None: + @dataclasses.dataclass(frozen=True) + class NumericTower: + a: complex | str = 3.0 + b: bytearray | str = dataclasses.field( + default_factory=lambda: bytearray(b"123") + ) + c: complex | str = True + d: int | complex = False + e: float | str = 3 + + assert tyro.cli(NumericTower, args=[]) == NumericTower(3.0) + assert tyro.cli(NumericTower, args="--a 1+3j".split(" ")) == NumericTower(1 + 3j) + assert tyro.cli(NumericTower, args="--c False".split(" ")) == NumericTower( + c="False" + ) + assert tyro.cli(NumericTower, args="--e 3.2".split(" ")) == NumericTower(e=3.2) + with pytest.raises(SystemExit): + tyro.cli(NumericTower, args="--d False".split(" ")) From 663114c38a6bece50bf374fea58b36400e6d1610 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 5 Nov 2024 14:09:10 -0800 Subject: [PATCH 472/491] Docs, custom constructor refinements (#194) * Docs pass, primitive rule priority * Fix default primitive rules * tests * sync docs * union fix --- docs/source/conf.py | 22 ++-- .../01_basics/03_dataclasses_defaults.rst | 1 + .../04_additional/06_generics_py312.rst | 3 +- .../{17_aliases.rst => 11_aliases.rst} | 28 ++--- .../04_additional/11_custom_constructors.rst | 84 -------------- .../12_custom_constructors_registry.rst | 92 ---------------- ...pe_statement.rst => 12_type_statement.rst} | 4 +- .../01_primitive_annotation.rst | 71 ++++++++++++ .../02_primitive_registry.rst | 81 ++++++++++++++ docs/source/goals_and_alternatives.md | 52 +++++---- docs/source/index.md | 86 ++++++++++----- docs/source/supported_types.md | 43 -------- docs/source/supported_types.rst | 46 ++++++++ examples/01_basics/03_dataclasses_defaults.py | 1 + examples/04_additional/06_generics_py312.py | 3 +- .../{17_aliases.py => 11_aliases.py} | 14 +-- ...type_statement.py => 12_type_statement.py} | 2 +- .../01_primitive_annotation.py} | 19 ++-- .../02_primitive_registry.py} | 20 ++-- src/tyro/_argparse_formatter.py | 4 +- src/tyro/_arguments.py | 27 ++--- src/tyro/_calling.py | 2 +- src/tyro/_cli.py | 78 ++++--------- src/tyro/_fields.py | 104 ++++++++++-------- src/tyro/_parsers.py | 38 +++++-- src/tyro/_subcommand_matching.py | 2 +- src/tyro/conf/__init__.py | 2 +- src/tyro/conf/_confstruct.py | 77 ++++++------- src/tyro/conf/_markers.py | 91 ++++++++------- src/tyro/constructors/__init__.py | 9 ++ src/tyro/constructors/_primitive_spec.py | 40 ++++--- src/tyro/constructors/_registry.py | 40 +++++-- src/tyro/constructors/_struct_spec.py | 27 ++++- src/tyro/extras/__init__.py | 5 +- src/tyro/extras/_base_configs.py | 80 +++++++------- src/tyro/extras/_choices_type.py | 5 +- src/tyro/extras/_serialization.py | 22 ++-- src/tyro/extras/_subcommand_app.py | 48 ++++---- src/tyro/extras/_subcommand_cli_from_dict.py | 62 +++++------ ...imitive.py => test_custom_constructors.py} | 21 +++- tests/test_errors.py | 2 +- tests/test_py311_generated/ok.py | 12 -- ... => test_custom_constructors_generated.py} | 19 ++++ .../test_errors_generated.py | 2 +- 44 files changed, 812 insertions(+), 679 deletions(-) rename docs/source/examples/04_additional/{17_aliases.rst => 11_aliases.rst} (61%) delete mode 100644 docs/source/examples/04_additional/11_custom_constructors.rst delete mode 100644 docs/source/examples/04_additional/12_custom_constructors_registry.rst rename docs/source/examples/04_additional/{16_type_statement.rst => 12_type_statement.rst} (88%) create mode 100644 docs/source/examples/05_custom_constructors/01_primitive_annotation.rst create mode 100644 docs/source/examples/05_custom_constructors/02_primitive_registry.rst delete mode 100644 docs/source/supported_types.md create mode 100644 docs/source/supported_types.rst rename examples/04_additional/{17_aliases.py => 11_aliases.py} (68%) rename examples/04_additional/{16_type_statement.py => 12_type_statement.py} (94%) rename examples/{04_additional/11_custom_constructors.py => 05_custom_constructors/01_primitive_annotation.py} (51%) rename examples/{04_additional/12_custom_constructors_registry.py => 05_custom_constructors/02_primitive_registry.py} (60%) rename tests/{test_custom_primitive.py => test_custom_constructors.py} (66%) delete mode 100644 tests/test_py311_generated/ok.py rename tests/test_py311_generated/{test_custom_primitive_generated.py => test_custom_constructors_generated.py} (68%) diff --git a/docs/source/conf.py b/docs/source/conf.py index 1ac56224a..a3f27c4b2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -8,8 +8,6 @@ from typing import Dict, List -import m2r2 - # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, @@ -55,7 +53,13 @@ "sphinxcontrib.programoutput", "sphinxcontrib.ansi", "sphinxcontrib.googleanalytics", + "sphinx.ext.intersphinx", ] + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", (None, "python-inv.txt")) +} + programoutput_use_ansi = True html_ansi_stylesheet = "black-on-white.css" html_static_path = ["_static"] @@ -390,13 +394,15 @@ # -- Enable Markdown -> RST conversion ---------------------------------------- -def docstring(app, what, name, obj, options, lines): - md = "\n".join(lines) - rst = m2r2.convert(md) - lines.clear() - lines += rst.splitlines() # type: ignore +# def docstring(app, what, name, obj, options, lines): +# md = "\n".join(lines) +# rst = m2r2.convert(md) +# lines.clear() +# lines += rst.splitlines() # type: ignore +# lines.append("") +# lines.append("") def setup(app): - app.connect("autodoc-process-docstring", docstring) + # app.connect("autodoc-process-docstring", docstring) app.add_css_file("css/compact_table_header.css") diff --git a/docs/source/examples/01_basics/03_dataclasses_defaults.rst b/docs/source/examples/01_basics/03_dataclasses_defaults.rst index ba87d9fac..6e857f189 100644 --- a/docs/source/examples/01_basics/03_dataclasses_defaults.rst +++ b/docs/source/examples/01_basics/03_dataclasses_defaults.rst @@ -9,6 +9,7 @@ types. .. warning:: + We advise against mutation of configuration objects from a dataclass's :code:`__post_init__` method [#f1]_. In the example below, :code:`__post_init__` would be called twice: once for the :code:`Args()` diff --git a/docs/source/examples/04_additional/06_generics_py312.rst b/docs/source/examples/04_additional/06_generics_py312.rst index 44dbb9d8e..e075e7586 100644 --- a/docs/source/examples/04_additional/06_generics_py312.rst +++ b/docs/source/examples/04_additional/06_generics_py312.rst @@ -1,13 +1,14 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -Generic Types (Python 3.12+ syntax) +Generic Types (Python 3.12+) ========================================== Example of parsing for generic dataclasses using syntax introduced in Python 3.12 (`PEP 695 `_). .. warning:: + If used in conjunction with :code:`from __future__ import annotations`, the updated type parameter syntax requires Python 3.12.4 or newer. For technical details, see `this CPython PR `_. diff --git a/docs/source/examples/04_additional/17_aliases.rst b/docs/source/examples/04_additional/11_aliases.rst similarity index 61% rename from docs/source/examples/04_additional/17_aliases.rst rename to docs/source/examples/04_additional/11_aliases.rst index e53f28421..ad728274e 100644 --- a/docs/source/examples/04_additional/17_aliases.rst +++ b/docs/source/examples/04_additional/11_aliases.rst @@ -43,54 +43,54 @@ Argument Aliases .. raw:: html - python 04_additional/17_aliases.py --help + python 04_additional/11_aliases.py --help -.. program-output:: python ../../examples/04_additional/17_aliases.py --help +.. program-output:: python ../../examples/04_additional/11_aliases.py --help ------------ .. raw:: html - python 04_additional/17_aliases.py commit --help + python 04_additional/11_aliases.py commit --help -.. program-output:: python ../../examples/04_additional/17_aliases.py commit --help +.. program-output:: python ../../examples/04_additional/11_aliases.py commit --help ------------ .. raw:: html - python 04_additional/17_aliases.py commit --message hello --all + python 04_additional/11_aliases.py commit --message hello --all -.. program-output:: python ../../examples/04_additional/17_aliases.py commit --message hello --all +.. program-output:: python ../../examples/04_additional/11_aliases.py commit --message hello --all ------------ .. raw:: html - python 04_additional/17_aliases.py commit -m hello -a + python 04_additional/11_aliases.py commit -m hello -a -.. program-output:: python ../../examples/04_additional/17_aliases.py commit -m hello -a +.. program-output:: python ../../examples/04_additional/11_aliases.py commit -m hello -a ------------ .. raw:: html - python 04_additional/17_aliases.py checkout --help + python 04_additional/11_aliases.py checkout --help -.. program-output:: python ../../examples/04_additional/17_aliases.py checkout --help +.. program-output:: python ../../examples/04_additional/11_aliases.py checkout --help ------------ .. raw:: html - python 04_additional/17_aliases.py checkout --branch main + python 04_additional/11_aliases.py checkout --branch main -.. program-output:: python ../../examples/04_additional/17_aliases.py checkout --branch main +.. program-output:: python ../../examples/04_additional/11_aliases.py checkout --branch main ------------ .. raw:: html - python 04_additional/17_aliases.py checkout -b main + python 04_additional/11_aliases.py checkout -b main -.. program-output:: python ../../examples/04_additional/17_aliases.py checkout -b main +.. program-output:: python ../../examples/04_additional/11_aliases.py checkout -b main diff --git a/docs/source/examples/04_additional/11_custom_constructors.rst b/docs/source/examples/04_additional/11_custom_constructors.rst deleted file mode 100644 index 24cea3871..000000000 --- a/docs/source/examples/04_additional/11_custom_constructors.rst +++ /dev/null @@ -1,84 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Custom Constructors -========================================== - -For additional flexibility, :module:`tyro.constructors` exposes -tyro's API for defining behavior for different types. This is the same -API that tyro relies on for the built-in types. - - -.. code-block:: python - :linenos: - - - import json - - from typing_extensions import Annotated - - import tyro - - # A dictionary type, but `tyro` will expect a JSON string from the CLI. - JsonDict = Annotated[ - dict, - tyro.constructors.PrimitiveConstructorSpec( - nargs=1, - metavar="JSON", - instance_from_str=lambda args: json.loads(args[0]), - is_instance=lambda instance: isinstance(instance, dict), - str_from_instance=lambda instance: [json.dumps(instance)], - ), - ] - - - def main( - dict1: JsonDict, - dict2: JsonDict = {"default": None}, - ) -> None: - print(f"{dict1=}") - print(f"{dict2=}") - - - if __name__ == "__main__": - tyro.cli(main) - ------------- - -.. raw:: html - - python 04_additional/11_custom_constructors.py --help - -.. program-output:: python ../../examples/04_additional/11_custom_constructors.py --help - ------------- - -.. raw:: html - - python 04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' - -.. program-output:: python ../../examples/04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' - ------------- - -.. raw:: html - - python 04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' - -.. program-output:: python ../../examples/04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' - ------------- - -.. raw:: html - - python 04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' - -.. program-output:: python ../../examples/04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' - ------------- - -.. raw:: html - - python 04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' - -.. program-output:: python ../../examples/04_additional/11_custom_constructors.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' diff --git a/docs/source/examples/04_additional/12_custom_constructors_registry.rst b/docs/source/examples/04_additional/12_custom_constructors_registry.rst deleted file mode 100644 index 942ec2b98..000000000 --- a/docs/source/examples/04_additional/12_custom_constructors_registry.rst +++ /dev/null @@ -1,92 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Custom Constructors (Registry) -========================================== - -For additional flexibility, :module:`tyro.constructors` exposes -tyro's API for defining behavior for different types. This is the same -API that tyro relies on for the built-in types. - - -.. code-block:: python - :linenos: - - - import json - from typing import Any - - import tyro - - custom_registry = tyro.constructors.PrimitiveConstructorRegistry() - - - @custom_registry.define_rule - def _( - type_info: tyro.constructors.PrimitiveTypeInfo, - ) -> tyro.constructors.PrimitiveConstructorSpec | None: - # We return `None` if the rule does not apply. - if type_info.type != dict[str, Any]: - return None - - # If the rule applies, we return the constructor spec. - return tyro.constructors.PrimitiveConstructorSpec( - nargs=1, - metavar="JSON", - instance_from_str=lambda args: json.loads(args[0]), - is_instance=lambda instance: isinstance(instance, dict), - str_from_instance=lambda instance: [json.dumps(instance)], - ) - - - def main( - dict1: dict[str, Any], - dict2: dict[str, Any] = {"default": None}, - ) -> None: - print(f"{dict1=}") - print(f"{dict2=}") - - - if __name__ == "__main__": - with custom_registry: - tyro.cli(main) - ------------- - -.. raw:: html - - python 04_additional/12_custom_constructors_registry.py --help - -.. program-output:: python ../../examples/04_additional/12_custom_constructors_registry.py --help - ------------- - -.. raw:: html - - python 04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' - -.. program-output:: python ../../examples/04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' - ------------- - -.. raw:: html - - python 04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' - -.. program-output:: python ../../examples/04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' - ------------- - -.. raw:: html - - python 04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' - -.. program-output:: python ../../examples/04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' - ------------- - -.. raw:: html - - python 04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' - -.. program-output:: python ../../examples/04_additional/12_custom_constructors_registry.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}' diff --git a/docs/source/examples/04_additional/16_type_statement.rst b/docs/source/examples/04_additional/12_type_statement.rst similarity index 88% rename from docs/source/examples/04_additional/16_type_statement.rst rename to docs/source/examples/04_additional/12_type_statement.rst index 8fe3d666e..63975f1e3 100644 --- a/docs/source/examples/04_additional/16_type_statement.rst +++ b/docs/source/examples/04_additional/12_type_statement.rst @@ -45,6 +45,6 @@ In Python 3.12, the :code:`type` statement is introduced to create type aliases. .. raw:: html - python 04_additional/16_type_statement.py --help + python 04_additional/12_type_statement.py --help -.. program-output:: python ../../examples/04_additional/16_type_statement.py --help +.. program-output:: python ../../examples/04_additional/12_type_statement.py --help diff --git a/docs/source/examples/05_custom_constructors/01_primitive_annotation.rst b/docs/source/examples/05_custom_constructors/01_primitive_annotation.rst new file mode 100644 index 000000000..e1e196c79 --- /dev/null +++ b/docs/source/examples/05_custom_constructors/01_primitive_annotation.rst @@ -0,0 +1,71 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Custom Primitive +========================================== + +For additional flexibility, :mod:`tyro.constructors` exposes tyro's API for +defining behavior for different types. There are two categories of types: +primitive types can be instantiated from a single commandline argument, while +struct types are broken down into multiple. + +In this example, we attach a custom constructor via a runtime annotation. + + +.. code-block:: python + :linenos: + + + import json + + from typing_extensions import Annotated + + import tyro + + # A dictionary type, but `tyro` will expect a JSON string from the CLI. + JsonDict = Annotated[ + dict, + tyro.constructors.PrimitiveConstructorSpec( + nargs=1, + metavar="JSON", + instance_from_str=lambda args: json.loads(args[0]), + is_instance=lambda instance: isinstance(instance, dict), + str_from_instance=lambda instance: [json.dumps(instance)], + ), + ] + + + def main( + dict1: JsonDict, + dict2: JsonDict = {"default": None}, + ) -> None: + print(f"{dict1=}") + print(f"{dict2=}") + + + if __name__ == "__main__": + tyro.cli(main) + +------------ + +.. raw:: html + + python 05_custom_constructors/01_primitive_annotation.py --help + +.. program-output:: python ../../examples/05_custom_constructors/01_primitive_annotation.py --help + +------------ + +.. raw:: html + + python 05_custom_constructors/01_primitive_annotation.py --dict1 '{"hello": "world"}' + +.. program-output:: python ../../examples/05_custom_constructors/01_primitive_annotation.py --dict1 '{"hello": "world"}' + +------------ + +.. raw:: html + + python 05_custom_constructors/01_primitive_annotation.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}' + +.. program-output:: python ../../examples/05_custom_constructors/01_primitive_annotation.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}' diff --git a/docs/source/examples/05_custom_constructors/02_primitive_registry.rst b/docs/source/examples/05_custom_constructors/02_primitive_registry.rst new file mode 100644 index 000000000..a6bd8bca8 --- /dev/null +++ b/docs/source/examples/05_custom_constructors/02_primitive_registry.rst @@ -0,0 +1,81 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +Custom Primitive (Registry) +========================================== + +For additional flexibility, :mod:`tyro.constructors` exposes tyro's API for +defining behavior for different types. There are two categories of types: +primitive types can be instantiated from a single commandline argument, while +struct types are broken down into multiple. + + +In this example, we attach a custom constructor by defining a rule that applies +to all types that match ``dict[str, Any]``. + + +.. code-block:: python + :linenos: + + + import json + from typing import Any + + import tyro + + custom_registry = tyro.constructors.ConstructorRegistry() + + + @custom_registry.primitive_rule + def _( + type_info: tyro.constructors.PrimitiveTypeInfo, + ) -> tyro.constructors.PrimitiveConstructorSpec | None: + # We return `None` if the rule does not apply. + if type_info.type != dict[str, Any]: + return None + + # If the rule applies, we return the constructor spec. + return tyro.constructors.PrimitiveConstructorSpec( + nargs=1, + metavar="JSON", + instance_from_str=lambda args: json.loads(args[0]), + is_instance=lambda instance: isinstance(instance, dict), + str_from_instance=lambda instance: [json.dumps(instance)], + ) + + + def main( + dict1: dict[str, Any], + dict2: dict[str, Any] = {"default": None}, + ) -> None: + print(f"{dict1=}") + print(f"{dict2=}") + + + if __name__ == "__main__": + with custom_registry: + tyro.cli(main) + +------------ + +.. raw:: html + + python 05_custom_constructors/02_primitive_registry.py --help + +.. program-output:: python ../../examples/05_custom_constructors/02_primitive_registry.py --help + +------------ + +.. raw:: html + + python 05_custom_constructors/02_primitive_registry.py --dict1 '{"hello": "world"}' + +.. program-output:: python ../../examples/05_custom_constructors/02_primitive_registry.py --dict1 '{"hello": "world"}' + +------------ + +.. raw:: html + + python 05_custom_constructors/02_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}' + +.. program-output:: python ../../examples/05_custom_constructors/02_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}' diff --git a/docs/source/goals_and_alternatives.md b/docs/source/goals_and_alternatives.md index e5f36e0ff..b8dc54843 100644 --- a/docs/source/goals_and_alternatives.md +++ b/docs/source/goals_and_alternatives.md @@ -10,7 +10,7 @@ Usage distinctions are the result of two API goals: `tyro` should reduce to learning to write type-annotated Python. For example, types are specified using standard annotations, helptext using docstrings, choices using the standard `typing.Literal` type, subcommands with - `typing.Union` of nested types, and positional arguments with `/`. + `typing.Union` of struct types, and positional arguments with `/`. - In contrast, similar libraries have more expansive APIs , and require more library-specific structures, decorators, or metadata formats for configuring parsing behavior. @@ -21,23 +21,32 @@ Usage distinctions are the result of two API goals: dynamic argparse-style namespaces, or string-based accessors that can't be statically checked. + + +.. warning:: + + This survey was conducted in late 2022. It may be out of date. + + + More concretely, we can also compare specific features. A noncomprehensive set: -| | Dataclasses | Functions | Literals | Docstrings as helptext | Nested structures | Unions over primitives | Unions over nested types | Lists, tuples | Dictionaries | Generics | -| -------------------------------------------- | ----------- | --------- | -------------------- | ---------------------- | ----------------- | ---------------------- | ------------------------- | -------------------- | ------------ | -------- | -| [argparse-dataclass][argparse-dataclass] | ✓ | | | | | | | | | | -| [argparse-dataclasses][argparse-dataclasses] | ✓ | | | | | | | | | | -| [datargs][datargs] | ✓ | | ✓[^datargs_literals] | | | | ✓[^datargs_unions_nested] | ✓ | | | -| [tap][tap] | | | ✓ | ✓ | | ✓ | ~[^tap_unions_nested] | ✓ | | | -| [simple-parsing][simple-parsing] | ✓ | | ✓[^simp_literals] | ✓ | ✓ | ✓ | ✓[^simp_unions_nested] | ✓ | ✓ | | -| [dataclass-cli][dataclass-cli] | ✓ | | | | | | | | | | -| [clout][clout] | ✓ | | | | ✓ | | | | | | -| [hf_argparser][hf_argparser] | ✓ | | | | | | | ✓ | ✓ | | -| [typer][typer] | | ✓ | | | | | ~[^typer_unions_nested] | ~[^typer_containers] | | | -| [pyrallis][pyrallis] | ✓ | | | ✓ | ✓ | | | ✓ | | | -| [yahp][yahp] | ✓ | | | ~[^yahp_docstrings] | ✓ | ✓ | ~[^yahp_unions_nested] | ✓ | | | -| [omegaconf][omegaconf] | ✓ | | | | ✓ | | | ✓ | ✓ | | -| **tyro** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| | Dataclasses | Functions | Literals | Docstrings as helptext | Nested structs | Unions over primitives | Unions over structs | Lists, tuples | Dicts | Generics | +| -------------------------------------------- | ----------- | --------- | -------------------- | ---------------------- | -------------- | ---------------------- | ------------------------- | -------------------- | ----- | -------- | +| [argparse-dataclass][argparse-dataclass] | ✓ | | | | | | | | | | +| [argparse-dataclasses][argparse-dataclasses] | ✓ | | | | | | | | | | +| [datargs][datargs] | ✓ | | ✓[^datargs_literals] | | | | ✓[^datargs_unions_struct] | ✓ | | | +| [tap][tap] | | | ✓ | ✓ | | ✓ | ~[^tap_unions_struct] | ✓ | | | +| [simple-parsing][simple-parsing] | ✓ | | ✓[^simp_literals] | ✓ | ✓ | ✓ | ✓[^simp_unions_struct] | ✓ | ✓ | | +| [dataclass-cli][dataclass-cli] | ✓ | | | | | | | | | | +| [clout][clout] | ✓ | | | | ✓ | | | | | | +| [hf_argparser][hf_argparser] | ✓ | | | | | | | ✓ | ✓ | | +| [typer][typer] | | ✓ | | | | | ~[^typer_unions_struct] | ~[^typer_containers] | | | +| [pyrallis][pyrallis] | ✓ | | | ✓ | ✓ | | | ✓ | | | +| [yahp][yahp] | ✓ | | | ~[^yahp_docstrings] | ✓ | ✓ | ~[^yahp_unions_struct] | ✓ | | | +| [omegaconf][omegaconf] | ✓ | | | | ✓ | | | ✓ | ✓ | | +| [defopt][defopt] | | ✓ | ✓ | ✓ | ✓ | ✓ | | ✓ | | | +| **tyro** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | @@ -53,12 +62,13 @@ More concretely, we can also compare specific features. A noncomprehensive set: [typer]: https://typer.tiangolo.com/ [yahp]: https://github.com/mosaicml/yahp [omegaconf]: https://omegaconf.readthedocs.io/en/2.1_branch/structured_config.html +[defopt]: https://github.com/anntzer/defopt/ -[^datargs_unions_nested]: One allowed per class. -[^tap_unions_nested]: Not supported, but API exists for creating subcommands that accomplish a similar goal. -[^simp_unions_nested]: One allowed per class. -[^yahp_unions_nested]: Not supported, but similar functionality available via ["registries"](https://docs.mosaicml.com/projects/yahp/en/stable/examples/registry.html). -[^typer_unions_nested]: Not supported, but API exists for creating subcommands that accomplish a similar goal. +[^datargs_unions_struct]: One allowed per class. +[^tap_unions_struct]: Not supported, but API exists for creating subcommands that accomplish a similar goal. +[^simp_unions_struct]: One allowed per class. +[^yahp_unions_struct]: Not supported, but similar functionality available via ["registries"](https://docs.mosaicml.com/projects/yahp/en/stable/examples/registry.html). +[^typer_unions_struct]: Not supported, but API exists for creating subcommands that accomplish a similar goal. [^simp_literals]: Not supported for mixed (eg `Literal[5, "five"]`) or in container (eg `List[Literal[1, 2]]`) types. [^datargs_literals]: Not supported for mixed types (eg `Literal[5, "five"]`). [^typer_containers]: `typer` uses positional arguments for all required fields, which means that only one variable-length argument (such as `List[int]`) without a default is supported per argument parser. diff --git a/docs/source/index.md b/docs/source/index.md index 1144b6090..dbcbae069 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -2,42 +2,67 @@ |build| |nbsp| |ruff| |nbsp| |mypy| |nbsp| |pyright| |nbsp| |coverage| |nbsp| |versions| -:code:`tyro` is a tool for generating command-line interfaces and configuration -objects in Python. +:func:`tyro.cli()` is a tool for generating CLI +interfaces. -Our core API, :func:`tyro.cli()`, +We can define configurable scripts using functions: -- **Generates CLI interfaces** from a comprehensive set of Python type - constructs. -- **Populates helptext automatically** from defaults, annotations, and - docstrings. -- **Understands nesting** of `dataclasses`, `pydantic`, and `attrs` structures. -- **Prioritizes static analysis** for type checking and autocompletion with - tools like - [Pylance](https://marketplace.visualstudio.com/items?itemName=ms-python.vscode-pylance), - [Pyright](https://github.com/microsoft/pyright), and - [mypy](https://github.com/python/mypy). +```python +"""A command-line interface defined using a function signature. -For advanced users, it also supports: +Usage: python script_name.py --foo INT [--bar STR] +""" -- **Subcommands**, as well as choosing between and overriding values in - configuration objects. -- **Completion script generation** for `bash`, `zsh`, and `tcsh`. -- **Fine-grained configuration** via [PEP - 593](https://peps.python.org/pep-0593/) annotations (`tyro.conf.*`). +import tyro -To get started, we recommend browsing the examples to the left. +def main( + foo: int, + bar: str = "default", +) -> None: + ... # Main body of a script. -### Why `tyro`? +if __name__ == "__main__": + # Generate a CLI and call `main` with its two arguments: `foo` and `bar`. + tyro.cli(main) +``` -1. **Strong typing.** +Or instantiate config objects defined using tools like `dataclasses`, `pydantic`, and `attrs`: + +```python +"""A command-line interface defined using a class signature. + +Usage: python script_name.py --foo INT [--bar STR] +""" + +from dataclasses import dataclass +import tyro + +@dataclass +class Config: + foo: int + bar: str = "default" + +if __name__ == "__main__": + # Generate a CLI and instantiate `Config` with its two arguments: `foo` and `bar`. + config = tyro.cli(Config) + + # Rest of script. + assert isinstance(config, Config) # Should pass. +``` + +Other features include helptext generation, nested structures, subcommands, and +shell completion. + +#### Why `tyro`? + +1. **Types.** Unlike tools dependent on dictionaries, YAML, or dynamic namespaces, arguments populated by `tyro` benefit from IDE and language server-supported operations — think tab completion, rename, jump-to-def, docstrings on hover — as well as static checking tools like `pyright` and `mypy`. -2. **Minimal overhead.** +2. **Define things once.** Standard Python type annotations, docstrings, and default values are parsed to automatically generate command-line interfaces with informative helptext. @@ -55,11 +80,6 @@ To get started, we recommend browsing the examples to the left. distribute definitions, defaults, and documentation of configurable fields across modules or source files. -4. **Tab completion.** - - By extending [shtab](https://github.com/iterative/shtab), `tyro` - automatically generates tab completion scripts for bash, zsh, and tcsh. - .. toctree:: @@ -112,6 +132,16 @@ To get started, we recommend browsing the examples to the left. examples/04_additional/* +.. toctree:: + :caption: Custom Constructors + :hidden: + :maxdepth: 1 + :titlesonly: + :glob: + + examples/05_custom_constructors/* + + .. toctree:: :caption: Notes :hidden: diff --git a/docs/source/supported_types.md b/docs/source/supported_types.md deleted file mode 100644 index 6e3a95100..000000000 --- a/docs/source/supported_types.md +++ /dev/null @@ -1,43 +0,0 @@ -# Supported types - -For minimum-boilerplate CLIs, `tyro` aims to maximize support of -Python's standard [typing](https://docs.python.org/3/library/typing.html) -features. - -As a partial list, inputs can be annotated with: - -- Basic types like `int`, `str`, `float`, `bool`, `pathlib.Path`, `None`. -- `datetime.date`, `datetime.datetime`, and `datetime.time`. -- Container types like `list`, `dict`, `tuple`, and `set`. -- Union types, like `X | Y`, `Union[X, Y]`, and `Optional[T]`. -- `typing.Literal` and `enum.Enum` . -- Type aliases, for example using Python 3.12's [PEP 695](https://peps.python.org/pep-0695/) `type` statement. -- Generics, such as those annotated with `typing.TypeVar` or with the type parameter syntax introduced by Python 3.12's [PEP 695](https://peps.python.org/pep-0695/). -- etc - -Compositions of the above types, like `tuple[int | str, ...] | None`, are also supported. - -Types can also be placed and nested in various structures, such as: - -- `dataclasses.dataclass`. -- `attrs`, `pydantic`, and `flax.linen` models. -- `typing.NamedTuple`. -- `typing.TypedDict`, flags like `total=`, and associated annotations like `typing.Required`, `typing.NotRequired`, `typing.ReadOnly`, - -### What's not supported - -There are some limitations. We currently _do not_ support: - -- Variable-length sequences over nested structures, unless a default is - provided. For types like `list[Dataclass]`, we require a default value to - infer length from. The length of the corresponding field cannot be changed - from the CLI interface. -- Nesting variable-length sequences in other sequences. `tuple[int, ...]` and - `tuple[tuple[int, int, int], ...]` are supported, as the variable-length - sequence is the outermost type. However, `tuple[tuple[int, ...], ...]` is - ambiguous to parse and not supported. -- Self-referential types, like `type RecursiveList[T] = T | list[RecursiveList[T]]`. - -In each of these cases, a [custom -constructor](https://brentyi.github.io/tyro/examples/04_additional/11_custom_constructors/) -can be defined as a workaround. diff --git a/docs/source/supported_types.rst b/docs/source/supported_types.rst new file mode 100644 index 000000000..3fdc1707a --- /dev/null +++ b/docs/source/supported_types.rst @@ -0,0 +1,46 @@ +What's supported +================ + +For minimum-boilerplate CLIs, `tyro` aims to maximize support of +Python's standard :mod:`typing` features. + +As a partial list, inputs can be annotated with: + +- Basic types like :class:`int`, :class:`str`, :class:`float`, :class:`bool`, :class:`pathlib.Path`, :data:`None`. +- :class:`datetime.date`, :class:`datetime.datetime`, and :class:`datetime.time`. +- Container types like :class:`list`, :class:`dict`, :class:`tuple`, and :class:`set`. +- Union types, like `X | Y`, :py:data:`typing.Union`, and :py:data:`typing.Optional`. +- :py:data:`typing.Literal` and :class:`enum.Enum`. +- Type aliases, for example using Python 3.12's `PEP 695 `_ `type` statement. +- Generics, such as those annotated with :py:class:`typing.TypeVar` or with the type parameter syntax introduced by Python 3.12's `PEP 695 `_. +- etc + +Compositions of the above types, like ``tuple[int | str, ...] | None``, are also supported. + +Types can also be placed and nested in various structures, such as: + +- :func:`dataclasses.dataclass`. +- ``attrs``, ``pydantic``, and ``flax.linen`` models. +- :py:class:`typing.NamedTuple`. +- :py:class:`typing.TypedDict`, flags like ``total=``, and associated annotations like :py:data:`typing.Required`, :py:data:`typing.NotRequired`, :py:data:`typing.ReadOnly`, + + +What's not supported +-------------------- + + +There are some limitations. We currently _do not_ support: + +- Variable-length sequences over nested structures, unless a default is + provided. For types like ``list[Dataclass]``, we require a default value to + infer length from. The length of the corresponding field cannot be changed + from the CLI interface. +- Nesting variable-length sequences in other sequences. ``tuple[int, ...]`` and + ``tuple[tuple[int, int, int], ...]`` are supported, as the variable-length + sequence is the outermost type. However, ``tuple[tuple[int, ...], ...]`` is + ambiguous to parse and not supported. +- Self-referential types, like ``type RecursiveList[T] = T | list[RecursiveList[T]]``. + +In each of these cases, a `custom +constructor `_ +can be defined as a workaround. diff --git a/examples/01_basics/03_dataclasses_defaults.py b/examples/01_basics/03_dataclasses_defaults.py index 2a5670383..9b207e8c4 100644 --- a/examples/01_basics/03_dataclasses_defaults.py +++ b/examples/01_basics/03_dataclasses_defaults.py @@ -5,6 +5,7 @@ .. warning:: + We advise against mutation of configuration objects from a dataclass's :code:`__post_init__` method [#f1]_. In the example below, :code:`__post_init__` would be called twice: once for the :code:`Args()` diff --git a/examples/04_additional/06_generics_py312.py b/examples/04_additional/06_generics_py312.py index 2fd4b4580..416d75e25 100644 --- a/examples/04_additional/06_generics_py312.py +++ b/examples/04_additional/06_generics_py312.py @@ -1,12 +1,13 @@ # mypy: ignore-errors # # PEP 695 isn't yet supported in mypy. (April 4, 2024) -"""Generic Types (Python 3.12+ syntax) +"""Generic Types (Python 3.12+) Example of parsing for generic dataclasses using syntax introduced in Python 3.12 (`PEP 695 `_). .. warning:: + If used in conjunction with :code:`from __future__ import annotations`, the updated type parameter syntax requires Python 3.12.4 or newer. For technical details, see `this CPython PR `_. Usage: diff --git a/examples/04_additional/17_aliases.py b/examples/04_additional/11_aliases.py similarity index 68% rename from examples/04_additional/17_aliases.py rename to examples/04_additional/11_aliases.py index 16c5b3b3c..5ee33de9c 100644 --- a/examples/04_additional/17_aliases.py +++ b/examples/04_additional/11_aliases.py @@ -3,13 +3,13 @@ :func:`tyro.conf.arg()` can be used to attach aliases to arguments. Usage: -`python ./12_aliases.py --help` -`python ./12_aliases.py commit --help` -`python ./12_aliases.py commit --message hello --all` -`python ./12_aliases.py commit -m hello -a` -`python ./12_aliases.py checkout --help` -`python ./12_aliases.py checkout --branch main` -`python ./12_aliases.py checkout -b main` +`python ./11_aliases.py --help` +`python ./11_aliases.py commit --help` +`python ./11_aliases.py commit --message hello --all` +`python ./11_aliases.py commit -m hello -a` +`python ./11_aliases.py checkout --help` +`python ./11_aliases.py checkout --branch main` +`python ./11_aliases.py checkout -b main` """ from typing_extensions import Annotated diff --git a/examples/04_additional/16_type_statement.py b/examples/04_additional/12_type_statement.py similarity index 94% rename from examples/04_additional/16_type_statement.py rename to examples/04_additional/12_type_statement.py index 9aa08cf93..fa8140297 100644 --- a/examples/04_additional/16_type_statement.py +++ b/examples/04_additional/12_type_statement.py @@ -6,7 +6,7 @@ In Python 3.12, the :code:`type` statement is introduced to create type aliases. Usage: -`python ./16_type_statement.py --help` +`python ./12_type_statement.py --help` """ import dataclasses diff --git a/examples/04_additional/11_custom_constructors.py b/examples/05_custom_constructors/01_primitive_annotation.py similarity index 51% rename from examples/04_additional/11_custom_constructors.py rename to examples/05_custom_constructors/01_primitive_annotation.py index 4d8cff568..505e8b9dd 100644 --- a/examples/04_additional/11_custom_constructors.py +++ b/examples/05_custom_constructors/01_primitive_annotation.py @@ -1,15 +1,16 @@ -"""Custom Constructors +"""Custom Primitive -For additional flexibility, :module:`tyro.constructors` exposes -tyro's API for defining behavior for different types. This is the same -API that tyro relies on for the built-in types. +For additional flexibility, :mod:`tyro.constructors` exposes tyro's API for +defining behavior for different types. There are two categories of types: +primitive types can be instantiated from a single commandline argument, while +struct types are broken down into multiple. + +In this example, we attach a custom constructor via a runtime annotation. Usage: -`python ./10_custom_constructors.py --help` -`python ./10_custom_constructors.py --dict1.json '{"hello": "world"}'` -`python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}"` -`python ./10_custom_constructors.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}'` -`python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}" --dict2.json "{\"hello\": \"world\"}"` +`python ./01_primitive_annotation.py --help` +`python ./01_primitive_annotation.py --dict1 '{"hello": "world"}'` +`python ./01_primitive_annotation.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'` """ import json diff --git a/examples/04_additional/12_custom_constructors_registry.py b/examples/05_custom_constructors/02_primitive_registry.py similarity index 60% rename from examples/04_additional/12_custom_constructors_registry.py rename to examples/05_custom_constructors/02_primitive_registry.py index 88eb13828..767765f0b 100644 --- a/examples/04_additional/12_custom_constructors_registry.py +++ b/examples/05_custom_constructors/02_primitive_registry.py @@ -1,15 +1,17 @@ -"""Custom Constructors (Registry) +"""Custom Primitive (Registry) +For additional flexibility, :mod:`tyro.constructors` exposes tyro's API for +defining behavior for different types. There are two categories of types: +primitive types can be instantiated from a single commandline argument, while +struct types are broken down into multiple. -For additional flexibility, :module:`tyro.constructors` exposes -tyro's API for defining behavior for different types. This is the same -API that tyro relies on for the built-in types. + +In this example, we attach a custom constructor by defining a rule that applies +to all types that match ``dict[str, Any]``. Usage: -`python ./10_custom_constructors.py --help` -`python ./10_custom_constructors.py --dict1.json '{"hello": "world"}'` -`python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}"` -`python ./10_custom_constructors.py --dict1.json '{"hello": "world"}' --dict2.json '{"hello": "world"}'` -`python ./10_custom_constructors.py --dict1.json "{\"hello\": \"world\"}" --dict2.json "{\"hello\": \"world\"}"` +`python ./02_primitive_registry.py --help` +`python ./02_primitive_registry.py --dict1 '{"hello": "world"}'` +`python ./02_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'` """ import json diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py index 1f632eb88..fda0df251 100644 --- a/src/tyro/_argparse_formatter.py +++ b/src/tyro/_argparse_formatter.py @@ -55,8 +55,8 @@ def as_rich_theme(self) -> Theme: def set_accent_color(accent_color: Optional[str]) -> None: - """Set an accent color to use in help messages. Takes any color supported by `rich`, - see `python -m rich.color`. Experimental.""" + """Set an accent color to use in help messages. Takes any color supported by ``rich``, + see ``python -m rich.color``. Experimental.""" THEME.border = Style(color=accent_color, dim=True) THEME.description = Style(color=accent_color, bold=True) THEME.invocation = Style() diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index a9f6489d6..d0b4aef9b 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -174,9 +174,9 @@ def add_argument( ) complete_as_path = ( # Catch types like Path, List[Path], Tuple[Path, ...] etc. - "Path" in str(self.field.type_or_callable) + "Path" in str(self.field.type_stripped) # For string types, we require more evidence. - or ("str" in str(self.field.type_or_callable) and name_suggests_path) + or ("str" in str(self.field.type_stripped) and name_suggests_path) ) if complete_as_path: arg.complete = shtab.DIRECTORY if name_suggests_dir else shtab.FILE # type: ignore @@ -233,7 +233,7 @@ def _rule_handle_boolean_flags( arg: ArgumentDefinition, lowered: LoweredArgumentDefinition, ) -> None: - if arg.field.type_or_callable is not bool: + if arg.field.type_stripped is not bool: return if ( @@ -281,15 +281,12 @@ def _rule_apply_primitive_specs( return try: - if arg.field.primitive_spec is not None: - spec = arg.field.primitive_spec - else: - spec = ConstructorRegistry._get_active_registry().get_primitive_spec( - PrimitiveTypeInfo.make( - cast(type, arg.field.type_or_callable), - arg.field.markers, - ) + spec = ConstructorRegistry._get_active_registry().get_primitive_spec( + PrimitiveTypeInfo.make( + cast(type, arg.field.type), + arg.field.markers, ) + ) except UnsupportedTypeAnnotationError as e: if arg.field.default in _singleton.MISSING_SINGLETONS: field_name = _strings.make_field_name( @@ -335,11 +332,11 @@ def _rule_apply_primitive_specs( def append_instantiator(x: list[list[str]]) -> Any: """Handle UseAppendAction effects.""" # We'll assume that the type is annotated as Dict[...], Tuple[...], List[...], etc. - container_type = get_origin(arg.field.type_or_callable) + container_type = get_origin(arg.field.type_stripped) if container_type is None: # Raw annotation, like `UseAppendAction[list]`. It's unlikely # that a user would use this but we can handle it. - container_type = arg.field.type_or_callable + container_type = arg.field.type_stripped # Instantiate initial output. out = ( @@ -407,7 +404,7 @@ def _rule_counters( """Handle counters, like -vvv for level-3 verbosity.""" if ( _markers.UseCounterAction in arg.field.markers - and arg.field.type_or_callable is int + and arg.field.type_stripped is int and not arg.field.is_positional() ): lowered.metavar = None @@ -459,7 +456,7 @@ def _rule_generate_helptext( if arg.field.argconf.constructor_factory is not None: default_label = ( str(default) - if arg.field.type_or_callable is not json.loads + if arg.field.type_stripped is not json.loads else json.dumps(arg.field.default) ) elif type(default) in (tuple, list, set): diff --git a/src/tyro/_calling.py b/src/tyro/_calling.py index b714c20ec..53c091a16 100644 --- a/src/tyro/_calling.py +++ b/src/tyro/_calling.py @@ -70,7 +70,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any: ) # Resolve field type. - field_type = field.type_or_callable + field_type = field.type_stripped if prefixed_field_name in arg_from_prefixed_field_name: assert prefixed_field_name not in consumed_keywords diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index d24ec32e8..ad203dcd4 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -113,79 +113,48 @@ def cli( config: None | Sequence[conf._markers.Marker] = None, **deprecated_kwargs, ) -> OutT | tuple[OutT, list[str]]: - """Call or instantiate `f`, with inputs populated from an automatically generated - CLI interface. + """Instantiate or call ``f``, with inputs populated from an automatically + generated CLI interface. `f` should have type-annotated inputs, and can be a function or type. If - `f` is a type, `tyro.cli()` returns an instance. - - The parser is generated by populating helptext from docstrings and types from - annotations; a broad range of core type annotations are supported. - - Types natively accepted by `argparse`: str, int, float, pathlib.Path, etc. - - Default values for optional parameters. - - Booleans, which are automatically converted to flags when provided a default - value. - - Enums (via `enum.Enum`). - - Various annotations from the standard typing library. Some examples: - - `typing.ClassVar[T]`. - - `typing.Optional[T]`. - - `typing.Literal[T]`. - - `typing.Sequence[T]`. - - `typing.List[T]`. - - `typing.Dict[K, V]`. - - `typing.Tuple`, such as `typing.Tuple[T1, T2, T3]` or - `typing.Tuple[T, ...]`. - - `typing.Set[T]`. - - `typing.Final[T]` and `typing.Annotated[T]`. - - `typing.Union[T1, T2]`. - - Various nested combinations of the above: `Optional[Literal[T]]`, - `Final[Optional[Sequence[T]]]`, etc. - - Hierarchical structures via nested dataclasses, TypedDict, NamedTuple, - classes. - - Simple nesting. - - Unions over nested structures (subparsers). - - Optional unions over nested structures (optional subparsers). - - Generics (including nested generics). - - Completion script generation for interactive shells is also provided. To write a - script that can be used for tab completion, pass in: - `--tyro-write-completion {bash/zsh/tcsh} {path to script to write}`. + ``f`` is a type, ``tyro.cli()`` returns an instance. If ``f`` is a + function, ``tyro.cli()`` returns the output of calling the function. Args: f: Function or type. prog: The name of the program printed in helptext. Mirrors argument from - `argparse.ArgumentParser()`. + :py:class:`argparse.ArgumentParser()`. description: Description text for the parser, displayed when the --help flag is - passed in. If not specified, `f`'s docstring is used. Mirrors argument from - `argparse.ArgumentParser()`. + passed in. If not specified, ``f``'s docstring is used. Mirrors argument from + :py:class:`argparse.ArgumentParser()`. args: If set, parse arguments from a sequence of strings instead of the - commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. - default: An instance of `OutT` to use for default values; supported if `f` is a - type like a dataclass or dictionary, but not if `f` is a general callable + commandline. Mirrors argument from :py:meth:`argparse.ArgumentParser.parse_args()`. + default: An instance of ``OutT`` to use for default values; supported if ``f`` is a + type like a dataclass or dictionary, but not if ``f`` is a general callable like a function or standard class. Helpful for merging CLI arguments with values loaded from elsewhere. (for example, a config object loaded from a yaml file) - return_unknown_args: If True, return a tuple of the output of `f` and a list of + return_unknown_args: If True, return a tuple of the output of ``f`` and a list of unknown arguments. Mirrors the unknown arguments returned from - `argparse.ArgumentParser.parse_known_args()`. + :py:meth:`argparse.ArgumentParser.parse_known_args()`. use_underscores: If True, use underscores as a word delimeter instead of hyphens. This primarily impacts helptext; underscores and hyphens are treated equivalently when parsing happens. We default helptext to hyphens to follow the GNU style guide. https://www.gnu.org/software/libc/manual/html_node/Argument-Syntax.html - console_outputs: If set to `False`, parsing errors and help messages will be - supressed. This can be useful for distributed settings, where `tyro.cli()` + console_outputs: If set to ``False``, parsing errors and help messages will be + supressed. This can be useful for distributed settings, where ``tyro.cli()`` is called from multiple workers but we only want console outputs from the main one. - config: Sequence of config marker objects, from `tyro.conf`. As an + config: Sequence of config marker objects, from :mod:`tyro.conf`. As an alternative to using them locally in annotations - (`tyro.conf.FlagConversionOff[bool]`), we can also pass in a sequence of + (:class:`tyro.conf.FlagConversionOff`[bool]), we can also pass in a sequence of them here to apply globally. Returns: - The output of `f(...)` or an instance `f`. If `f` is a class, the two are - equivalent. If `return_unknown_args` is True, returns a tuple of the output of - `f(...)` and a list of unknown arguments. + The output of ``f(...)`` or an instance ``f``. If ``f`` is a class, the two are + equivalent. If ``return_unknown_args`` is True, returns a tuple of the output of + ``f(...)`` and a list of unknown arguments. """ # Make sure we start on a clean slate. Some tests may fail without this due to @@ -256,11 +225,11 @@ def get_parser( use_underscores: bool = False, console_outputs: bool = True, ) -> argparse.ArgumentParser: - """Get the `argparse.ArgumentParser` object generated under-the-hood by - `tyro.cli()`. Useful for tools like `sphinx-argparse`, `argcomplete`, etc. + """Get the ``argparse.ArgumentParser`` object generated under-the-hood by + :func:`tyro.cli()`. Useful for tools like ``sphinx-argparse``, ``argcomplete``, etc. - For tab completion, we recommend using `tyro.cli()`'s built-in `--tyro-write-completion` - flag.""" + For tab completion, we recommend using :func:`tyro.cli()`'s built-in + ``--tyro-write-completion`` flag.""" with _strings.delimeter_context("_" if use_underscores else "-"): return cast( argparse.ArgumentParser, @@ -408,7 +377,6 @@ def _cli_impl( default_instance=default_instance_internal, # Overrides for default values. intern_prefix="", # Used for recursive calls. extern_prefix="", # Used for recursive calls. - subcommand_prefix="", # Used for recursive calls. ) # Generate parser! diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index b1f50680d..2cc973c93 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -12,19 +12,22 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union import docstring_parser -from typing_extensions import Annotated +from typing_extensions import Annotated, get_args, get_origin from . import _docstrings, _resolver, _strings, _unsafe_cache from ._singleton import DEFAULT_SENTINEL_SINGLETONS, MISSING_SINGLETONS from ._typing import TypeForm from .conf import _confstruct, _markers from .constructors._primitive_spec import ( - PrimitiveConstructorSpec, PrimitiveTypeInfo, UnsupportedTypeAnnotationError, ) from .constructors._registry import ConstructorRegistry -from .constructors._struct_spec import StructTypeInfo, UnsupportedStructTypeMessage +from .constructors._struct_spec import ( + StructFieldSpec, + StructTypeInfo, + UnsupportedStructTypeMessage, +) global_context_markers: List[Tuple[_markers.Marker, ...]] = [] @@ -33,7 +36,9 @@ class FieldDefinition: intern_name: str extern_name: str - type_or_callable: Union[TypeForm[Any], Callable] + type: TypeForm[Any] | Callable + """Full type, including runtime annotations.""" + type_stripped: TypeForm[Any] | Callable default: Any # We need to record whether defaults are from default instances to # determine if they should override the default in @@ -44,7 +49,6 @@ class FieldDefinition: custom_constructor: bool argconf: _confstruct._ArgConfig - primitive_spec: PrimitiveConstructorSpec | None # Override the name in our kwargs. Useful whenever the user-facing argument name # doesn't match the keyword expected by our callable. @@ -67,6 +71,17 @@ def marker_context(markers: Tuple[_markers.Marker, ...]): yield global_context_markers.pop() + @staticmethod + def from_field_spec(field_spec: StructFieldSpec) -> FieldDefinition: + return FieldDefinition.make( + name=field_spec.name, + typ=field_spec.type, + default=field_spec.default, + is_default_from_default_instance=field_spec.is_default_overridden, + helptext=field_spec.helptext, + call_argname_override=field_spec._call_argname, + ) + @staticmethod def make( name: str, @@ -114,18 +129,31 @@ def make( if argconf.help is not None: helptext = argconf.help - _, primitive_specs = _resolver.unwrap_annotated(typ, PrimitiveConstructorSpec) - if len(primitive_specs) > 0: - primitive_spec = primitive_specs[0] - else: - primitive_spec = None - - typ, markers = _resolver.unwrap_annotated(typ, _markers._Marker) + type_stripped, markers = _resolver.unwrap_annotated(typ, _markers._Marker) # Include markers set via context manager. for context_markers in global_context_markers: markers += context_markers + out = FieldDefinition( + intern_name=name, + extern_name=name if argconf.name is None else argconf.name, + type=typ, + type_stripped=type_stripped, + default=default, + is_default_from_default_instance=is_default_from_default_instance, + helptext=helptext, + markers=set(markers), + custom_constructor=argconf.constructor_factory is not None, + argconf=argconf, + call_argname=( + call_argname_override if call_argname_override is not None else name + ), + ) + + if argconf.constructor_factory is not None: + out = out.with_new_type_stripped(argconf.constructor_factory()) + # Check that the default value matches the final resolved type. # There's some similar Union-specific logic for this in narrow_union_type(). We # may be able to consolidate this. @@ -133,8 +161,8 @@ def make( # Be relatively conservative: isinstance() can be checked on non-type # types (like unions in Python >=3.10), but we'll only consider single types # for now. - type(typ) is type - and not isinstance(default, typ) # type: ignore + type(out.type_stripped) is type + and not isinstance(default, out.type_stripped) # type: ignore # If a custom constructor is set, static_type may not be # matched to the annotated type. and argconf.constructor_factory is None @@ -150,29 +178,25 @@ def make( f"but the default value {default} has type {type(default)}. " f"We'll try to handle this gracefully, but it may cause unexpected behavior." ) - typ = Union[typ, type(default)] # type: ignore + out = out.with_new_type_stripped(Union[out.type_stripped, type(default)]) # type: ignore - out = FieldDefinition( - intern_name=name, - extern_name=name if argconf.name is None else argconf.name, - type_or_callable=( - typ - if argconf.constructor_factory is None - else argconf.constructor_factory() - ), - default=default, - is_default_from_default_instance=is_default_from_default_instance, - helptext=helptext, - markers=set(markers), - custom_constructor=argconf.constructor_factory is not None, - argconf=argconf, - call_argname=( - call_argname_override if call_argname_override is not None else name - ), - primitive_spec=primitive_spec, - ) return out + def with_new_type_stripped( + self, new_type_stripped: TypeForm[Any] | Callable + ) -> FieldDefinition: + if get_origin(self.type) is Annotated: + new_type = Annotated.__class_getitem__( # type: ignore + (new_type_stripped, *get_args(self.type)[1:]) + ) + else: + new_type = new_type_stripped + return dataclasses.replace( + self, + type=new_type, + type_stripped=new_type_stripped, + ) + def is_positional(self) -> bool: """Returns True if the argument should be positional in the commandline.""" return ( @@ -240,17 +264,7 @@ def field_list_from_type_or_callable( with FieldDefinition.marker_context(type_info.markers): if spec is not None: - return f, [ - FieldDefinition.make( - f.name, - f.type, - f.default, - f.is_default_overridden, - f.helptext, - call_argname_override=f._call_argname, - ) - for f in spec.fields - ] + return f, [FieldDefinition.from_field_spec(f) for f in spec.fields] try: registry.get_primitive_spec(PrimitiveTypeInfo.make(f, set())) diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 068b1ac8e..06c7cb79b 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -19,6 +19,7 @@ from typing_extensions import Annotated, get_args, get_origin +from tyro.constructors._registry import ConstructorRegistry from tyro.constructors._struct_spec import UnsupportedStructTypeMessage from . import _argparse as argparse @@ -34,7 +35,11 @@ ) from ._typing import TypeForm from .conf import _confstruct, _markers -from .constructors._primitive_spec import UnsupportedTypeAnnotationError +from .constructors._primitive_spec import ( + PrimitiveConstructorSpec, + PrimitiveTypeInfo, + UnsupportedTypeAnnotationError, +) T = TypeVar("T") @@ -315,8 +320,20 @@ def handle_field( ]: """Determine what to do with a single field definition.""" + registry = ConstructorRegistry._get_active_registry() + + # Force primitive if (1) the field is annotated with a primitive constructor spec, or (2) if + force_primitive = len( + _resolver.unwrap_annotated(field.type, PrimitiveConstructorSpec)[1] + ) > 0 or ( + len(registry._custom_primitive_rules) > 0 + and registry.get_primitive_spec( + PrimitiveTypeInfo.make(field.type, field.markers), rule_mode="custom" + ) + is not None + ) if ( - field.primitive_spec is None + not force_primitive and _markers.Fixed not in field.markers and _markers.Suppress not in field.markers ): @@ -333,21 +350,22 @@ def handle_field( and _markers.AvoidSubcommands in field.markers ): # Don't make a subparser. - field = dataclasses.replace(field, type_or_callable=type(field.default)) + field = field.with_new_type_stripped(type(field.default)) else: return subparsers_attempt # (2) Handle nested callables. - if _fields.is_struct_type(field.type_or_callable, field.default): - field = dataclasses.replace( - field, - type_or_callable=_resolver.narrow_subtypes( - field.type_or_callable, + if force_primitive == "struct" or _fields.is_struct_type( + field.type_stripped, field.default + ): + field = field.with_new_type_stripped( + _resolver.narrow_subtypes( + field.type_stripped, field.default, ), ) return ParserSpecification.from_callable_or_type( - field.type_or_callable, + field.type_stripped, markers=field.markers, description=None, parent_classes=parent_classes, @@ -393,7 +411,7 @@ def from_field( extern_prefix: str, ) -> Optional[SubparsersSpecification]: # Union of classes should create subparsers. - typ = _resolver.unwrap_annotated(field.type_or_callable) + typ = _resolver.unwrap_annotated(field.type_stripped) if get_origin(typ) not in (Union, _resolver.UnionType): return None diff --git a/src/tyro/_subcommand_matching.py b/src/tyro/_subcommand_matching.py index 82e8d20bb..ed7bed7e0 100644 --- a/src/tyro/_subcommand_matching.py +++ b/src/tyro/_subcommand_matching.py @@ -90,7 +90,7 @@ def make( return _TypeTree( typ_unwrap, { - field.intern_name: _TypeTree.make(field.type_or_callable, field.default) + field.intern_name: _TypeTree.make(field.type_stripped, field.default) for field in field_list }, ) diff --git a/src/tyro/conf/__init__.py b/src/tyro/conf/__init__.py index 7ace6c511..cfa88d12e 100644 --- a/src/tyro/conf/__init__.py +++ b/src/tyro/conf/__init__.py @@ -3,7 +3,7 @@ annotations. Configuration flags are applied recursively, and should generally be subscripted: -`Fixed[T]`, `Suppress[T]`, etc. +``Fixed[T]``, ``Suppress[T]``, etc. Features here are supported, but generally unnecessary and should be used sparingly. """ diff --git a/src/tyro/conf/_confstruct.py b/src/tyro/conf/_confstruct.py index 0104c4d01..99ba847f0 100644 --- a/src/tyro/conf/_confstruct.py +++ b/src/tyro/conf/_confstruct.py @@ -53,35 +53,35 @@ def subcommand( constructor: type | Callable | None = None, constructor_factory: Callable[[], type | Callable] | None = None, ) -> Any: - """Returns a metadata object for configuring subcommands with `typing.Annotated`. - Useful for aesthetics. + """Returns a metadata object for configuring subcommands with + :py:data:`typing.Annotated`. Useful for aesthetics. Consider the standard approach for creating subcommands: - ```python - tyro.cli( - Union[NestedTypeA, NestedTypeB] - ) - ``` + .. code-block:: python + + tyro.cli( + Union[NestedTypeA, NestedTypeB] + ) This will create two subcommands: `nested-type-a` and `nested-type-b`. - Annotating each type with `tyro.conf.subcommand()` allows us to override for - each subcommand the (a) name, (b) defaults, (c) helptext, and (d) whether to prefix - the name or not. - - ```python - tyro.cli( - Union[ - Annotated[ - NestedTypeA, subcommand("a", ...) - ], - Annotated[ - NestedTypeB, subcommand("b", ...) - ], - ] - ) - ``` + Annotating each type with :func:`tyro.conf.subcommand()` allows us to + override for each subcommand the (a) name, (b) defaults, (c) helptext, and + (d) whether to prefix the name or not. + + .. code-block:: python + + tyro.cli( + Union[ + Annotated[ + NestedTypeA, subcommand("a", ...) + ], + Annotated[ + NestedTypeB, subcommand("b", ...) + ], + ] + ) Arguments: name: The name of the subcommand in the CLI. @@ -93,10 +93,10 @@ def subcommand( is in a nested structure. constructor: A constructor type or function. This will be used in place of the argument's type for parsing arguments. For more - configurability, see :module:`tyro.constructors`. + configurability, see :mod:`tyro.constructors`. constructor_factory: A function that returns a constructor type. This will be used in place of the argument's type for parsing arguments. - For more configurability, see :module:`tyro.constructors`. + For more configurability, see :mod:`tyro.constructors`. """ assert not ( constructor is not None and constructor_factory is not None @@ -163,18 +163,21 @@ def arg( constructor_factory: Callable[[], type | Callable] | None = None, ) -> Any: """Returns a metadata object for fine-grained argument configuration with - `typing.Annotated`. Should typically not be required. + :py:data:`typing.Annotated`. Should typically not be required. + + We support using :func:`arg()` at the root of arguments. For example: + + .. code-block:: python + + x: Annotated[int, tyro.conf.arg(...)] + + Nesting :func:`arg()` within other types is generally not supported: + + .. code-block:: python - We support using `arg()` at the root of arguments. For example: - ```python - x: Annotated[int, tyro.conf.arg(...)] - ``` + # Not supported. + x: list[Annotated[int, tyro.conf.arg(...)]] - Nesting `arg()` within other types is generally not supported: - ```python - # Not supported. - x: tuple[Annotated[int, tyro.conf.arg(...)], ...] - ``` Arguments: name: A new name for the argument in the CLI. @@ -191,10 +194,10 @@ def arg( it is in a nested structure. Arguments are prefixed by default. constructor: A constructor type or function. This will be used in place of the argument's type for parsing arguments. For more - configurability, see :module:`tyro.constructors`. + configurability, see :mod:`tyro.constructors`. constructor_factory: A function that returns a constructor type. This will be used in place of the argument's type for parsing arguments. - For more configurability, see :module:`tyro.constructors`. + For more configurability, see :mod:`tyro.constructors`. Returns: Object to attach via `typing.Annotated[]`. diff --git a/src/tyro/conf/_markers.py b/src/tyro/conf/_markers.py index 50377aeca..f96aa68e5 100644 --- a/src/tyro/conf/_markers.py +++ b/src/tyro/conf/_markers.py @@ -16,7 +16,7 @@ T = TypeVar("T") Positional = Annotated[T, None] -"""A type `T` can be annotated as `Positional[T]` if we want to parse it as a positional +"""A type ``T`` can be annotated as ``Positional[T]`` if we want to parse it as a positional argument.""" PositionalRequiredArgs = Annotated[T, None] @@ -37,12 +37,12 @@ # Perhaps Suppress should be Suppressed? But SuppressedFixed would be weird. Fixed = Annotated[T, None] -"""A type `T` can be annotated as `Fixed[T]` to prevent `tyro.cli` from parsing it; a -default value should be set instead. Fields that can't be parsed with defaults will also -be marked as fixed automatically.""" +"""A type ``T`` can be annotated as ``Fixed[T]`` to prevent :func:`tyro.cli` +from parsing it; a default value should be set instead. Fields that can't be +parsed with defaults will also be marked as fixed automatically.""" Suppress = Annotated[T, None] -"""A type `T` can be annotated as `Suppress[T]` to prevent `tyro.cli` from parsing it, and +"""A type ``T`` can be annotated as ``Suppress[T]`` to prevent :func:`tyro.cli` from parsing it, and to prevent it from showing up in helptext.""" SuppressFixed = Annotated[T, None] @@ -52,33 +52,33 @@ """Turn off flag conversion for booleans with default values. Instead, types annotated with `bool` will expect an explicit True or False. -Can be used directly on boolean annotations, `FlagConversionOff[bool]`, or recursively +Can be used directly on boolean annotations, ``FlagConversionOff[bool]``, or recursively applied to nested types.""" AvoidSubcommands = Annotated[T, None] """Avoid creating subcommands when a default is provided for unions over nested types. This simplifies CLI interfaces, but makes them less expressive. -Can be used directly on union types, `AvoidSubcommands[Union[...]]`, or recursively +Can be used directly on union types, ``AvoidSubcommands[Union[...]]``, or recursively applied to nested types.""" ConsolidateSubcommandArgs = Annotated[T, None] """Consolidate arguments applied to subcommands. Makes CLI less sensitive to argument ordering, at the cost of support for optional subcommands. -By default, `tyro` will generate a traditional CLI interface where args are applied to -the directly preceding subcommand. When we have two subcommands `s1` and `s2`: -``` +By default, :mod:`tyro` will generate a traditional CLI interface where args are applied to +the directly preceding subcommand. When we have two subcommands ``s1`` and ``s2``: +`` python x.py {--root options} s1 {--s1 options} s2 {--s2 options} -``` +`` This can be frustrating because the resulting CLI is sensitive to the positioning of options. To consolidate subcommands, we push arguments to the end, after all subcommands: -``` +`` python x.py s1 s2 {--root, s1, and s2 options} -``` +`` This is more robust to reordering of options, ensuring that any new options can simply be placed at the end of the command. @@ -88,16 +88,14 @@ """Make CLI inputs used for subcommands shorter by omitting the subcommand-specific portion of the prefix. -If we have a structure with the field: - - cmd: Union[NestedTypeA, NestedTypeB] +If we have a structure with the field ``cmd: Union[NestedTypeA, NestedTypeB]``: -By default, `--cmd.arg` may be generated as a flag for each dataclass in the union. -If subcommand prefixes are omitted, we would instead have `--arg`. +By default, ``--cmd.arg`` may be generated as a flag for each dataclass in the union. +If subcommand prefixes are omitted, we would instead have ``--arg``. -By default, `cmd:nested-type-a` and `cmd:nested-type-b` may be generated as subcommand. -If subcommand prefixes are omitted, we would instead have `nested-type-a` and -`nested-type-b`. +By default, ``cmd:nested-type-a`` and ``cmd:nested-type-b`` may be generated as subcommand. +If subcommand prefixes are omitted, we would instead have ``nested-type-a`` and +``nested-type-b``. """ OmitArgPrefixes = Annotated[T, None] @@ -107,31 +105,33 @@ cmd: NestedType -By default, `--cmd.arg` may be generated as a flag. If prefixes are omitted, we would -instead simply have `--arg`. +By default, ``--cmd.arg`` may be generated as a flag. If prefixes are omitted, we would +instead simply have ``--arg``. """ UseAppendAction = Annotated[T, None] """Use "append" actions for variable-length arguments. -Given an annotation like `x: list[int]`, this means that `x = [0, 1, 2]` can be set via -the CLI syntax `--x 0 --x 1 --x 2` instead of the default of `--x 0 1 2`. +Given an annotation like ``x: list[int]``, this means that ``x = [0, 1, 2]`` can be set via +the CLI syntax ``--x 0 --x 1 --x 2`` instead of the default of ``--x 0 1 2``. -The resulting syntax may be more user-friendly; for `tyro`, it also enables support for -otherwise ambiguous annotations like `list[list[int]]`. +The resulting syntax may be more user-friendly; for :mod:`tyro`, it also enables support for +otherwise ambiguous annotations like ``list[list[int]]``. -Can be applied to all variable-length sequences (`list[T]`, `Sequence[T]`, -`tuple[T, ...]`, etc), including dictionaries without default values. +Can be applied to all variable-length sequences (``list[T]``, ``Sequence[T]``, +``tuple[T, ...]``, etc), including dictionaries without default values. """ UseCounterAction = Annotated[T, None] -"""Use "counter" actions for integer arguments. Example usage: `verbose: UseCounterAction[int]`.""" +"""Use "counter" actions for integer arguments. Example usage: ``verbose: UseCounterAction[int]``.""" EnumChoicesFromValues = Annotated[T, None] """Populate choices from enum values rather than enum names. Example: -``` + +.. code-block:: python + class OutputFormats(enum.StrEnum): JSON = enum.auto() PRETTY = enum.auto() @@ -143,11 +143,10 @@ class Args: display_format: Annotated[ OutputFormats, tyro.conf.EnumChoicesFromValues ] = OutputFormats.PRETTY -``` -The above will result in `json`, `pretty`, `rich`, and `toml` (all lowercase) as choices, +The above will result in ``json``, ``pretty``, ``rich``, and ``toml`` (all lowercase) as choices, since the auto values for `StrEnum` (Python 3.11+) are lowercase transformations of the -names. Without this marker, the choices would be `JSON`, `PRETTY`, `RICH`, and `TOML`. +names. Without this marker, the choices would be ``JSON``, ``PRETTY``, ``RICH``, and ``TOML``. Enum aliases are not relevant when this marker is present. The first entry matching the chosen value will be selected. @@ -173,24 +172,24 @@ def __getitem__(self, key): def configure(*markers: Marker) -> Callable[[CallableType], CallableType]: """Decorator for applying configuration options. - Consider using the `config=` argument of `tyro.cli()` instead, which takes the same - config marker objects as inputs. + Consider using the ``config=`` argument of :func:`tyro.cli()` instead, + which takes the same config marker objects as inputs. - Configuration markers are implemented via `typing.Annotated` and straightforward + Configuration markers are implemented via :py:data:`typing.Annotated` and straightforward to apply to types, for example: - ```python - field: tyro.conf.FlagConversionOff[bool] - ``` + .. code-block:: python + + field: tyro.conf.FlagConversionOff[bool] This decorator makes markers applicable to general functions as well: - ```python - # Recursively apply FlagConversionOff to all fields in `main()`. - @tyro.conf.configure(tyro.conf.FlagConversionOff) - def main(field: bool) -> None: - ... - ``` + .. code-block:: python + + # Recursively apply FlagConversionOff to all fields in `main()`. + @tyro.conf.configure(tyro.conf.FlagConversionOff) + def main(field: bool) -> None: + ... Args: markers: Options to apply. diff --git a/src/tyro/constructors/__init__.py b/src/tyro/constructors/__init__.py index d93bfe0ac..5c41e2b0d 100644 --- a/src/tyro/constructors/__init__.py +++ b/src/tyro/constructors/__init__.py @@ -1,3 +1,12 @@ +"""The :mod:`tyro.constructors` submodule exposes tyro's API for defining +behavior for different types. + +.. warning:: + + This submodule is not needed for the majority of users. + +""" + from ._primitive_spec import PrimitiveConstructorSpec as PrimitiveConstructorSpec from ._primitive_spec import PrimitiveTypeInfo as PrimitiveTypeInfo from ._primitive_spec import ( diff --git a/src/tyro/constructors/_primitive_spec.py b/src/tyro/constructors/_primitive_spec.py index 8c113d6e8..56a1d4d7b 100644 --- a/src/tyro/constructors/_primitive_spec.py +++ b/src/tyro/constructors/_primitive_spec.py @@ -62,12 +62,19 @@ class PrimitiveTypeInfo: """The output of get_origin() on the static type.""" markers: set[_markers.Marker] """Set of tyro markers used to configure this field.""" + _primitive_spec: PrimitiveConstructorSpec | None + """Primitive constructor spec that was scraped from runtime annotations.""" @staticmethod def make( raw_annotation: TypeForm | Callable, parent_markers: set[_markers.Marker], ) -> PrimitiveTypeInfo: + _, primitive_specs = _resolver.unwrap_annotated( + raw_annotation, search_type=PrimitiveConstructorSpec + ) + primitive_spec = primitive_specs[0] if len(primitive_specs) > 0 else None + typ, extra_markers = _resolver.unwrap_annotated( raw_annotation, search_type=_markers._Marker ) @@ -75,6 +82,7 @@ def make( type=cast(TypeForm, typ), type_origin=get_origin(typ), markers=parent_markers | set(extra_markers), + _primitive_spec=primitive_spec, ) @@ -84,11 +92,11 @@ class PrimitiveConstructorSpec(Generic[T]): There are two ways to use this class: - First, we can include it in a type signature via `typing.Annotated`. + First, we can include it in a type signature via :class:`typing.Annotated`. This is the simplest for making local modifications to parsing behavior for individual fields. - Alternatively, it can be returned by a rule in a `PrimitiveConstructorRegistry`. + Alternatively, it can be returned by a rule in a :class:`ConstructorRegistry`. """ nargs: int | Literal["*"] @@ -123,7 +131,7 @@ def apply_default_primitive_rules(registry: ConstructorRegistry) -> None: from ._registry import ConstructorRegistry - @registry.primitive_rule + @registry._default_primitive_rule def any_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: if type_info.type is not Any: return None @@ -136,7 +144,7 @@ def any_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: # not to break it. vanilla_types = (int, str, float, complex, bytes, bytearray, json.loads) - @registry.primitive_rule + @registry._default_primitive_rule def basics_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: if type_info.type not in vanilla_types: return None @@ -159,7 +167,7 @@ def basics_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None if "torch" in sys.modules.keys(): import torch - @registry.primitive_rule + @registry._default_primitive_rule def torch_device_rule( type_info: PrimitiveTypeInfo, ) -> PrimitiveConstructorSpec | None: @@ -173,7 +181,7 @@ def torch_device_rule( str_from_instance=lambda instance: [str(instance)], ) - @registry.primitive_rule + @registry._default_primitive_rule def bool_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: if type_info.type is not bool: return None @@ -186,7 +194,7 @@ def bool_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: str_from_instance=lambda instance: ["True" if instance else "False"], ) - @registry.primitive_rule + @registry._default_primitive_rule def nonetype_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: if type_info.type is not type(None): return None @@ -199,7 +207,7 @@ def nonetype_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | No str_from_instance=lambda instance: ["None"], ) - @registry.primitive_rule + @registry._default_primitive_rule def path_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: if not ( type_info.type in (os.PathLike, pathlib.Path) @@ -217,7 +225,7 @@ def path_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: str_from_instance=lambda instance: [str(instance)], ) - @registry.primitive_rule + @registry._default_primitive_rule def enum_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: if not ( inspect.isclass(type_info.type) and issubclass(type_info.type, enum.Enum) @@ -250,7 +258,7 @@ def enum_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: choices=choices, ) - @registry.primitive_rule + @registry._default_primitive_rule def datetime_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: if type_info.type not in (datetime.datetime, datetime.date, datetime.time): return None @@ -271,7 +279,7 @@ def datetime_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | No str_from_instance=lambda instance: [instance.isoformat()], ) - @registry.primitive_rule + @registry._default_primitive_rule def vague_container_rule( type_info: PrimitiveTypeInfo, ) -> PrimitiveConstructorSpec | None: @@ -306,7 +314,7 @@ def vague_container_rule( ) ) - @registry.primitive_rule + @registry._default_primitive_rule def sequence_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: if type_info.type_origin not in ( collections.abc.Sequence, @@ -391,7 +399,7 @@ def str_from_instance(instance: Sequence) -> list[str]: choices=inner_spec.choices, ) - @registry.primitive_rule + @registry._default_primitive_rule def tuple_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: if type_info.type_origin is not tuple: return None @@ -456,7 +464,7 @@ def str_from_instance(instance: tuple) -> list[str]: and all(spec.is_instance(member) for member, spec in zip(x, inner_specs)), ) - @registry.primitive_rule + @registry._default_primitive_rule def dict_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: if type_info.type_origin not in (dict, collections.abc.Mapping): return None @@ -549,7 +557,7 @@ def str_from_instance(instance: dict) -> list[str]: str_from_instance=str_from_instance, ) - @registry.primitive_rule + @registry._default_primitive_rule def literal_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: if type_info.type_origin not in (Literal, LiteralAlternate): return None @@ -575,7 +583,7 @@ def literal_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | Non choices=str_choices, ) - @registry.primitive_rule + @registry._default_primitive_rule def union_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: if type_info.type_origin not in (Union, _resolver.UnionType): return None diff --git a/src/tyro/constructors/_registry.py b/src/tyro/constructors/_registry.py index 654379fc0..781d406aa 100644 --- a/src/tyro/constructors/_registry.py +++ b/src/tyro/constructors/_registry.py @@ -2,6 +2,8 @@ from typing import Any, Callable, ClassVar, Union +from typing_extensions import Literal + from ._primitive_spec import ( PrimitiveConstructorSpec, PrimitiveTypeInfo, @@ -58,7 +60,8 @@ class ConstructorRegistry: _old_registry: ConstructorRegistry | None = None def __init__(self) -> None: - self._primitive_rules: list[PrimitiveSpecRule] = [] + self._default_primitive_rules: list[PrimitiveSpecRule] = [] + self._custom_primitive_rules: list[PrimitiveSpecRule] = [] self._struct_rules: list[StructSpecRule] = [] # Apply the default primitive-handling rules. @@ -67,9 +70,17 @@ def __init__(self) -> None: def primitive_rule(self, rule: PrimitiveSpecRule) -> PrimitiveSpecRule: """Define a rule for constructing a primitive type from a string. The - most recently added rule will be applied first.""" + most recently added rule will be applied first. + + Custom primitive rules will take precedence over both default primitive + rules and struct rules + """ - self._primitive_rules.append(rule) + self._custom_primitive_rules.append(rule) + return rule + + def _default_primitive_rule(self, rule: PrimitiveSpecRule) -> PrimitiveSpecRule: + self._default_primitive_rules.append(rule) return rule def struct_rule(self, rule: StructSpecRule) -> StructSpecRule: @@ -80,13 +91,25 @@ def struct_rule(self, rule: StructSpecRule) -> StructSpecRule: return rule def get_primitive_spec( - self, type_info: PrimitiveTypeInfo + self, + type_info: PrimitiveTypeInfo, + rule_mode: Literal["default", "custom", "all"] = "all", ) -> PrimitiveConstructorSpec: """Get a constructor specification for a given type.""" - for spec_factory in self._primitive_rules[::-1]: - maybe_spec = spec_factory(type_info) - if maybe_spec is not None: - return maybe_spec + + if type_info._primitive_spec is not None: + return type_info._primitive_spec + + if rule_mode in ("custom", "all"): + for spec_factory in self._custom_primitive_rules[::-1]: + maybe_spec = spec_factory(type_info) + if maybe_spec is not None: + return maybe_spec + if rule_mode in ("default", "all"): + for spec_factory in self._default_primitive_rules[::-1]: + maybe_spec = spec_factory(type_info) + if maybe_spec is not None: + return maybe_spec raise UnsupportedTypeAnnotationError( f"Unsupported type annotation: {type_info.type}" @@ -97,6 +120,7 @@ def get_struct_spec( ) -> StructConstructorSpec | None: """Get a constructor specification for a given type. Returns `None` if unsuccessful.""" + for spec_factory in self._struct_rules[::-1]: maybe_spec = spec_factory(type_info) if maybe_spec is not None: diff --git a/src/tyro/constructors/_struct_spec.py b/src/tyro/constructors/_struct_spec.py index 06f0776f4..8f54c21c2 100644 --- a/src/tyro/constructors/_struct_spec.py +++ b/src/tyro/constructors/_struct_spec.py @@ -43,18 +43,40 @@ class StructFieldSpec: """Behavior specification for a single field in our callable.""" name: str + """The name of the field. This will be used as a keyword argument for the + struct's associated `instantiate(**kwargs)` function.""" type: TypeForm + """The type of the field. Can be either a primitive or a nested struct type.""" default: Any - is_default_overridden: bool - helptext: str | None + """The default value of the field.""" + is_default_overridden: bool = False + """Whether the default value was overridden by the default instance. Should + be set to False if the default value was assigned by the field itself.""" + helptext: str | None = None + """Helpjext for the field.""" # TODO: it's theoretically possible to override the argname with `None`. _call_argname: Any = None + """Private: the name of the argument to pass to the callable. This is used + for dictionary types.""" @dataclasses.dataclass(frozen=True) class StructConstructorSpec: + """Specification for a struct type, which is broken down into multiple + fields. + + Each struct type is instantiated by calling an `instantiate(**kwargs)` + function with keyword a set of keyword arguments. + + Unlike `PrimitiveConstructorSpec`, there is only one way to use this class. + It must be returned by a rule in `ConstructorRegistry`. + """ + instantiate: Callable[..., Any] + """Function to call to instantiate the struct.""" fields: tuple[StructFieldSpec, ...] + """Fields used to construct the callable. Each field is used as a keyword + argument for the `instantiate(**kwargs)` function.""" @dataclasses.dataclass(frozen=True) @@ -81,6 +103,7 @@ def make(f: TypeForm | Callable, default: Any) -> StructTypeInfo: f = typevar_context.origin_type f = _resolver.narrow_subtypes(f, default) f = _resolver.narrow_collection_types(f, default) + return StructTypeInfo( cast(TypeForm, f), parent_markers, default, typevar_context ) diff --git a/src/tyro/extras/__init__.py b/src/tyro/extras/__init__.py index 5341bc3df..fc85cb07b 100644 --- a/src/tyro/extras/__init__.py +++ b/src/tyro/extras/__init__.py @@ -1,6 +1,9 @@ """The :mod:`tyro.extras` submodule contains helpers that complement :func:`tyro.cli()`. -Compared to the core interface, APIs here are more likely to be changed or deprecated. +.. warning:: + + Compared to the core interface, APIs here are more likely to be changed or deprecated. + """ from .._argparse_formatter import set_accent_color as set_accent_color diff --git a/src/tyro/extras/_base_configs.py b/src/tyro/extras/_base_configs.py index 33157e076..44f90efa5 100644 --- a/src/tyro/extras/_base_configs.py +++ b/src/tyro/extras/_base_configs.py @@ -23,31 +23,32 @@ def overridable_config_cli( Example usage: - ```python - import dataclasses - import tyro + .. code-block:: python + import dataclasses - @dataclasses.dataclass - class Config: - a: int - b: str + import tyro - default_configs = { - "small": ( - "Small config", - Config(1, "small"), - ), - "big": ( - "Big config", - Config(100, "big"), - ), - } - config = tyro.extras.overridable_config_cli(default_configs) - print(config) - ``` + @dataclasses.dataclass + class Config: + a: int + b: str + + + default_configs = { + "small": ( + "Small config", + Config(1, "small"), + ), + "big": ( + "Big config", + Config(100, "big"), + ), + } + config = tyro.extras.overridable_config_cli(default_configs) + print(config) Args: configs: A dictionary of config names mapped to a tuple of @@ -75,34 +76,35 @@ def subcommand_type_from_defaults( ) -> TypeForm[T]: """Construct a Union type for defining subcommands that choose between defaults. - For example, when `defaults` is set to: + For example, when ``defaults`` is set to: + + .. code-block:: python - ```python - { - "small": Config(...), - "big": Config(...), - } - ``` + { + "small": Config(...), + "big": Config(...), + } We return: - ```python - Union[ - Annotated[ - Config, - tyro.conf.subcommand("small", default=Config(...)) - ], - Annotated[ - Config, - tyro.conf.subcommand("big", default=Config(...)) + .. code-block:: python + + Union[ + Annotated[ + Config, + tyro.conf.subcommand("small", default=Config(...)) + ], + Annotated[ + Config, + tyro.conf.subcommand("big", default=Config(...)) + ] ] - ] - ``` - Direct use of `typing.Union` and :func:`tyro.conf.subcommand()` should generally be + Direct use of :py:data:`typing.Union` and :func:`tyro.conf.subcommand()` should generally be preferred, but this function can be helpful for succinctness. .. warning:: + The type returned by this function can be safely used as an input to :func:`tyro.cli()`, but for static analysis when used for annotations we recommend applying a `TYPE_CHECKING` guard: diff --git a/src/tyro/extras/_choices_type.py b/src/tyro/extras/_choices_type.py index 57998afc2..78a47eb2c 100644 --- a/src/tyro/extras/_choices_type.py +++ b/src/tyro/extras/_choices_type.py @@ -9,12 +9,13 @@ def literal_type_from_choices(choices: Iterable[T]) -> TypeForm[T]: - """Generate a `typing.Literal[]` type that constrains values to a set of choices. + """Generate a :py:data:`typing.Literal` type that constrains values to a set of choices. - Using `Literal[...]` directly should generally be preferred, but this function can be + Using ``Literal[...]`` directly should generally be preferred, but this function can be helpful when choices are generated dynamically. .. warning:: + The type returned by this function can be safely used as an input to :func:`tyro.cli()`, but for static analysis when used for annotations we recommend applying a `TYPE_CHECKING` guard: diff --git a/src/tyro/extras/_serialization.py b/src/tyro/extras/_serialization.py index 352f974ab..1dab5787c 100644 --- a/src/tyro/extras/_serialization.py +++ b/src/tyro/extras/_serialization.py @@ -177,7 +177,12 @@ def from_yaml( stream: Union[str, IO[str], bytes, IO[bytes]], ) -> DataclassType: """Re-construct a dataclass instance from a yaml-compatible string, which should be - generated from `tyro.extras.to_yaml()`. + generated from :func:`tyro.extras.to_yaml()`. + + .. warning:: + + **Deprecated.** Serialization functionality is stable but deprecated. + It may be removed in a future version of :code:`tyro`. As a secondary feature aimed at enabling the use of :func:`tyro.cli` for general configuration use cases, we also introduce functions for human-readable dataclass @@ -187,10 +192,6 @@ def from_yaml( references enable custom tags that are robust against code reorganization and refactor, while a PyYAML backend enables serialization of arbitrary Python objects. - .. warning:: - Serialization functionality is stable but deprecated. It may be removed in a - future version of :code:`tyro`. - Args: cls: Type to reconstruct. stream: YAML to read from. @@ -208,7 +209,12 @@ def from_yaml( def to_yaml(instance: Any) -> str: """Serialize a dataclass; returns a yaml-compatible string that can be deserialized - via `tyro.extras.from_yaml()`. + via :func:`tyro.extras.from_yaml()`. + + .. warning:: + + **Deprecated.** Serialization functionality is stable but deprecated. + It may be removed in a future version of :code:`tyro`. As a secondary feature aimed at enabling the use of :func:`tyro.cli` for general configuration use cases, we also introduce functions for human-readable dataclass @@ -218,10 +224,6 @@ def to_yaml(instance: Any) -> str: references enable custom tags that are robust against code reorganization and refactor, while a PyYAML backend enables serialization of arbitrary Python objects. - .. warning:: - Serialization functionality is stable but deprecated. It may be removed in a - future version of :code:`tyro`. - Args: instance: Dataclass instance to serialize. diff --git a/src/tyro/extras/_subcommand_app.py b/src/tyro/extras/_subcommand_app.py index e0cac9ddc..21e72be4e 100644 --- a/src/tyro/extras/_subcommand_app.py +++ b/src/tyro/extras/_subcommand_app.py @@ -8,36 +8,40 @@ class SubcommandApp: - """This module provides a decorator-based API for subcommands in `tyro`, inspired by click. + """This module provides a decorator-based API for subcommands in :mod:`tyro`, inspired by click. Example: - ```python - from tyro.extras import SubcommandApp + .. code-block:: python - app = SubcommandApp() + from tyro.extras import SubcommandApp - @app.command - def greet(name: str, loud: bool = False): - '''Greet someone.''' - greeting = f"Hello, {name}!" - if loud: - greeting = greeting.upper() - print(greeting) + app = SubcommandApp() - @app.command(name="addition") - def add(a: int, b: int): - '''Add two numbers.''' - print(f"{a} + {b} = {a + b}") + @app.command + def greet(name: str, loud: bool = False): + '''Greet someone.''' + greeting = f"Hello, {name}!" + if loud: + greeting = greeting.upper() + print(greeting) - if __name__ == "__main__": - app.cli() - ``` + @app.command(name="addition") + def add(a: int, b: int): + '''Add two numbers.''' + print(f"{a} + {b} = {a + b}") + + if __name__ == "__main__": + app.cli() Usage: - `python my_script.py greet Alice` - `python my_script.py greet Bob --loud` - `python my_script.py addition 5 3` + + .. code-block:: bash + + python my_script.py greet Alice + python my_script.py greet Bob --loud + python my_script.py addition 5 3 + """ def __init__(self) -> None: @@ -62,7 +66,7 @@ def command( ) -> CallableT | Callable[[CallableT], CallableT]: """A decorator to register a function as a subcommand. - This method is inspired by Click's @cli.command() decorator. + This method is inspired by Click's ``@cli.command()`` decorator. It adds the decorated function to the list of subcommands. Args: diff --git a/src/tyro/extras/_subcommand_cli_from_dict.py b/src/tyro/extras/_subcommand_cli_from_dict.py index 038d3f635..b7933afcc 100644 --- a/src/tyro/extras/_subcommand_cli_from_dict.py +++ b/src/tyro/extras/_subcommand_cli_from_dict.py @@ -52,53 +52,53 @@ def subcommand_cli_from_dict( For an input like: - ```python - tyro.extras.subcommand_cli_from_dict( - { - "checkout": checkout, - "commit": commit, - } - ) - ``` + .. code-block:: python + + tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + } + ) This is internally accomplished by generating and calling: - ```python - from typing import Annotated, Any, Union - import tyro - - tyro.cli( - Union[ - Annotated[ - Any, - tyro.conf.subcommand(name="checkout", constructor=checkout), - ], - Annotated[ - Any, - tyro.conf.subcommand(name="commit", constructor=commit), - ], - ] - ) - ``` + .. code-block:: python + + from typing import Annotated, Any, Union + import tyro + + tyro.cli( + Union[ + Annotated[ + Any, + tyro.conf.subcommand(name="checkout", constructor=checkout), + ], + Annotated[ + Any, + tyro.conf.subcommand(name="commit", constructor=commit), + ], + ] + ) Args: subcommands: Dictionary that maps the subcommand name to function to call. prog: The name of the program printed in helptext. Mirrors argument from - `argparse.ArgumentParser()`. + :py:class:`argparse.ArgumentParser`. description: Description text for the parser, displayed when the --help flag is passed in. If not specified, `f`'s docstring is used. Mirrors argument from - `argparse.ArgumentParser()`. + :py:class:`argparse.ArgumentParser`. args: If set, parse arguments from a sequence of strings instead of the - commandline. Mirrors argument from `argparse.ArgumentParser.parse_args()`. + commandline. Mirrors argument from :py:meth:`argparse.ArgumentParser.parse_args()`. use_underscores: If True, use underscores as a word delimeter instead of hyphens. This primarily impacts helptext; underscores and hyphens are treated equivalently when parsing happens. We default helptext to hyphens to follow the GNU style guide. https://www.gnu.org/software/libc/manual/html_node/Argument-Syntax.html - console_outputs: If set to `False`, parsing errors and help messages will be - supressed. This can be useful for distributed settings, where `tyro.cli()` + console_outputs: If set to ``False``, parsing errors and help messages will be + supressed. This can be useful for distributed settings, where :func:`tyro.cli()` is called from multiple workers but we only want console outputs from the main one. - config: Sequence of config marker objects, from `tyro.conf`. + config: Sequence of config marker objects, from :mod:`tyro.conf`. """ # We need to form a union type, which requires at least two elements. assert len(subcommands) >= 2, "At least two subcommands are required." diff --git a/tests/test_custom_primitive.py b/tests/test_custom_constructors.py similarity index 66% rename from tests/test_custom_primitive.py rename to tests/test_custom_constructors.py index a2758999d..4c2a1cbe7 100644 --- a/tests/test_custom_primitive.py +++ b/tests/test_custom_constructors.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from typing import Any, Dict +from typing import Any, Dict, Union from typing_extensions import Annotated, get_args @@ -36,6 +36,13 @@ def main(x: Dict[str, Any]) -> Dict[str, Any]: with primitive_registry: assert tyro.cli(main, args=["--x", '{"a": 1}']) == {"a": 1} + def main_with_default(x: Dict[str, Any] = {"hello": 5}) -> Dict[str, Any]: + return x + + with primitive_registry: + assert tyro.cli(main_with_default, args=[]) == {"hello": 5} + assert tyro.cli(main_with_default, args=["--x", '{"a": 1}']) == {"a": 1} + def test_custom_primitive_annotated(): """Test that we can use typing.Annotated to specify custom constructors.""" @@ -44,3 +51,15 @@ def main(x: Annotated[Dict[str, Any], json_constructor_spec]) -> Dict[str, Any]: return x assert tyro.cli(main, args=["--x", '{"a": 1}']) == {"a": 1} + + +def test_custom_primitive_union(): + """Test that we can use typing.Annotated to specify custom constructors.""" + + def main( + x: Union[int, Annotated[Dict[str, Any], json_constructor_spec]], + ) -> Union[int, Dict[str, Any]]: + return x + + assert tyro.cli(main, args=["--x", "3"]) == 3 + assert tyro.cli(main, args=["--x", '{"a": 1}']) == {"a": 1} diff --git a/tests/test_errors.py b/tests/test_errors.py index 1e4c9fd5a..09ea7b532 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -572,7 +572,7 @@ def main2() -> None: def test_wrong_annotation() -> None: @dataclasses.dataclass class Args: - x: dict = None # type: ignore + x: Union[dict, int] = None # type: ignore with pytest.warns(UserWarning): assert tyro.cli(Args, args=[]).x is None diff --git a/tests/test_py311_generated/ok.py b/tests/test_py311_generated/ok.py deleted file mode 100644 index 73e5b33bc..000000000 --- a/tests/test_py311_generated/ok.py +++ /dev/null @@ -1,12 +0,0 @@ -from dataclasses import dataclass -from typing import Literal - -import tyro - - -@dataclass(frozen=True) -class Container[T]: - a: T - - -tyro.cli(Container[Container[bool] | Container[Literal["1", "2"]]]) diff --git a/tests/test_py311_generated/test_custom_primitive_generated.py b/tests/test_py311_generated/test_custom_constructors_generated.py similarity index 68% rename from tests/test_py311_generated/test_custom_primitive_generated.py rename to tests/test_py311_generated/test_custom_constructors_generated.py index 4244759a2..6e0cd35eb 100644 --- a/tests/test_py311_generated/test_custom_primitive_generated.py +++ b/tests/test_py311_generated/test_custom_constructors_generated.py @@ -34,6 +34,13 @@ def main(x: Dict[str, Any]) -> Dict[str, Any]: with primitive_registry: assert tyro.cli(main, args=["--x", '{"a": 1}']) == {"a": 1} + def main_with_default(x: Dict[str, Any] = {"hello": 5}) -> Dict[str, Any]: + return x + + with primitive_registry: + assert tyro.cli(main_with_default, args=[]) == {"hello": 5} + assert tyro.cli(main_with_default, args=["--x", '{"a": 1}']) == {"a": 1} + def test_custom_primitive_annotated(): """Test that we can use typing.Annotated to specify custom constructors.""" @@ -42,3 +49,15 @@ def main(x: Annotated[Dict[str, Any], json_constructor_spec]) -> Dict[str, Any]: return x assert tyro.cli(main, args=["--x", '{"a": 1}']) == {"a": 1} + + +def test_custom_primitive_union(): + """Test that we can use typing.Annotated to specify custom constructors.""" + + def main( + x: int | Annotated[Dict[str, Any], json_constructor_spec], + ) -> int | Dict[str, Any]: + return x + + assert tyro.cli(main, args=["--x", "3"]) == 3 + assert tyro.cli(main, args=["--x", '{"a": 1}']) == {"a": 1} diff --git a/tests/test_py311_generated/test_errors_generated.py b/tests/test_py311_generated/test_errors_generated.py index 81fdd1502..7cbe5a546 100644 --- a/tests/test_py311_generated/test_errors_generated.py +++ b/tests/test_py311_generated/test_errors_generated.py @@ -583,7 +583,7 @@ def main2() -> None: def test_wrong_annotation() -> None: @dataclasses.dataclass class Args: - x: dict = None # type: ignore + x: dict | int = None # type: ignore with pytest.warns(UserWarning): assert tyro.cli(Args, args=[]).x is None From afd5486f0812ee61f5d46dbd700d7002d0f2712f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 5 Nov 2024 17:17:04 -0800 Subject: [PATCH 473/491] `setuptools` => `hatchling` (#196) * Switch to hatchling * bump years --- LICENSE | 2 +- docs/source/conf.py | 2 +- pyproject.toml | 15 ++++++--------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/LICENSE b/LICENSE index 83cd2bb52..2fb1edd38 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 Brent Yi +Copyright (c) 2024 Brent Yi Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docs/source/conf.py b/docs/source/conf.py index a3f27c4b2..1142e677a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,7 +18,7 @@ # -- Project information ----------------------------------------------------- project = "tyro" -copyright = "2022" +copyright = "2024" author = "brentyi" # The short X.Y version diff --git a/pyproject.toml b/pyproject.toml index fe3ef57aa..5ba0d3cae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,9 @@ [build-system] -requires = ["setuptools>=61.0"] -build-backend = "setuptools.build_meta" +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build] +packages = ["src/tyro"] [project] name = "tyro" @@ -10,7 +13,7 @@ authors = [ version = "0.8.14" # TODO: currently needs to be synchronized manually with __init__.py. description = "Strongly typed, zero-effort CLI interfaces" readme = "README.md" -license = { text="MIT" } +license = { text = "MIT" } requires-python = ">=3.7" classifiers = [ "Programming Language :: Python :: 3", @@ -58,12 +61,6 @@ dev = [ [project.urls] "GitHub" = "https://github.com/brentyi/tyro" -[tool.setuptools.package-data] -tyro = ["py.typed"] - -[tool.isort] -profile = "black" - [tool.mypy] python_version = "3.12" ignore_missing_imports = true From afb9636fc1b0f380ecf78f41df33638665f14675 Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 7 Nov 2024 03:25:41 -0800 Subject: [PATCH 474/491] Rework docs and examples (#198) * Docs / examples rework * nits * Refinements * ruff * ignore mypy errors for docs update script --- README.md | 5 +- docs/requirements.txt | 1 + docs/source/conf.py | 2 +- .../examples/01_basics/01_functions.rst | 56 - .../examples/01_basics/02_dataclasses.rst | 57 - .../01_basics/03_dataclasses_defaults.rst | 81 -- .../examples/01_basics/04_collections.rst | 61 -- docs/source/examples/01_basics/05_flags.rst | 71 -- .../source/examples/01_basics/06_literals.rst | 40 - docs/source/examples/01_basics/07_unions.rst | 57 - docs/source/examples/01_basics/08_enums.rst | 62 -- .../source/examples/02_nesting/01_nesting.rst | 99 -- .../examples/02_nesting/02_subcommands.rst | 87 -- .../02_nesting/03_multiple_subcommands.rst | 103 -- .../02_nesting/04_nesting_in_containers.rst | 72 -- .../02_nesting/05_subcommands_func.rst | 76 -- .../03_config_systems/01_base_configs.rst | 132 --- .../03_config_systems/02_overriding_yaml.rst | 61 -- .../04_additional/01_positional_args.rst | 83 -- .../04_additional/02_dictionaries.rst | 82 -- .../examples/04_additional/03_tuples.rst | 65 -- .../examples/04_additional/04_classes.rst | 53 - .../examples/04_additional/05_generics.rst | 53 - .../04_additional/06_generics_py312.rst | 54 - .../source/examples/04_additional/07_conf.rst | 63 -- .../examples/04_additional/08_pydantic.rst | 54 - .../examples/04_additional/09_attrs.rst | 58 - .../source/examples/04_additional/10_flax.rst | 74 -- .../examples/04_additional/11_aliases.rst | 96 -- .../04_additional/12_type_statement.rst | 50 - .../examples/04_additional/13_counters.rst | 67 -- .../14_suppress_console_outputs.rst | 57 - .../15_decorator_subcommands.rst | 84 -- .../01_primitive_annotation.rst | 71 -- .../02_primitive_registry.rst | 81 -- docs/source/examples/basics.rst | 990 ++++++++++++++++++ docs/source/examples/custom_constructors.rst | 190 ++++ docs/source/examples/generics.rst | 157 +++ docs/source/examples/nested_structures.rst | 594 +++++++++++ docs/source/examples/overriding_configs.rst | 375 +++++++ docs/source/examples/pytorch_jax.rst | 159 +++ docs/source/examples/subcommands.rst | 573 ++++++++++ docs/source/goals_and_alternatives.md | 2 +- docs/source/helptext_generation.md | 5 +- docs/source/index.md | 55 +- docs/source/supported_types.rst | 7 +- docs/update_example_docs.py | 248 ++++- examples/01_basics/01_functions.py | 15 +- examples/01_basics/02_dataclasses.py | 19 +- examples/01_basics/03_dataclasses_defaults.py | 55 - examples/01_basics/03_multivalue.py | 39 + .../04_classes.py | 5 +- examples/01_basics/04_collections.py | 34 - .../01_basics/{05_flags.py => 04_flags.py} | 16 +- .../{06_literals.py => 05_choices.py} | 14 +- examples/01_basics/06_enums.py | 36 + examples/01_basics/07_unions.py | 10 +- examples/01_basics/08_enums.py | 35 - examples/01_basics/08_positional.py | 39 + .../07_conf.py => 01_basics/09_conf.py} | 10 +- examples/01_basics/10_aliases.py | 25 + .../11_type_aliases_py312.py} | 5 +- .../12_counters.py} | 9 +- examples/01_basics/README.rst | 5 + examples/02_nested_structures/01_nesting.py | 35 + .../02_nesting_in_func.py | 48 + .../03_nesting_containers.py | 44 + .../04_dictionaries.py} | 9 +- .../05_tuples.py} | 11 +- .../06_pydantic.py} | 7 +- .../07_attrs.py} | 7 +- examples/02_nested_structures/README.rst | 6 + examples/02_nesting/01_nesting.py | 72 -- examples/02_nesting/02_subcommands.py | 46 - .../02_nesting/04_nesting_in_containers.py | 60 -- examples/03_subcommands/01_subcommands.py | 53 + .../03_subcommands/02_subcommands_in_func.py | 60 ++ .../03_multiple_subcommands.py | 13 +- .../04_decorator_subcommands.py} | 13 +- .../05_subcommands_func.py | 11 +- examples/03_subcommands/README.rst | 5 + examples/04_additional/01_positional_args.py | 63 -- examples/04_additional/11_aliases.py | 41 - .../01_dataclasses_defaults.py | 56 + .../02_overriding_yaml.py | 15 +- .../03_choosing_base_configs.py} | 46 +- examples/04_overriding_configs/README.rst | 5 + .../01_generics_py312.py} | 16 +- .../02_generics.py} | 8 +- examples/05_generics/README.rst | 5 + .../01_primitive_annotation.py | 22 +- .../02_primitive_registry.py | 17 +- examples/06_custom_constructors/README.rst | 4 + .../01_pytorch_parallelism.py} | 5 +- .../10_flax.py => 07_pytorch_jax/02_flax.py} | 5 +- examples/07_pytorch_jax/README.rst | 4 + src/tyro/conf/_markers.py | 19 +- src/tyro/extras/_subcommand_app.py | 4 +- 98 files changed, 3923 insertions(+), 2906 deletions(-) delete mode 100644 docs/source/examples/01_basics/01_functions.rst delete mode 100644 docs/source/examples/01_basics/02_dataclasses.rst delete mode 100644 docs/source/examples/01_basics/03_dataclasses_defaults.rst delete mode 100644 docs/source/examples/01_basics/04_collections.rst delete mode 100644 docs/source/examples/01_basics/05_flags.rst delete mode 100644 docs/source/examples/01_basics/06_literals.rst delete mode 100644 docs/source/examples/01_basics/07_unions.rst delete mode 100644 docs/source/examples/01_basics/08_enums.rst delete mode 100644 docs/source/examples/02_nesting/01_nesting.rst delete mode 100644 docs/source/examples/02_nesting/02_subcommands.rst delete mode 100644 docs/source/examples/02_nesting/03_multiple_subcommands.rst delete mode 100644 docs/source/examples/02_nesting/04_nesting_in_containers.rst delete mode 100644 docs/source/examples/02_nesting/05_subcommands_func.rst delete mode 100644 docs/source/examples/03_config_systems/01_base_configs.rst delete mode 100644 docs/source/examples/03_config_systems/02_overriding_yaml.rst delete mode 100644 docs/source/examples/04_additional/01_positional_args.rst delete mode 100644 docs/source/examples/04_additional/02_dictionaries.rst delete mode 100644 docs/source/examples/04_additional/03_tuples.rst delete mode 100644 docs/source/examples/04_additional/04_classes.rst delete mode 100644 docs/source/examples/04_additional/05_generics.rst delete mode 100644 docs/source/examples/04_additional/06_generics_py312.rst delete mode 100644 docs/source/examples/04_additional/07_conf.rst delete mode 100644 docs/source/examples/04_additional/08_pydantic.rst delete mode 100644 docs/source/examples/04_additional/09_attrs.rst delete mode 100644 docs/source/examples/04_additional/10_flax.rst delete mode 100644 docs/source/examples/04_additional/11_aliases.rst delete mode 100644 docs/source/examples/04_additional/12_type_statement.rst delete mode 100644 docs/source/examples/04_additional/13_counters.rst delete mode 100644 docs/source/examples/04_additional/14_suppress_console_outputs.rst delete mode 100644 docs/source/examples/04_additional/15_decorator_subcommands.rst delete mode 100644 docs/source/examples/05_custom_constructors/01_primitive_annotation.rst delete mode 100644 docs/source/examples/05_custom_constructors/02_primitive_registry.rst create mode 100644 docs/source/examples/basics.rst create mode 100644 docs/source/examples/custom_constructors.rst create mode 100644 docs/source/examples/generics.rst create mode 100644 docs/source/examples/nested_structures.rst create mode 100644 docs/source/examples/overriding_configs.rst create mode 100644 docs/source/examples/pytorch_jax.rst create mode 100644 docs/source/examples/subcommands.rst delete mode 100644 examples/01_basics/03_dataclasses_defaults.py create mode 100644 examples/01_basics/03_multivalue.py rename examples/{04_additional => 01_basics}/04_classes.py (86%) delete mode 100644 examples/01_basics/04_collections.py rename examples/01_basics/{05_flags.py => 04_flags.py} (70%) rename examples/01_basics/{06_literals.py => 05_choices.py} (61%) create mode 100644 examples/01_basics/06_enums.py delete mode 100644 examples/01_basics/08_enums.py create mode 100644 examples/01_basics/08_positional.py rename examples/{04_additional/07_conf.py => 01_basics/09_conf.py} (81%) create mode 100644 examples/01_basics/10_aliases.py rename examples/{04_additional/12_type_statement.py => 01_basics/11_type_aliases_py312.py} (89%) rename examples/{04_additional/13_counters.py => 01_basics/12_counters.py} (81%) create mode 100644 examples/01_basics/README.rst create mode 100644 examples/02_nested_structures/01_nesting.py create mode 100644 examples/02_nested_structures/02_nesting_in_func.py create mode 100644 examples/02_nested_structures/03_nesting_containers.py rename examples/{04_additional/02_dictionaries.py => 02_nested_structures/04_dictionaries.py} (82%) rename examples/{04_additional/03_tuples.py => 02_nested_structures/05_tuples.py} (71%) rename examples/{04_additional/08_pydantic.py => 02_nested_structures/06_pydantic.py} (77%) rename examples/{04_additional/09_attrs.py => 02_nested_structures/07_attrs.py} (79%) create mode 100644 examples/02_nested_structures/README.rst delete mode 100644 examples/02_nesting/01_nesting.py delete mode 100644 examples/02_nesting/02_subcommands.py delete mode 100644 examples/02_nesting/04_nesting_in_containers.py create mode 100644 examples/03_subcommands/01_subcommands.py create mode 100644 examples/03_subcommands/02_subcommands_in_func.py rename examples/{02_nesting => 03_subcommands}/03_multiple_subcommands.py (68%) rename examples/{04_additional/15_decorator_subcommands.py => 03_subcommands/04_decorator_subcommands.py} (64%) rename examples/{02_nesting => 03_subcommands}/05_subcommands_func.py (72%) create mode 100644 examples/03_subcommands/README.rst delete mode 100644 examples/04_additional/01_positional_args.py delete mode 100644 examples/04_additional/11_aliases.py create mode 100644 examples/04_overriding_configs/01_dataclasses_defaults.py rename examples/{03_config_systems => 04_overriding_configs}/02_overriding_yaml.py (72%) rename examples/{03_config_systems/01_base_configs.py => 04_overriding_configs/03_choosing_base_configs.py} (63%) create mode 100644 examples/04_overriding_configs/README.rst rename examples/{04_additional/06_generics_py312.py => 05_generics/01_generics_py312.py} (68%) rename examples/{04_additional/05_generics.py => 05_generics/02_generics.py} (77%) create mode 100644 examples/05_generics/README.rst rename examples/{05_custom_constructors => 06_custom_constructors}/01_primitive_annotation.py (50%) rename examples/{05_custom_constructors => 06_custom_constructors}/02_primitive_registry.py (62%) create mode 100644 examples/06_custom_constructors/README.rst rename examples/{04_additional/14_suppress_console_outputs.py => 07_pytorch_jax/01_pytorch_parallelism.py} (88%) rename examples/{04_additional/10_flax.py => 07_pytorch_jax/02_flax.py} (94%) create mode 100644 examples/07_pytorch_jax/README.rst diff --git a/README.md b/README.md index 2efb09f4e..a2455b4f0 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@
tyro.cli() is a tool for generating CLI -interfaces. +interfaces in Python. We can define configurable scripts using functions: @@ -211,9 +211,10 @@ particularly like: [jsonargparse](https://github.com/omni-us/jsonargparse), which provide deeper integration with configuration file formats like YAML and JSON. - [clipstick](https://github.com/sander76/clipstick), which focuses on - generating CLIs from Pydantic models. + simplicity + generating CLIs from Pydantic models. - [datargs](https://github.com/roee30/datargs), which provides a minimal API for dataclasses. +- [defopt](https://defopt.readthedocs.io/), which has similarly comprehensive type annotation support. - [fire](https://github.com/google/python-fire) and [clize](https://github.com/epsy/clize), which support arguments without type annotations. diff --git a/docs/requirements.txt b/docs/requirements.txt index 5e86e9dc9..597a81bca 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,6 +3,7 @@ furo==2024.8.6 docutils==0.20.1 sphinx-autoapi==3.0.0 m2r2==0.3.3.post2 +ansi2html==1.9.2 git+https://github.com/brentyi/sphinxcontrib-programoutput.git git+https://github.com/brentyi/ansi.git sphinxcontrib-googleanalytics==0.4 diff --git a/docs/source/conf.py b/docs/source/conf.py index 1142e677a..02c1a5177 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -112,7 +112,7 @@ exclude_patterns: List[str] = [] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = "monokai" +pygments_style = "default" # -- Options for HTML output ------------------------------------------------- diff --git a/docs/source/examples/01_basics/01_functions.rst b/docs/source/examples/01_basics/01_functions.rst deleted file mode 100644 index e5729e93b..000000000 --- a/docs/source/examples/01_basics/01_functions.rst +++ /dev/null @@ -1,56 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Functions -========================================== - -In the simplest case, :func:`tyro.cli()` can be used to run a function with -arguments populated from the CLI. - - -.. code-block:: python - :linenos: - - - import tyro - - - def main( - field1: str, - field2: int = 3, - ) -> None: - """Function, whose arguments will be populated from a CLI interface. - - Args: - field1: A string field. - field2: A numeric field, with a default value. - """ - print(field1, field2) - - - if __name__ == "__main__": - tyro.cli(main) - ------------- - -.. raw:: html - - python 01_basics/01_functions.py --help - -.. program-output:: python ../../examples/01_basics/01_functions.py --help - ------------- - -.. raw:: html - - python 01_basics/01_functions.py --field1 hello - -.. program-output:: python ../../examples/01_basics/01_functions.py --field1 hello - ------------- - -.. raw:: html - - python 01_basics/01_functions.py --field1 hello --field2 10 - -.. program-output:: python ../../examples/01_basics/01_functions.py --field1 hello --field2 10 diff --git a/docs/source/examples/01_basics/02_dataclasses.rst b/docs/source/examples/01_basics/02_dataclasses.rst deleted file mode 100644 index 01f7acb78..000000000 --- a/docs/source/examples/01_basics/02_dataclasses.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Dataclasses -========================================== - -Common pattern: use :func:`tyro.cli()` to instantiate a dataclass. - - -.. code-block:: python - :linenos: - - - import dataclasses - - import tyro - - - @dataclasses.dataclass - class Args: - """Description. - This should show up in the helptext!""" - - field1: str - """A string field.""" - - field2: int = 3 - """A numeric field, with a default value.""" - - - if __name__ == "__main__": - args = tyro.cli(Args) - print(args) - ------------- - -.. raw:: html - - python 01_basics/02_dataclasses.py --help - -.. program-output:: python ../../examples/01_basics/02_dataclasses.py --help - ------------- - -.. raw:: html - - python 01_basics/02_dataclasses.py --field1 hello - -.. program-output:: python ../../examples/01_basics/02_dataclasses.py --field1 hello - ------------- - -.. raw:: html - - python 01_basics/02_dataclasses.py --field1 hello --field2 5 - -.. program-output:: python ../../examples/01_basics/02_dataclasses.py --field1 hello --field2 5 diff --git a/docs/source/examples/01_basics/03_dataclasses_defaults.rst b/docs/source/examples/01_basics/03_dataclasses_defaults.rst deleted file mode 100644 index 6e857f189..000000000 --- a/docs/source/examples/01_basics/03_dataclasses_defaults.rst +++ /dev/null @@ -1,81 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Dataclasses + Defaults -========================================== - -The :code:`default=` argument can be used to override default values in dataclass -types. - - -.. warning:: - - We advise against mutation of configuration objects from a dataclass's - :code:`__post_init__` method [#f1]_. In the example below, - :code:`__post_init__` would be called twice: once for the :code:`Args()` - object provided as a default value and another time for the :code:`Args()` - objected instantiated by :func:`tyro.cli()`. This can cause confusing - behavior! Instead, we show below one example of how derived fields can be - defined immutably. - - .. [#f1] Official Python docs for ``__post_init__`` can be found `here `_. - - -.. code-block:: python - :linenos: - - - import dataclasses - - import tyro - - - @dataclasses.dataclass - class Args: - """Description. - This should show up in the helptext!""" - - field1: str - """A string field.""" - - field2: int = 3 - """A numeric field, with a default value.""" - - @property - def derived_field(self) -> str: - return ", ".join([self.field1] * self.field2) - - - if __name__ == "__main__": - args = tyro.cli( - Args, - default=Args( - field1="default string", - field2=tyro.MISSING, - ), - ) - print(args.derived_field) - ------------- - -.. raw:: html - - python 01_basics/03_dataclasses_defaults.py --help - -.. program-output:: python ../../examples/01_basics/03_dataclasses_defaults.py --help - ------------- - -.. raw:: html - - python 01_basics/03_dataclasses_defaults.py --field2 3 - -.. program-output:: python ../../examples/01_basics/03_dataclasses_defaults.py --field2 3 - ------------- - -.. raw:: html - - python 01_basics/03_dataclasses_defaults.py --field1 hello --field2 5 - -.. program-output:: python ../../examples/01_basics/03_dataclasses_defaults.py --field1 hello --field2 5 diff --git a/docs/source/examples/01_basics/04_collections.rst b/docs/source/examples/01_basics/04_collections.rst deleted file mode 100644 index bd9cf55cd..000000000 --- a/docs/source/examples/01_basics/04_collections.rst +++ /dev/null @@ -1,61 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Multi-value Arguments -========================================== - -Arguments of both fixed and variable lengths can be annotated with standard -Python collection types. For Python 3.7 and 3.8, we can use either :code:`from -__future__ import annotations` to support :code:`list[T]` and :code:`tuple[T]`, -or the older API :code:`typing.List[T]` and :code:`typing.Tuple[T1, T2]`. - - -.. code-block:: python - :linenos: - - - import dataclasses - import pathlib - - import tyro - - - @dataclasses.dataclass(frozen=True) - class TrainConfig: - # Example of a variable-length tuple. `list[T]`, `set[T]`, - # `dict[K, V]`, etc are supported as well. - dataset_sources: tuple[pathlib.Path, ...] - """Paths to load training data from. This can be multiple!""" - - # Fixed-length tuples are also okay. - image_dimensions: tuple[int, int] = (32, 32) - """Height and width of some image data.""" - - - if __name__ == "__main__": - config = tyro.cli(TrainConfig) - print(config) - ------------- - -.. raw:: html - - python 01_basics/04_collections.py --help - -.. program-output:: python ../../examples/01_basics/04_collections.py --help - ------------- - -.. raw:: html - - python 01_basics/04_collections.py --dataset-sources ./data --image-dimensions 16 16 - -.. program-output:: python ../../examples/01_basics/04_collections.py --dataset-sources ./data --image-dimensions 16 16 - ------------- - -.. raw:: html - - python 01_basics/04_collections.py --dataset-sources ./data - -.. program-output:: python ../../examples/01_basics/04_collections.py --dataset-sources ./data diff --git a/docs/source/examples/01_basics/05_flags.rst b/docs/source/examples/01_basics/05_flags.rst deleted file mode 100644 index 433e6af72..000000000 --- a/docs/source/examples/01_basics/05_flags.rst +++ /dev/null @@ -1,71 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Booleans and Flags -========================================== - -Booleans can either be expected to be explicitly passed in, or, if given a default -value, automatically converted to flags. - -To turn off conversion, see :class:`tyro.conf.FlagConversionOff`. - - -.. code-block:: python - :linenos: - - - import dataclasses - - import tyro - - - @dataclasses.dataclass - class Args: - # Boolean. This expects an explicit "True" or "False". - boolean: bool - - # Optional boolean. Same as above, but can be omitted. - optional_boolean: bool | None = None - - # Pass --flag-a in to set this value to True. - flag_a: bool = False - - # Pass --no-flag-b in to set this value to False. - flag_b: bool = True - - - if __name__ == "__main__": - args = tyro.cli(Args) - print(args) - ------------- - -.. raw:: html - - python 01_basics/05_flags.py --help - -.. program-output:: python ../../examples/01_basics/05_flags.py --help - ------------- - -.. raw:: html - - python 01_basics/05_flags.py --boolean True - -.. program-output:: python ../../examples/01_basics/05_flags.py --boolean True - ------------- - -.. raw:: html - - python 01_basics/05_flags.py --boolean False --flag-a - -.. program-output:: python ../../examples/01_basics/05_flags.py --boolean False --flag-a - ------------- - -.. raw:: html - - python 01_basics/05_flags.py --boolean False --no-flag-b - -.. program-output:: python ../../examples/01_basics/05_flags.py --boolean False --no-flag-b diff --git a/docs/source/examples/01_basics/06_literals.rst b/docs/source/examples/01_basics/06_literals.rst deleted file mode 100644 index 746e7ec61..000000000 --- a/docs/source/examples/01_basics/06_literals.rst +++ /dev/null @@ -1,40 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Choices -========================================== - -:code:`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices. - - -.. code-block:: python - :linenos: - - - import dataclasses - from typing import Literal - - import tyro - - - @dataclasses.dataclass(frozen=True) - class Args: - # We can use Literal[] to restrict the set of allowable inputs, for example, over - # a set of strings. - strings: Literal["red", "green"] = "red" - - # Integers also work. (as well as booleans, enums, etc) - numbers: Literal[0, 1, 2] = 0 - - - if __name__ == "__main__": - args = tyro.cli(Args) - print(args) - ------------- - -.. raw:: html - - python 01_basics/06_literals.py --help - -.. program-output:: python ../../examples/01_basics/06_literals.py --help diff --git a/docs/source/examples/01_basics/07_unions.rst b/docs/source/examples/01_basics/07_unions.rst deleted file mode 100644 index 8ce43f511..000000000 --- a/docs/source/examples/01_basics/07_unions.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Unions -========================================== - -:code:`X | Y` or :code:`typing.Union[X, Y]` can be used to expand inputs to -multiple types. - - -.. code-block:: python - :linenos: - - - import dataclasses - import enum - from typing import Literal, Optional - - import tyro - - - class Color(enum.Enum): - RED = enum.auto() - GREEN = enum.auto() - BLUE = enum.auto() - - - @dataclasses.dataclass(frozen=True) - class Args: - # Unions can be used to specify multiple allowable types. - union_over_types: int | str = 0 - string_or_enum: Literal["red", "green"] | Color = "red" - - # Unions also work over more complex nested types. - union_over_tuples: tuple[int, int] | tuple[str] = ("1",) - - # And can be nested in other types. - tuple_of_string_or_enum: tuple[Literal["red", "green"] | Color, ...] = ( - "red", - Color.RED, - ) - - # Optional[T] is equivalent to `T | None`. - integer: Optional[Literal[0, 1, 2, 3]] = None - - - if __name__ == "__main__": - args = tyro.cli(Args) - print(args) - ------------- - -.. raw:: html - - python 01_basics/07_unions.py --help - -.. program-output:: python ../../examples/01_basics/07_unions.py --help diff --git a/docs/source/examples/01_basics/08_enums.rst b/docs/source/examples/01_basics/08_enums.rst deleted file mode 100644 index 08f142963..000000000 --- a/docs/source/examples/01_basics/08_enums.rst +++ /dev/null @@ -1,62 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Enums -========================================== - -In addition to literals, enums can also be used to provide a fixed set of -choices. - - -.. code-block:: python - :linenos: - - - import dataclasses - import enum - - import tyro - - - class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - - @dataclasses.dataclass(frozen=True) - class TrainConfig: - # Enums are handled seamlessly. - optimizer_type: OptimizerType = OptimizerType.ADAM - """Gradient-based optimizer to use.""" - - learning_rate: float = 1e-4 - """Learning rate for optimizer.""" - - - if __name__ == "__main__": - config = tyro.cli(TrainConfig) - print(config) - ------------- - -.. raw:: html - - python 01_basics/08_enums.py --help - -.. program-output:: python ../../examples/01_basics/08_enums.py --help - ------------- - -.. raw:: html - - python 01_basics/08_enums.py --optimizer-type SGD - -.. program-output:: python ../../examples/01_basics/08_enums.py --optimizer-type SGD - ------------- - -.. raw:: html - - python 01_basics/08_enums.py --optimizer-type ADAM --learning-rate 3e-4 - -.. program-output:: python ../../examples/01_basics/08_enums.py --optimizer-type ADAM --learning-rate 3e-4 diff --git a/docs/source/examples/02_nesting/01_nesting.rst b/docs/source/examples/02_nesting/01_nesting.rst deleted file mode 100644 index d56858eeb..000000000 --- a/docs/source/examples/02_nesting/01_nesting.rst +++ /dev/null @@ -1,99 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Hierarchical Configs -========================================== - -Structures (typically dataclasses) can be nested to build hierarchical configuration -objects. This helps with modularity and grouping in larger projects. - - -.. code-block:: python - :linenos: - - - import dataclasses - import enum - import pathlib - - import tyro - - - class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - - @dataclasses.dataclass - class OptimizerConfig: - # Gradient-based optimizer to use. - algorithm: OptimizerType = OptimizerType.ADAM - - # Learning rate to use. - learning_rate: float = 3e-4 - - # Coefficient for L2 regularization. - weight_decay: float = 1e-2 - - - @dataclasses.dataclass - class ExperimentConfig: - # Various configurable options for our optimizer. - optimizer: OptimizerConfig - - # Batch size. - batch_size: int = 32 - - # Total number of training steps. - train_steps: int = 100_000 - - # Random seed. This is helpful for making sure that our experiments are all - # reproducible! - seed: int = 0 - - - def train( - out_dir: pathlib.Path, - config: ExperimentConfig, - restore_checkpoint: bool = False, - checkpoint_interval: int = 1000, - ) -> None: - """Train a model. - - Args: - out_dir: Where to save logs and checkpoints. - config: Experiment configuration. - restore_checkpoint: Set to restore an existing checkpoint. - checkpoint_interval: Training steps between each checkpoint save. - """ - print(f"{out_dir=}, {restore_checkpoint=}, {checkpoint_interval=}") - print() - print(f"{config=}") - - - if __name__ == "__main__": - tyro.cli(train) - ------------- - -.. raw:: html - - python 02_nesting/01_nesting.py --help - -.. program-output:: python ../../examples/02_nesting/01_nesting.py --help - ------------- - -.. raw:: html - - python 02_nesting/01_nesting.py --out-dir . --config.optimizer.algorithm SGD - -.. program-output:: python ../../examples/02_nesting/01_nesting.py --out-dir . --config.optimizer.algorithm SGD - ------------- - -.. raw:: html - - python 02_nesting/01_nesting.py --out-dir . --restore-checkpoint - -.. program-output:: python ../../examples/02_nesting/01_nesting.py --out-dir . --restore-checkpoint diff --git a/docs/source/examples/02_nesting/02_subcommands.rst b/docs/source/examples/02_nesting/02_subcommands.rst deleted file mode 100644 index 1992a3bbc..000000000 --- a/docs/source/examples/02_nesting/02_subcommands.rst +++ /dev/null @@ -1,87 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Subcommands -========================================== - -Unions over nested types (classes or dataclasses) are populated using subcommands. - -For configuring subcommands beyond what can be expressed with type annotations, see -:func:`tyro.conf.subcommand()`. - - -.. code-block:: python - :linenos: - - - from __future__ import annotations - - import dataclasses - - import tyro - - - @dataclasses.dataclass(frozen=True) - class Checkout: - """Checkout a branch.""" - - branch: str - - - @dataclasses.dataclass(frozen=True) - class Commit: - """Commit changes.""" - - message: str - all: bool = False - - - def main(cmd: Checkout | Commit) -> None: - print(cmd) - - - if __name__ == "__main__": - # Note that we can also pass `Checkout | Command` directly into - # `tyro.cli()`; this is understood by tyro and pyright, but unfortunately not by - # mypy. - tyro.cli(main) - ------------- - -.. raw:: html - - python 02_nesting/02_subcommands.py --help - -.. program-output:: python ../../examples/02_nesting/02_subcommands.py --help - ------------- - -.. raw:: html - - python 02_nesting/02_subcommands.py cmd:commit --help - -.. program-output:: python ../../examples/02_nesting/02_subcommands.py cmd:commit --help - ------------- - -.. raw:: html - - python 02_nesting/02_subcommands.py cmd:commit --cmd.message hello --cmd.all - -.. program-output:: python ../../examples/02_nesting/02_subcommands.py cmd:commit --cmd.message hello --cmd.all - ------------- - -.. raw:: html - - python 02_nesting/02_subcommands.py cmd:checkout --help - -.. program-output:: python ../../examples/02_nesting/02_subcommands.py cmd:checkout --help - ------------- - -.. raw:: html - - python 02_nesting/02_subcommands.py cmd:checkout --cmd.branch main - -.. program-output:: python ../../examples/02_nesting/02_subcommands.py cmd:checkout --cmd.branch main diff --git a/docs/source/examples/02_nesting/03_multiple_subcommands.rst b/docs/source/examples/02_nesting/03_multiple_subcommands.rst deleted file mode 100644 index 5064916a8..000000000 --- a/docs/source/examples/02_nesting/03_multiple_subcommands.rst +++ /dev/null @@ -1,103 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Sequenced Subcommands -========================================== - -Multiple unions over nested types are populated using a series of subcommands. - - -.. code-block:: python - :linenos: - - - from __future__ import annotations - - import dataclasses - from typing import Literal - - import tyro - - # Possible dataset configurations. - - - @dataclasses.dataclass - class Mnist: - binary: bool = False - """Set to load binary version of MNIST dataset.""" - - - @dataclasses.dataclass - class ImageNet: - subset: Literal[50, 100, 1000] - """Choose between ImageNet-50, ImageNet-100, ImageNet-1000, etc.""" - - - # Possible optimizer configurations. - - - @dataclasses.dataclass - class Adam: - learning_rate: float = 1e-3 - betas: tuple[float, float] = (0.9, 0.999) - - - @dataclasses.dataclass - class Sgd: - learning_rate: float = 3e-4 - - - # Train script. - - - def train( - dataset: Mnist | ImageNet = Mnist(), - optimizer: Adam | Sgd = Adam(), - ) -> None: - """Example training script. - - Args: - dataset: Dataset to train on. - optimizer: Optimizer to train with. - - Returns: - None: - """ - print(dataset) - print(optimizer) - - - if __name__ == "__main__": - tyro.cli(train, config=(tyro.conf.ConsolidateSubcommandArgs,)) - ------------- - -.. raw:: html - - python 02_nesting/03_multiple_subcommands.py --help - -.. program-output:: python ../../examples/02_nesting/03_multiple_subcommands.py --help - ------------- - -.. raw:: html - - python 02_nesting/03_multiple_subcommands.py dataset:mnist --help - -.. program-output:: python ../../examples/02_nesting/03_multiple_subcommands.py dataset:mnist --help - ------------- - -.. raw:: html - - python 02_nesting/03_multiple_subcommands.py dataset:mnist optimizer:adam --help - -.. program-output:: python ../../examples/02_nesting/03_multiple_subcommands.py dataset:mnist optimizer:adam --help - ------------- - -.. raw:: html - - python 02_nesting/03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 --dataset.binary - -.. program-output:: python ../../examples/02_nesting/03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 --dataset.binary diff --git a/docs/source/examples/02_nesting/04_nesting_in_containers.rst b/docs/source/examples/02_nesting/04_nesting_in_containers.rst deleted file mode 100644 index d9d5a812f..000000000 --- a/docs/source/examples/02_nesting/04_nesting_in_containers.rst +++ /dev/null @@ -1,72 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Nesting in Containers -========================================== - -Structures can be nested inside of standard containers. - -Note that lengths must be inferable, either via a fixed-length tuple annotation or by -parsing default values. - - -.. code-block:: python - :linenos: - - - import dataclasses - - import tyro - - - class Color: - pass - - - @dataclasses.dataclass - class RGB(Color): - r: int - g: int - b: int - - - @dataclasses.dataclass - class HSL(Color): - h: int - s: int - l: int - - - @dataclasses.dataclass - class Args: - # Example of specifying nested structures via a fixed-length tuple. - color_tuple: tuple[RGB, HSL] - - # Examples of nested structures in variable-length containers. These need a default - # provided for length inference; we don't currently support specifying dynamic - # container lengths directly from the commandline. - color_tuple_alt: tuple[Color, ...] = ( - RGB(255, 0, 0), - HSL(0, 255, 0), - ) - color_map: dict[str, RGB] = dataclasses.field( - # We can't use mutable values as defaults directly. - default_factory={ - "red": RGB(255, 0, 0), - "green": RGB(0, 255, 0), - "blue": RGB(0, 0, 255), - }.copy - ) - - - if __name__ == "__main__": - args = tyro.cli(Args) - print(args) - ------------- - -.. raw:: html - - python 02_nesting/04_nesting_in_containers.py --help - -.. program-output:: python ../../examples/02_nesting/04_nesting_in_containers.py --help diff --git a/docs/source/examples/02_nesting/05_subcommands_func.rst b/docs/source/examples/02_nesting/05_subcommands_func.rst deleted file mode 100644 index 141c9a53d..000000000 --- a/docs/source/examples/02_nesting/05_subcommands_func.rst +++ /dev/null @@ -1,76 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Subcommands from Functions -========================================== - -We provide a shorthand for generating a subcommand CLI from a dictionary. This -is a thin wrapper around :func:`tyro.cli()`'s more verbose, type-based API. If -more generality is needed, the internal working are explained in the docs for -:func:`tyro.extras.subcommand_cli_from_dict()`. - - -.. code-block:: python - :linenos: - - - import tyro - - - def checkout(branch: str) -> None: - """Check out a branch.""" - print(f"{branch=}") - - - def commit(message: str, all: bool = False) -> None: - """Make a commit.""" - print(f"{message=} {all=}") - - - if __name__ == "__main__": - tyro.extras.subcommand_cli_from_dict( - { - "checkout": checkout, - "commit": commit, - } - ) - ------------- - -.. raw:: html - - python 02_nesting/05_subcommands_func.py --help - -.. program-output:: python ../../examples/02_nesting/05_subcommands_func.py --help - ------------- - -.. raw:: html - - python 02_nesting/05_subcommands_func.py commit --help - -.. program-output:: python ../../examples/02_nesting/05_subcommands_func.py commit --help - ------------- - -.. raw:: html - - python 02_nesting/05_subcommands_func.py commit --message hello --all - -.. program-output:: python ../../examples/02_nesting/05_subcommands_func.py commit --message hello --all - ------------- - -.. raw:: html - - python 02_nesting/05_subcommands_func.py checkout --help - -.. program-output:: python ../../examples/02_nesting/05_subcommands_func.py checkout --help - ------------- - -.. raw:: html - - python 02_nesting/05_subcommands_func.py checkout --branch main - -.. program-output:: python ../../examples/02_nesting/05_subcommands_func.py checkout --branch main diff --git a/docs/source/examples/03_config_systems/01_base_configs.rst b/docs/source/examples/03_config_systems/01_base_configs.rst deleted file mode 100644 index 0ac47f25e..000000000 --- a/docs/source/examples/03_config_systems/01_base_configs.rst +++ /dev/null @@ -1,132 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Base Configurations -========================================== - -We can integrate `tyro` into common configuration patterns: here, we select -one of multiple possible base configurations, create a subcommand for each one, and then -use the CLI to either override (existing) or fill in (missing) values. - -The helper function used here, :func:`tyro.extras.overridable_config_cli()`, is a -lightweight wrapper over :func:`tyro.cli()`. - - -.. code-block:: python - :linenos: - - - from dataclasses import dataclass - from typing import Callable, Literal - - from torch import nn - - import tyro - - - @dataclass(frozen=True) - class AdamOptimizer: - learning_rate: float = 1e-3 - betas: tuple[float, float] = (0.9, 0.999) - - - @dataclass(frozen=True) - class ExperimentConfig: - # Dataset to run experiment on. - dataset: Literal["mnist", "imagenet-50"] - - # Optimizer parameters. - optimizer: AdamOptimizer - - # Model size. - num_layers: int - units: int - - # Batch size. - batch_size: int - - # Total number of training steps. - train_steps: int - - # Random seed. This is helpful for making sure that our experiments are all - # reproducible! - seed: int - - # Activation to use. Not specifiable via the commandline. - activation: Callable[[], nn.Module] - - - # Note that we could also define this library using separate YAML files (similar to - # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type - # checking + IDE support. - default_configs = { - "small": ( - "Small experiment.", - ExperimentConfig( - dataset="mnist", - optimizer=AdamOptimizer(), - batch_size=2048, - num_layers=4, - units=64, - train_steps=30_000, - seed=0, - activation=nn.ReLU, - ), - ), - "big": ( - "Big experiment.", - ExperimentConfig( - dataset="imagenet-50", - optimizer=AdamOptimizer(), - batch_size=32, - num_layers=8, - units=256, - train_steps=100_000, - seed=0, - activation=nn.GELU, - ), - ), - } - if __name__ == "__main__": - config = tyro.extras.overridable_config_cli(default_configs) - print(config) - ------------- - -.. raw:: html - - python 03_config_systems/01_base_configs.py --help - -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py --help - ------------- - -.. raw:: html - - python 03_config_systems/01_base_configs.py small --help - -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py small --help - ------------- - -.. raw:: html - - python 03_config_systems/01_base_configs.py small --seed 94720 - -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py small --seed 94720 - ------------- - -.. raw:: html - - python 03_config_systems/01_base_configs.py big --help - -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py big --help - ------------- - -.. raw:: html - - python 03_config_systems/01_base_configs.py big --seed 94720 - -.. program-output:: python ../../examples/03_config_systems/01_base_configs.py big --seed 94720 diff --git a/docs/source/examples/03_config_systems/02_overriding_yaml.rst b/docs/source/examples/03_config_systems/02_overriding_yaml.rst deleted file mode 100644 index e926429cc..000000000 --- a/docs/source/examples/03_config_systems/02_overriding_yaml.rst +++ /dev/null @@ -1,61 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Overriding YAML Configs -========================================== - -If you have a library of existing YAML files that you want to use, `tyro` can -be used to override values in them. We generally recommend dataclass configs -for new projects. - - -.. code-block:: python - :linenos: - - - import yaml - - import tyro - - # YAML configuration. Note that this could also be loaded from a file! Environment - # variables are an easy way to select between different YAML files. - default_yaml = r""" - exp_name: test - optimizer: - learning_rate: 0.0001 - type: adam - training: - batch_size: 32 - num_steps: 10000 - checkpoint_steps: - - 500 - - 1000 - - 1500 - """.strip() - - if __name__ == "__main__": - # Convert our YAML config into a nested dictionary. - default_config = yaml.safe_load(default_yaml) - - # Override fields in the dictionary. - overridden_config = tyro.cli(dict, default=default_config) - - # Print the overridden config. - overridden_yaml = yaml.safe_dump(overridden_config) - print(overridden_yaml) - ------------- - -.. raw:: html - - python 03_config_systems/02_overriding_yaml.py --help - -.. program-output:: python ../../examples/03_config_systems/02_overriding_yaml.py --help - ------------- - -.. raw:: html - - python 03_config_systems/02_overriding_yaml.py --training.checkpoint-steps 300 1000 9000 - -.. program-output:: python ../../examples/03_config_systems/02_overriding_yaml.py --training.checkpoint-steps 300 1000 9000 diff --git a/docs/source/examples/04_additional/01_positional_args.rst b/docs/source/examples/04_additional/01_positional_args.rst deleted file mode 100644 index b547f0218..000000000 --- a/docs/source/examples/04_additional/01_positional_args.rst +++ /dev/null @@ -1,83 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Positional Arguments -========================================== - -Positional-only arguments in functions are converted to positional CLI arguments. - -For more general positional arguments, see :class:`tyro.conf.Positional`. - - -.. code-block:: python - :linenos: - - - from __future__ import annotations - - import dataclasses - import enum - import pathlib - from typing import Tuple - - import tyro - - - def main( - source: pathlib.Path, - dest: pathlib.Path, - /, # Mark the end of positional arguments. - optimizer: OptimizerConfig, - force: bool = False, - verbose: bool = False, - background_rgb: Tuple[float, float, float] = (1.0, 0.0, 0.0), - ) -> None: - """Command-line interface defined using a function signature. Note that this - docstring is parsed to generate helptext. - - Args: - source: Source path. - dest: Destination path. - optimizer: Configuration for our optimizer object. - force: Do not prompt before overwriting. - verbose: Explain what is being done. - background_rgb: Background color. Red by default. - """ - print(f"{source=}\n{dest=}\n{optimizer=}\n{force=}\n{verbose=}\n{background_rgb=}") - - - class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - - @dataclasses.dataclass(frozen=True) - class OptimizerConfig: - algorithm: OptimizerType = OptimizerType.ADAM - """Gradient-based optimizer to use.""" - - learning_rate: float = 3e-4 - """Learning rate to use.""" - - weight_decay: float = 1e-2 - """Coefficient for L2 regularization.""" - - - if __name__ == "__main__": - tyro.cli(main) - ------------- - -.. raw:: html - - python 04_additional/01_positional_args.py --help - -.. program-output:: python ../../examples/04_additional/01_positional_args.py --help - ------------- - -.. raw:: html - - python 04_additional/01_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 - -.. program-output:: python ../../examples/04_additional/01_positional_args.py ./a ./b --optimizer.learning-rate 1e-5 diff --git a/docs/source/examples/04_additional/02_dictionaries.rst b/docs/source/examples/04_additional/02_dictionaries.rst deleted file mode 100644 index 7d28077e6..000000000 --- a/docs/source/examples/04_additional/02_dictionaries.rst +++ /dev/null @@ -1,82 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Dictionaries and TypedDict -========================================== - -Dictionary inputs can be specified using either a standard `Dict[K, V]` -annotation, or a :code:`TypedDict` subclass. - -For configuring :code:`TypedDict`, we also support :code:`total={True/False}`, -:code:`typing.Required`, and :code:`typing.NotRequired`. See the `Python docs `_ for all :code:`TypedDict` features. - - -.. code-block:: python - :linenos: - - - from typing import TypedDict - - from typing_extensions import NotRequired - - import tyro - - - class DictionarySchemaA( - TypedDict, - # Setting `total=False` specifies that not all keys need to exist. - total=False, - ): - learning_rate: float - betas: tuple[float, float] - - - class DictionarySchemaB(TypedDict): - learning_rate: NotRequired[float] - """NotRequired[] specifies that a particular key doesn't need to exist.""" - betas: tuple[float, float] - - - def main( - typed_dict_a: DictionarySchemaA, - typed_dict_b: DictionarySchemaB, - standard_dict: dict[str, float] = { - "learning_rate": 3e-4, - "beta1": 0.9, - "beta2": 0.999, - }, - ) -> None: - assert isinstance(typed_dict_a, dict) - assert isinstance(typed_dict_b, dict) - assert isinstance(standard_dict, dict) - print("Typed dict A:", typed_dict_a) - print("Typed dict B:", typed_dict_b) - print("Standard dict:", standard_dict) - - - if __name__ == "__main__": - tyro.cli(main) - ------------- - -.. raw:: html - - python 04_additional/02_dictionaries.py --help - -.. program-output:: python ../../examples/04_additional/02_dictionaries.py --help - ------------- - -.. raw:: html - - python 04_additional/02_dictionaries.py --typed-dict-a.learning-rate 3e-4 --typed-dict-b.betas 0.9 0.999 - -.. program-output:: python ../../examples/04_additional/02_dictionaries.py --typed-dict-a.learning-rate 3e-4 --typed-dict-b.betas 0.9 0.999 - ------------- - -.. raw:: html - - python 04_additional/02_dictionaries.py --typed-dict-b.betas 0.9 0.999 - -.. program-output:: python ../../examples/04_additional/02_dictionaries.py --typed-dict-b.betas 0.9 0.999 diff --git a/docs/source/examples/04_additional/03_tuples.rst b/docs/source/examples/04_additional/03_tuples.rst deleted file mode 100644 index cb979eda9..000000000 --- a/docs/source/examples/04_additional/03_tuples.rst +++ /dev/null @@ -1,65 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Tuples -========================================== - -Example using :func:`tyro.cli()` to instantiate tuple types. :code:`tuple`, -:code:`typing.Tuple`, and :code:`NamedTuple` are all supported. - - -.. code-block:: python - :linenos: - - - from typing import NamedTuple - - import tyro - - - # Named tuples are interpreted as nested structures. - class Color(NamedTuple): - r: int - g: int - b: int - - - class TupleType(NamedTuple): - """Description. - This should show up in the helptext!""" - - # Tuple types can contain raw values. - color: tuple[int, int, int] = (255, 0, 0) - - # Tuple types can contain nested structures. - two_colors: tuple[Color, Color] = (Color(255, 0, 0), Color(0, 255, 0)) - - - if __name__ == "__main__": - x = tyro.cli(TupleType) - assert isinstance(x, tuple) - print(x) - ------------- - -.. raw:: html - - python 04_additional/03_tuples.py --help - -.. program-output:: python ../../examples/04_additional/03_tuples.py --help - ------------- - -.. raw:: html - - python 04_additional/03_tuples.py --color 127 127 127 - -.. program-output:: python ../../examples/04_additional/03_tuples.py --color 127 127 127 - ------------- - -.. raw:: html - - python 04_additional/03_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0 - -.. program-output:: python ../../examples/04_additional/03_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0 diff --git a/docs/source/examples/04_additional/04_classes.rst b/docs/source/examples/04_additional/04_classes.rst deleted file mode 100644 index bf6f64526..000000000 --- a/docs/source/examples/04_additional/04_classes.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Instantiating Classes -========================================== - -In addition to functions and dataclasses, we can also generate CLIs from the -constructors of standard Python classes. - - -.. code-block:: python - :linenos: - - - import tyro - - - class Args: - def __init__( - self, - field1: str, - field2: int, - flag: bool = False, - ): - """Arguments. - - Args: - field1: A string field. - field2: A numeric field. - flag: A boolean flag. - """ - self.data = [field1, field2, flag] - - - if __name__ == "__main__": - args = tyro.cli(Args) - print(args.data) - ------------- - -.. raw:: html - - python 04_additional/04_classes.py --help - -.. program-output:: python ../../examples/04_additional/04_classes.py --help - ------------- - -.. raw:: html - - python 04_additional/04_classes.py --field1 hello --field2 7 - -.. program-output:: python ../../examples/04_additional/04_classes.py --field1 hello --field2 7 diff --git a/docs/source/examples/04_additional/05_generics.rst b/docs/source/examples/04_additional/05_generics.rst deleted file mode 100644 index 6578dcd93..000000000 --- a/docs/source/examples/04_additional/05_generics.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Generic Types -========================================== - -Example of parsing for generic dataclasses. - - -.. code-block:: python - :linenos: - - - import dataclasses - from typing import Generic, TypeVar - - import tyro - - ScalarType = TypeVar("ScalarType", int, float) - ShapeType = TypeVar("ShapeType") - - - @dataclasses.dataclass(frozen=True) - class Point3(Generic[ScalarType]): - x: ScalarType - y: ScalarType - z: ScalarType - frame_id: str - - - @dataclasses.dataclass(frozen=True) - class Triangle: - a: Point3[float] - b: Point3[float] - c: Point3[float] - - - @dataclasses.dataclass(frozen=True) - class Args(Generic[ShapeType]): - shape: ShapeType - - - if __name__ == "__main__": - args = tyro.cli(Args[Triangle]) - print(args) - ------------- - -.. raw:: html - - python 04_additional/05_generics.py --help - -.. program-output:: python ../../examples/04_additional/05_generics.py --help diff --git a/docs/source/examples/04_additional/06_generics_py312.rst b/docs/source/examples/04_additional/06_generics_py312.rst deleted file mode 100644 index e075e7586..000000000 --- a/docs/source/examples/04_additional/06_generics_py312.rst +++ /dev/null @@ -1,54 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Generic Types (Python 3.12+) -========================================== - -Example of parsing for generic dataclasses using syntax introduced in Python -3.12 (`PEP 695 `_). - -.. warning:: - - If used in conjunction with :code:`from __future__ import annotations`, the updated type parameter syntax requires Python 3.12.4 or newer. For technical details, see `this CPython PR `_. - - -.. code-block:: python - :linenos: - - - import dataclasses - - import tyro - - - @dataclasses.dataclass(frozen=True) - class Point3[ScalarType: (int, float)]: - x: ScalarType - y: ScalarType - z: ScalarType - frame_id: str - - - @dataclasses.dataclass(frozen=True) - class Triangle: - a: Point3[float] - b: Point3[float] - c: Point3[float] - - - @dataclasses.dataclass(frozen=True) - class Args[ShapeType]: - shape: ShapeType - - - if __name__ == "__main__": - args = tyro.cli(Args[Triangle]) - print(args) - ------------- - -.. raw:: html - - python 04_additional/06_generics_py312.py --help - -.. program-output:: python ../../examples/04_additional/06_generics_py312.py --help diff --git a/docs/source/examples/04_additional/07_conf.rst b/docs/source/examples/04_additional/07_conf.rst deleted file mode 100644 index 2faacdf48..000000000 --- a/docs/source/examples/04_additional/07_conf.rst +++ /dev/null @@ -1,63 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Configuration via typing.Annotated[] -========================================== - -The :mod:`tyro.conf` module contains utilities that can be used to configure -command-line interfaces beyond what is expressible via static type annotations. - -Features here are supported, but generally unnecessary and should be used sparingly. - - -.. code-block:: python - :linenos: - - - import dataclasses - - from typing_extensions import Annotated - - import tyro - - - @dataclasses.dataclass - class Args: - # A numeric field parsed as a positional argument. - positional: tyro.conf.Positional[int] - - # A boolean field with flag conversion turned off. - boolean: tyro.conf.FlagConversionOff[bool] = False - - # A numeric field that can't be changed via the CLI. - fixed: tyro.conf.Fixed[int] = 5 - - # A field with manually overridden properties. - manual: Annotated[ - str, - tyro.conf.arg( - name="renamed", - metavar="STRING", - help="A field with manually overridden properties!", - ), - ] = "Hello" - - - if __name__ == "__main__": - print(tyro.cli(Args)) - ------------- - -.. raw:: html - - python 04_additional/07_conf.py --help - -.. program-output:: python ../../examples/04_additional/07_conf.py --help - ------------- - -.. raw:: html - - python 04_additional/07_conf.py 5 --boolean True - -.. program-output:: python ../../examples/04_additional/07_conf.py 5 --boolean True diff --git a/docs/source/examples/04_additional/08_pydantic.rst b/docs/source/examples/04_additional/08_pydantic.rst deleted file mode 100644 index 9f529d173..000000000 --- a/docs/source/examples/04_additional/08_pydantic.rst +++ /dev/null @@ -1,54 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Pydantic Integration -========================================== - -In addition to standard dataclasses, :func:`tyro.cli()` also supports -`Pydantic `_ models. - - -.. code-block:: python - :linenos: - - - from pydantic import BaseModel, Field - - import tyro - - - class Args(BaseModel): - """Description. - This should show up in the helptext!""" - - field1: str - field2: int = Field(3, description="An integer field.") - - - if __name__ == "__main__": - args = tyro.cli(Args) - print(args) - ------------- - -.. raw:: html - - python 04_additional/08_pydantic.py --help - -.. program-output:: python ../../examples/04_additional/08_pydantic.py --help - ------------- - -.. raw:: html - - python 04_additional/08_pydantic.py --field1 hello - -.. program-output:: python ../../examples/04_additional/08_pydantic.py --field1 hello - ------------- - -.. raw:: html - - python 04_additional/08_pydantic.py --field1 hello --field2 5 - -.. program-output:: python ../../examples/04_additional/08_pydantic.py --field1 hello --field2 5 diff --git a/docs/source/examples/04_additional/09_attrs.rst b/docs/source/examples/04_additional/09_attrs.rst deleted file mode 100644 index e52a189f5..000000000 --- a/docs/source/examples/04_additional/09_attrs.rst +++ /dev/null @@ -1,58 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Attrs Integration -========================================== - -In addition to standard dataclasses, :func:`tyro.cli()` also supports -`attrs `_ classes. - - -.. code-block:: python - :linenos: - - - import attr - - import tyro - - - @attr.s - class Args: - """Description. - This should show up in the helptext!""" - - field1: str = attr.ib() - """A string field.""" - - field2: int = attr.ib(factory=lambda: 5) - """A required integer field.""" - - - if __name__ == "__main__": - args = tyro.cli(Args) - print(args) - ------------- - -.. raw:: html - - python 04_additional/09_attrs.py --help - -.. program-output:: python ../../examples/04_additional/09_attrs.py --help - ------------- - -.. raw:: html - - python 04_additional/09_attrs.py --field1 hello - -.. program-output:: python ../../examples/04_additional/09_attrs.py --field1 hello - ------------- - -.. raw:: html - - python 04_additional/09_attrs.py --field1 hello --field2 5 - -.. program-output:: python ../../examples/04_additional/09_attrs.py --field1 hello --field2 5 diff --git a/docs/source/examples/04_additional/10_flax.rst b/docs/source/examples/04_additional/10_flax.rst deleted file mode 100644 index aec226d00..000000000 --- a/docs/source/examples/04_additional/10_flax.rst +++ /dev/null @@ -1,74 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -JAX/Flax Integration -========================================== - -If you use `flax.linen `_, modules can be instantiated -directly from :func:`tyro.cli()`. - - -.. code-block:: python - :linenos: - - - from flax import linen as nn - from jax import numpy as jnp - - import tyro - - - class Classifier(nn.Module): - layers: int - """Layers in our network.""" - units: int = 32 - """Hidden unit count.""" - output_dim: int = 10 - """Number of classes.""" - - @nn.compact - def __call__(self, x: jnp.ndarray) -> jnp.ndarray: # type: ignore - for i in range(self.layers - 1): - x = nn.Dense( - self.units, - kernel_init=nn.initializers.kaiming_normal(), - )(x) - x = nn.relu(x) - - x = nn.Dense( - self.output_dim, - kernel_init=nn.initializers.xavier_normal(), - )(x) - x = nn.sigmoid(x) - return x - - - def train(model: Classifier, num_iterations: int = 1000) -> None: - """Train a model. - - Args: - model: Model to train. - num_iterations: Number of training iterations. - """ - print(f"{model=}") - print(f"{num_iterations=}") - - - if __name__ == "__main__": - tyro.cli(train) - ------------- - -.. raw:: html - - python 04_additional/10_flax.py --help - -.. program-output:: python ../../examples/04_additional/10_flax.py --help - ------------- - -.. raw:: html - - python 04_additional/10_flax.py --model.layers 4 - -.. program-output:: python ../../examples/04_additional/10_flax.py --model.layers 4 diff --git a/docs/source/examples/04_additional/11_aliases.rst b/docs/source/examples/04_additional/11_aliases.rst deleted file mode 100644 index ad728274e..000000000 --- a/docs/source/examples/04_additional/11_aliases.rst +++ /dev/null @@ -1,96 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Argument Aliases -========================================== - -:func:`tyro.conf.arg()` can be used to attach aliases to arguments. - - -.. code-block:: python - :linenos: - - - from typing_extensions import Annotated - - import tyro - - - def checkout( - branch: Annotated[str, tyro.conf.arg(aliases=["-b"])], - ) -> None: - """Check out a branch.""" - print(f"{branch=}") - - - def commit( - message: Annotated[str, tyro.conf.arg(aliases=["-m"])], - all: Annotated[bool, tyro.conf.arg(aliases=["-a"])] = False, - ) -> None: - """Make a commit.""" - print(f"{message=} {all=}") - - - if __name__ == "__main__": - tyro.extras.subcommand_cli_from_dict( - { - "checkout": checkout, - "commit": commit, - } - ) - ------------- - -.. raw:: html - - python 04_additional/11_aliases.py --help - -.. program-output:: python ../../examples/04_additional/11_aliases.py --help - ------------- - -.. raw:: html - - python 04_additional/11_aliases.py commit --help - -.. program-output:: python ../../examples/04_additional/11_aliases.py commit --help - ------------- - -.. raw:: html - - python 04_additional/11_aliases.py commit --message hello --all - -.. program-output:: python ../../examples/04_additional/11_aliases.py commit --message hello --all - ------------- - -.. raw:: html - - python 04_additional/11_aliases.py commit -m hello -a - -.. program-output:: python ../../examples/04_additional/11_aliases.py commit -m hello -a - ------------- - -.. raw:: html - - python 04_additional/11_aliases.py checkout --help - -.. program-output:: python ../../examples/04_additional/11_aliases.py checkout --help - ------------- - -.. raw:: html - - python 04_additional/11_aliases.py checkout --branch main - -.. program-output:: python ../../examples/04_additional/11_aliases.py checkout --branch main - ------------- - -.. raw:: html - - python 04_additional/11_aliases.py checkout -b main - -.. program-output:: python ../../examples/04_additional/11_aliases.py checkout -b main diff --git a/docs/source/examples/04_additional/12_type_statement.rst b/docs/source/examples/04_additional/12_type_statement.rst deleted file mode 100644 index 63975f1e3..000000000 --- a/docs/source/examples/04_additional/12_type_statement.rst +++ /dev/null @@ -1,50 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Type Aliases (Python 3.12+) -========================================== - -In Python 3.12, the :code:`type` statement is introduced to create type aliases. - - -.. code-block:: python - :linenos: - - - import dataclasses - - import tyro - - # Lazily-evaluated type alias. - type Field1Type = Inner - - - @dataclasses.dataclass - class Inner: - a: int - b: str - - - @dataclasses.dataclass - class Args: - """Description. - This should show up in the helptext!""" - - field1: Field1Type - """A field.""" - - field2: int = 3 - """A numeric field, with a default value.""" - - - if __name__ == "__main__": - args = tyro.cli(Args) - print(args) - ------------- - -.. raw:: html - - python 04_additional/12_type_statement.py --help - -.. program-output:: python ../../examples/04_additional/12_type_statement.py --help diff --git a/docs/source/examples/04_additional/13_counters.rst b/docs/source/examples/04_additional/13_counters.rst deleted file mode 100644 index 4068f1375..000000000 --- a/docs/source/examples/04_additional/13_counters.rst +++ /dev/null @@ -1,67 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Counters -========================================== - -Repeatable 'counter' arguments can be specified via :data:`tyro.conf.UseCounterAction`. - - -.. code-block:: python - :linenos: - - - from typing_extensions import Annotated - - import tyro - from tyro.conf import UseCounterAction - - - def main( - verbosity: UseCounterAction[int], - aliased_verbosity: Annotated[UseCounterAction[int], tyro.conf.arg(aliases=["-v"])], - ) -> None: - """Example showing how to use counter actions. - - Args: - verbosity: Verbosity level. - aliased_verbosity: Same as above, but can also be specified with -v, -vv, -vvv, etc. - """ - print("Verbosity level:", verbosity) - print("Verbosity level (aliased):", aliased_verbosity) - - - if __name__ == "__main__": - tyro.cli(main) - ------------- - -.. raw:: html - - python 04_additional/13_counters.py --help - -.. program-output:: python ../../examples/04_additional/13_counters.py --help - ------------- - -.. raw:: html - - python 04_additional/13_counters.py --verbosity - -.. program-output:: python ../../examples/04_additional/13_counters.py --verbosity - ------------- - -.. raw:: html - - python 04_additional/13_counters.py --verbosity --verbosity - -.. program-output:: python ../../examples/04_additional/13_counters.py --verbosity --verbosity - ------------- - -.. raw:: html - - python 04_additional/13_counters.py -vvv - -.. program-output:: python ../../examples/04_additional/13_counters.py -vvv diff --git a/docs/source/examples/04_additional/14_suppress_console_outputs.rst b/docs/source/examples/04_additional/14_suppress_console_outputs.rst deleted file mode 100644 index e6056ea69..000000000 --- a/docs/source/examples/04_additional/14_suppress_console_outputs.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Cleaner Console Outputs for Scripts with Multiple Workers -========================================== - -The :code:`console_outputs=` argument can be set to :code:`False` to suppress helptext and -error message printing. - -This is useful in PyTorch for distributed training scripts, where you only want -to print the helptext from the main process: - - -.. code-block:: python - - # HuggingFace Accelerate. - args = tyro.cli(Args, console_outputs=accelerator.is_main_process) - - # PyTorch DDP. - args = tyro.cli(Args, console_outputs=(rank == 0)) - - # PyTorch Lightning. - args = tyro.cli(Args, console_outputs=trainer.is_global_zero) - - -.. code-block:: python - :linenos: - - - import dataclasses - - import tyro - - - @dataclasses.dataclass - class Args: - """Description. - This should show up in the helptext!""" - - field1: int - """A field.""" - - field2: int = 3 - """A numeric field, with a default value.""" - - - if __name__ == "__main__": - args = tyro.cli(Args, console_outputs=False) - print(args) - ------------- - -.. raw:: html - - python 04_additional/14_suppress_console_outputs.py --help - -.. program-output:: python ../../examples/04_additional/14_suppress_console_outputs.py --help diff --git a/docs/source/examples/04_additional/15_decorator_subcommands.rst b/docs/source/examples/04_additional/15_decorator_subcommands.rst deleted file mode 100644 index 1def51afe..000000000 --- a/docs/source/examples/04_additional/15_decorator_subcommands.rst +++ /dev/null @@ -1,84 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Decorator-based Subcommands -========================================== - -:func:`tyro.extras.SubcommandApp()` provides a decorator-based API for -subcommands, which is inspired by `click `_. - - -.. code-block:: python - :linenos: - - - from tyro.extras import SubcommandApp - - app = SubcommandApp() - - - @app.command - def greet(name: str, loud: bool = False) -> None: - """Greet someone.""" - greeting = f"Hello, {name}!" - if loud: - greeting = greeting.upper() - print(greeting) - - - @app.command(name="addition") - def add(a: int, b: int) -> None: - """Add two numbers.""" - print(f"{a} + {b} = {a + b}") - - - if __name__ == "__main__": - app.cli() - ------------- - -.. raw:: html - - python 04_additional/15_decorator_subcommands.py --help - -.. program-output:: python ../../examples/04_additional/15_decorator_subcommands.py --help - ------------- - -.. raw:: html - - python 04_additional/15_decorator_subcommands.py greet --help - -.. program-output:: python ../../examples/04_additional/15_decorator_subcommands.py greet --help - ------------- - -.. raw:: html - - python 04_additional/15_decorator_subcommands.py greet --name Alice - -.. program-output:: python ../../examples/04_additional/15_decorator_subcommands.py greet --name Alice - ------------- - -.. raw:: html - - python 04_additional/15_decorator_subcommands.py greet --name Bob --loud - -.. program-output:: python ../../examples/04_additional/15_decorator_subcommands.py greet --name Bob --loud - ------------- - -.. raw:: html - - python 04_additional/15_decorator_subcommands.py addition --help - -.. program-output:: python ../../examples/04_additional/15_decorator_subcommands.py addition --help - ------------- - -.. raw:: html - - python 04_additional/15_decorator_subcommands.py addition --a 5 --b 3 - -.. program-output:: python ../../examples/04_additional/15_decorator_subcommands.py addition --a 5 --b 3 diff --git a/docs/source/examples/05_custom_constructors/01_primitive_annotation.rst b/docs/source/examples/05_custom_constructors/01_primitive_annotation.rst deleted file mode 100644 index e1e196c79..000000000 --- a/docs/source/examples/05_custom_constructors/01_primitive_annotation.rst +++ /dev/null @@ -1,71 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Custom Primitive -========================================== - -For additional flexibility, :mod:`tyro.constructors` exposes tyro's API for -defining behavior for different types. There are two categories of types: -primitive types can be instantiated from a single commandline argument, while -struct types are broken down into multiple. - -In this example, we attach a custom constructor via a runtime annotation. - - -.. code-block:: python - :linenos: - - - import json - - from typing_extensions import Annotated - - import tyro - - # A dictionary type, but `tyro` will expect a JSON string from the CLI. - JsonDict = Annotated[ - dict, - tyro.constructors.PrimitiveConstructorSpec( - nargs=1, - metavar="JSON", - instance_from_str=lambda args: json.loads(args[0]), - is_instance=lambda instance: isinstance(instance, dict), - str_from_instance=lambda instance: [json.dumps(instance)], - ), - ] - - - def main( - dict1: JsonDict, - dict2: JsonDict = {"default": None}, - ) -> None: - print(f"{dict1=}") - print(f"{dict2=}") - - - if __name__ == "__main__": - tyro.cli(main) - ------------- - -.. raw:: html - - python 05_custom_constructors/01_primitive_annotation.py --help - -.. program-output:: python ../../examples/05_custom_constructors/01_primitive_annotation.py --help - ------------- - -.. raw:: html - - python 05_custom_constructors/01_primitive_annotation.py --dict1 '{"hello": "world"}' - -.. program-output:: python ../../examples/05_custom_constructors/01_primitive_annotation.py --dict1 '{"hello": "world"}' - ------------- - -.. raw:: html - - python 05_custom_constructors/01_primitive_annotation.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}' - -.. program-output:: python ../../examples/05_custom_constructors/01_primitive_annotation.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}' diff --git a/docs/source/examples/05_custom_constructors/02_primitive_registry.rst b/docs/source/examples/05_custom_constructors/02_primitive_registry.rst deleted file mode 100644 index a6bd8bca8..000000000 --- a/docs/source/examples/05_custom_constructors/02_primitive_registry.rst +++ /dev/null @@ -1,81 +0,0 @@ -.. Comment: this file is automatically generated by `update_example_docs.py`. - It should not be modified manually. - -Custom Primitive (Registry) -========================================== - -For additional flexibility, :mod:`tyro.constructors` exposes tyro's API for -defining behavior for different types. There are two categories of types: -primitive types can be instantiated from a single commandline argument, while -struct types are broken down into multiple. - - -In this example, we attach a custom constructor by defining a rule that applies -to all types that match ``dict[str, Any]``. - - -.. code-block:: python - :linenos: - - - import json - from typing import Any - - import tyro - - custom_registry = tyro.constructors.ConstructorRegistry() - - - @custom_registry.primitive_rule - def _( - type_info: tyro.constructors.PrimitiveTypeInfo, - ) -> tyro.constructors.PrimitiveConstructorSpec | None: - # We return `None` if the rule does not apply. - if type_info.type != dict[str, Any]: - return None - - # If the rule applies, we return the constructor spec. - return tyro.constructors.PrimitiveConstructorSpec( - nargs=1, - metavar="JSON", - instance_from_str=lambda args: json.loads(args[0]), - is_instance=lambda instance: isinstance(instance, dict), - str_from_instance=lambda instance: [json.dumps(instance)], - ) - - - def main( - dict1: dict[str, Any], - dict2: dict[str, Any] = {"default": None}, - ) -> None: - print(f"{dict1=}") - print(f"{dict2=}") - - - if __name__ == "__main__": - with custom_registry: - tyro.cli(main) - ------------- - -.. raw:: html - - python 05_custom_constructors/02_primitive_registry.py --help - -.. program-output:: python ../../examples/05_custom_constructors/02_primitive_registry.py --help - ------------- - -.. raw:: html - - python 05_custom_constructors/02_primitive_registry.py --dict1 '{"hello": "world"}' - -.. program-output:: python ../../examples/05_custom_constructors/02_primitive_registry.py --dict1 '{"hello": "world"}' - ------------- - -.. raw:: html - - python 05_custom_constructors/02_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}' - -.. program-output:: python ../../examples/05_custom_constructors/02_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}' diff --git a/docs/source/examples/basics.rst b/docs/source/examples/basics.rst new file mode 100644 index 000000000..ffa143219 --- /dev/null +++ b/docs/source/examples/basics.rst @@ -0,0 +1,990 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +.. _example-category-basics: + +Basics +====== + +In these examples, we show basic examples of using :func:`tyro.cli`: functions, +dataclasses, supported annotations, and configuration. + + +.. _example-01_functions: + +Functions +--------- + +In the simplest case, :func:`tyro.cli()` can be used to run a function with +arguments populated from the CLI. + + +.. code-block:: python + :linenos: + + # 01_functions.py + import tyro + + def main(field1: str, field2: int = 3) -> None: + """Function, whose arguments will be populated from a CLI interface. + + Args: + field1: A string field. + field2: A numeric field, with a default value. + """ + print(field1, field2) + + if __name__ == "__main__": + tyro.cli(main) + + +We can use ``--help`` to show the help message, or ``--field1`` and +``--field2`` to set the arguments: + +.. raw:: html + + 

+    $ python ./01_functions.py --help
+    usage: 01_functions.py [-h] --field1 STR [--field2 INT]
+    
+    Function, whose arguments will be populated from a CLI interface.
+    
+    ╭─ options ───────────────────────────────────────────────────────────────╮
+    │ -h, --help          show this help message and exit                     │
+    │ --field1 STR        A string field. (required)                          │
+    │ --field2 INT        A numeric field, with a default value. (default: 3) │
+    ╰─────────────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./01_functions.py --field1 hello
+    hello 3
+
+ + + +.. raw:: html + + 
+    $ python ./01_functions.py --field1 hello --field2 10
+    hello 10
+
+.. _example-02_dataclasses: + +Dataclasses +----------- + +In addition to functions, :func:`tyro.cli()` can also take dataclasses as input. + + +.. code-block:: python + :linenos: + + # 02_dataclasses.py + from dataclasses import dataclass + from pprint import pprint + + import tyro + + @dataclass + class Args: + """Description. + This should show up in the helptext!""" + + field1: str + """A string field.""" + + field2: int = 3 + """A numeric field, with a default value.""" + + if __name__ == "__main__": + args = tyro.cli(Args) + pprint(args) + + +To show the help message, we can use the ``--help`` flag: + +.. raw:: html + + 
+    $ python ./02_dataclasses.py --help
+    usage: 02_dataclasses.py [-h] --field1 STR [--field2 INT]
+    
+    Description. This should show up in the helptext!
+    
+    ╭─ options ───────────────────────────────────────────────────────────────╮
+    │ -h, --help          show this help message and exit                     │
+    │ --field1 STR        A string field. (required)                          │
+    │ --field2 INT        A numeric field, with a default value. (default: 3) │
+    ╰─────────────────────────────────────────────────────────────────────────╯
+
+ +We can override ``field1`` and ``field2``: + +.. raw:: html + + 
+    $ python ./02_dataclasses.py --field1 hello
+    Args(field1='hello', field2=3)
+
+ + + +.. raw:: html + + 
+    $ python ./02_dataclasses.py --field1 hello --field2 5
+    Args(field1='hello', field2=5)
+
+.. _example-03_multivalue: + +Multi-value Arguments +--------------------- + +Arguments of both fixed and variable lengths can be annotated with standard +Python collection types. For Python 3.7 and 3.8, we can use either ``from +__future__ import annotations`` to support ``list[T]`` and ``tuple[T]``, +or the older :py:class:`typing.List` and :py:data:`typing.Tuple`. + + +.. code-block:: python + :linenos: + + # 03_multivalue.py + import pathlib + from dataclasses import dataclass + from pprint import pprint + + import tyro + + @dataclass + class Config: + # Example of a variable-length tuple. `list[T]`, `set[T]`, + # `dict[K, V]`, etc are supported as well. + source_paths: tuple[pathlib.Path, ...] + """This can be multiple!""" + + # Fixed-length tuples are also okay. + dimensions: tuple[int, int] = (32, 32) + """Height and width.""" + + if __name__ == "__main__": + config = tyro.cli(Config) + pprint(config) + + +To print help: + +.. raw:: html + + 
+    $ python ./03_multivalue.py --help
+    usage: 03_multivalue.py [-h] --source-paths [PATH
+                            [PATH ...]] [--dimensions INT INT]
+    
+    ╭─ options ──────────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit    │
+    │ --source-paths [PATH [PATH ...]]                           │
+    │                         This can be multiple! (required)   │
+    │ --dimensions INT INT    Height and width. (default: 32 32) │
+    ╰────────────────────────────────────────────────────────────╯
+
+ +We can override arguments: + +.. raw:: html + + 
+    $ python ./03_multivalue.py --source-paths ./data --dimensions 16 16
+    Config(source_paths=(PosixPath('data'),), dimensions=(16, 16))
+
+ + + +.. raw:: html + + 
+    $ python ./03_multivalue.py --source-paths ./data1 ./data2
+    Config(source_paths=(PosixPath('data1'), PosixPath('data2')),
+           dimensions=(32, 32))
+
+.. _example-04_classes: + +Instantiating Classes +--------------------- + +In addition to functions and dataclasses, we can also generate CLIs from the +constructors of standard Python classes. + + +.. code-block:: python + :linenos: + + # 04_classes.py + import tyro + + class Args: + def __init__( + self, + field1: str, + field2: int, + flag: bool = False, + ): + """Arguments. + + Args: + field1: A string field. + field2: A numeric field. + flag: A boolean flag. + """ + self.data = [field1, field2, flag] + + if __name__ == "__main__": + args = tyro.cli(Args) + print(args.data) + + + + +.. raw:: html + + 
+    $ python ./04_classes.py --help
+    usage: 04_classes.py [-h] --field1 STR --field2 INT [--flag | --no-flag]
+    
+    Arguments.
+    
+    ╭─ options ───────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit │
+    │ --field1 STR            A string field. (required)      │
+    │ --field2 INT            A numeric field. (required)     │
+    │ --flag, --no-flag       A boolean flag. (default: None) │
+    ╰─────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./04_classes.py --field1 hello --field2 7
+    ['hello', 7, False]
+
+.. _example-04_flags: + +Booleans and Flags +------------------ + +Booleans can either be expected to be explicitly passed in, or, if given a default +value, automatically converted to flags. + +To turn off conversion, see :class:`tyro.conf.FlagConversionOff`. + + +.. code-block:: python + :linenos: + + # 04_flags.py + from dataclasses import dataclass + from pprint import pprint + + import tyro + + @dataclass + class Args: + # Boolean. This expects an explicit "True" or "False". + boolean: bool + + # Optional boolean. Same as above, but can be omitted. + optional_boolean: bool | None = None + + # Pass --flag-a in to set this value to True. + flag_a: bool = False + + # Pass --no-flag-b in to set this value to False. + flag_b: bool = True + + if __name__ == "__main__": + args = tyro.cli(Args) + pprint(args) + + + + +.. raw:: html + + 
+    $ python ./04_flags.py --help
+    usage: 04_flags.py [-h] [OPTIONS]
+    
+    ╭─ options ────────────────────────────────────────────────────────────────╮
+    │ -h, --help                                                               │
+    │     show this help message and exit                                      │
+    │ --boolean {True,False}                                                   │
+    │     Boolean. This expects an explicit "True" or "False". (required)      │
+    │ --optional-boolean {None,True,False}                                     │
+    │     Optional boolean. Same as above, but can be omitted. (default: None) │
+    │ --flag-a, --no-flag-a                                                    │
+    │     Pass --flag-a in to set this value to True. (default: None)          │
+    │ --flag-b, --no-flag-b                                                    │
+    │     Pass --no-flag-b in to set this value to False. (default: None)      │
+    ╰──────────────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./04_flags.py --boolean True
+    Args(boolean=True, optional_boolean=None, flag_a=False, flag_b=True)
+
+ + + +.. raw:: html + + 
+    $ python ./04_flags.py --boolean False --flag-a
+    Args(boolean=False, optional_boolean=None, flag_a=True, flag_b=True)
+
+ + + +.. raw:: html + + 
+    $ python ./04_flags.py --boolean False --no-flag-b
+    Args(boolean=False, optional_boolean=None, flag_a=False, flag_b=False)
+
+.. _example-05_choices: + +Choices +------- + +:code:`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices. + + +.. code-block:: python + :linenos: + + # 05_choices.py + import dataclasses + from pprint import pprint + from typing import Literal + + import tyro + + @dataclasses.dataclass + class Args: + # We can use Literal[] to restrict the set of allowable inputs, for example, over + # a set of strings. + string: Literal["red", "green"] = "red" + + # Integers also work. (as well as booleans, enums, etc) + number: Literal[0, 1, 2] = 0 + + if __name__ == "__main__": + args = tyro.cli(Args) + pprint(args) + + + + +.. raw:: html + + 
+    $ python ./05_choices.py --help
+    usage: 05_choices.py [-h] [--string {red,green}] [--number {0,1,2}]
+    
+    ╭─ options ──────────────────────────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit                    │
+    │ --string {red,green}    We can use Literal[] to restrict the set of        │
+    │                         allowable inputs, for example, over a set of       │
+    │                         strings. (default: red)                            │
+    │ --number {0,1,2}        Integers also work. (as well as booleans, enums,   │
+    │                         etc) (default: 0)                                  │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./05_choices.py --string red
+    Args(string='red', number=0)
+
+ + + +.. raw:: html + + 
+    $ python ./05_choices.py --string blue
+    ╭─ Parsing error ────────────────────────────────────────────────────────╮
+    │ Argument --string: invalid choice: 'blue' (choose from 'red', 'green') │
+    │ ────────────────────────────────────────────────────────────────────── │
+    │ For full helptext, run 05_choices.py --help                            │
+    ╰────────────────────────────────────────────────────────────────────────╯
+
+.. _example-06_enums: + +Enums +----- + +In addition to literals, enums can also be used to provide a fixed set of +choices. + + +.. code-block:: python + :linenos: + + # 06_enums.py + import enum + from dataclasses import dataclass + from pprint import pprint + + import tyro + + class Color(enum.Enum): + RED = enum.auto() + BLUE = enum.auto() + + @dataclass + class Config: + color: Color = Color.RED + """Color argument.""" + + opacity: float = 0.5 + """Opacity argument.""" + + if __name__ == "__main__": + config = tyro.cli(Config) + pprint(config) + + + + +.. raw:: html + + 
+    $ python ./06_enums.py --help
+    usage: 06_enums.py [-h] [--color {RED,BLUE}] [--opacity FLOAT]
+    
+    ╭─ options ────────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit  │
+    │ --color {RED,BLUE}      Color argument. (default: RED)   │
+    │ --opacity FLOAT         Opacity argument. (default: 0.5) │
+    ╰──────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./06_enums.py --color RED
+    Config(color=<Color.RED: 1>, opacity=0.5)
+
+ + + +.. raw:: html + + 
+    $ python ./06_enums.py --color BLUE --opacity 0.75
+    Config(color=<Color.BLUE: 2>, opacity=0.75)
+
+.. _example-07_unions: + +Unions +------ + +:code:`X | Y` or :code:`typing.Union[X, Y]` can be used to expand inputs to +multiple types. + + +.. code-block:: python + :linenos: + + # 07_unions.py + import dataclasses + import enum + from pprint import pprint + from typing import Literal, Optional + + import tyro + + class Color(enum.Enum): + RED = enum.auto() + GREEN = enum.auto() + BLUE = enum.auto() + + @dataclasses.dataclass(frozen=True) + class Args: + # Unions can be used to specify multiple allowable types. + union_over_types: int | str = 0 + string_or_enum: Literal["red", "green"] | Color = "red" + + # Unions also work over more complex nested types. + union_over_tuples: tuple[int, int] | tuple[str] = ("1",) + + # And can be nested in other types. + tuple_of_string_or_enum: tuple[Literal["red", "green"] | Color, ...] = ( + "red", + Color.RED, + ) + + # Optional[T] is equivalent to `T | None`. + integer: Optional[Literal[0, 1, 2, 3]] = None + + if __name__ == "__main__": + args = tyro.cli(Args) + pprint(args) + + + + +.. raw:: html + + 
+    $ python ./07_unions.py --help
+    usage: 07_unions.py [-h] [OPTIONS]
+    
+    ╭─ options ──────────────────────────────────────────────────────────────────╮
+    │ -h, --help                                                                 │
+    │     show this help message and exit                                        │
+    │ --union-over-types INT|STR                                                 │
+    │     Unions can be used to specify multiple allowable types. (default: 0)   │
+    │ --string-or-enum {red,green,RED,GREEN,BLUE}                                │
+    │     Unions can be used to specify multiple allowable types. (default: red) │
+    │ --union-over-tuples {INT INT}|STR                                          │
+    │     Unions also work over more complex nested types. (default: 1)          │
+    │ --tuple-of-string-or-enum [{red,green,RED,GREEN,BLUE}                      │
+    │ [{red,green,RED,GREEN,BLUE} ...]]                                          │
+    │     And can be nested in other types. (default: red RED)                   │
+    │ --integer {None,0,1,2,3}                                                   │
+    │     Optional[T] is equivalent to `T | None`. (default: None)               │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./07_unions.py --union-over-types 3
+    Args(union_over_types=3,
+         string_or_enum='red',
+         union_over_tuples=('1',),
+         tuple_of_string_or_enum=('red', <Color.RED: 1>),
+         integer=None)
+
+ + + +.. raw:: html + + 
+    $ python ./07_unions.py --union-over-types three
+    Args(union_over_types='three',
+         string_or_enum='red',
+         union_over_tuples=('1',),
+         tuple_of_string_or_enum=('red', <Color.RED: 1>),
+         integer=None)
+
+ + + +.. raw:: html + + 
+    $ python ./07_unions.py --integer None
+    Args(union_over_types=0,
+         string_or_enum='red',
+         union_over_tuples=('1',),
+         tuple_of_string_or_enum=('red', <Color.RED: 1>),
+         integer=None)
+
+ + + +.. raw:: html + + 
+    $ python ./07_unions.py --integer 0
+    Args(union_over_types=0,
+         string_or_enum='red',
+         union_over_tuples=('1',),
+         tuple_of_string_or_enum=('red', <Color.RED: 1>),
+         integer=0)
+
+.. _example-08_positional: + +Positional Arguments +-------------------- + +Positional-only arguments in functions are converted to positional CLI arguments. + +For more general positional arguments, see :class:`tyro.conf.Positional`. + + +.. code-block:: python + :linenos: + + # 08_positional.py + from __future__ import annotations + + import pathlib + + import tyro + + def main( + source: pathlib.Path, + dest: pathlib.Path, + /, # Mark the end of positional arguments. + verbose: bool = False, + ) -> None: + """Command-line interface defined using a function signature. Note that this + docstring is parsed to generate helptext. + + Args: + source: Source path. + dest: Destination path. + verbose: Explain what is being done. + """ + print(f"{source=}\n{dest=}\n{verbose=}") + + if __name__ == "__main__": + tyro.cli(main) + + + + +.. raw:: html + + 
+    $ python 08_positional.py --help
+    usage: 08_positional.py [-h] [--verbose | --no-verbose] PATH PATH
+    
+    Command-line interface defined using a function signature. Note that this 
+    docstring is parsed to generate helptext.
+    
+    ╭─ positional arguments ────────────────────────────────────────╮
+    │ PATH              Source path. (required)                     │
+    │ PATH              Destination path. (required)                │
+    ╰───────────────────────────────────────────────────────────────╯
+    ╭─ options ─────────────────────────────────────────────────────╮
+    │ -h, --help        show this help message and exit             │
+    │ --verbose, --no-verbose                                       │
+    │                   Explain what is being done. (default: None) │
+    ╰───────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python 08_positional.py ./a ./b
+    source=PosixPath('a')
+    dest=PosixPath('b')
+    verbose=False
+
+ + + +.. raw:: html + + 
+    $ python 08_positional.py ./test1 ./test2 --verbose
+    source=PosixPath('test1')
+    dest=PosixPath('test2')
+    verbose=True
+
+.. _example-09_conf: + +Configuration via typing.Annotated[] +------------------------------------ + +The :mod:`tyro.conf` module contains utilities that can be used in conjunction +with :py:data:`typing.Annotated` to configure command-line interfaces beyond +what is expressible via static type annotations. + +Features here are supported, but generally unnecessary and should be used sparingly. + + +.. code-block:: python + :linenos: + + # 09_conf.py + import dataclasses + + from typing_extensions import Annotated + + import tyro + + @dataclasses.dataclass + class Args: + # A numeric field parsed as a positional argument. + positional: tyro.conf.Positional[int] + + # A boolean field with flag conversion turned off. + boolean: tyro.conf.FlagConversionOff[bool] = False + + # A numeric field that can't be changed via the CLI. + fixed: tyro.conf.Fixed[int] = 5 + + # A field with manually overridden properties. + manual: Annotated[ + str, + tyro.conf.arg( + name="renamed", + metavar="STRING", + help="A field with manually overridden properties!", + ), + ] = "Hello" + + if __name__ == "__main__": + print(tyro.cli(Args)) + + + + +.. raw:: html + + 
+    $ python ./09_conf.py --help
+    usage: 09_conf.py [-h] [OPTIONS] INT
+    
+    ╭─ positional arguments ─────────────────────────────────────────────────────╮
+    │ INT                     A numeric field parsed as a positional argument.   │
+    │                         (required)                                         │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+    ╭─ options ──────────────────────────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit                    │
+    │ --boolean {True,False}  A boolean field with flag conversion turned off.   │
+    │                         (default: False)                                   │
+    │ --fixed {fixed}         A numeric field that can't be changed via the CLI. │
+    │                         (fixed to: 5)                                      │
+    │ --renamed STRING        A field with manually overridden properties!       │
+    │                         (default: Hello)                                   │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./09_conf.py 5 --boolean True
+    Args(positional=5, boolean=True, fixed=5, manual='Hello')
+
+.. _example-10_aliases: + +Argument Aliases +---------------- + +:func:`tyro.conf.arg()` can be used to attach aliases to arguments. + + +.. code-block:: python + :linenos: + + # 10_aliases.py + from typing import Annotated + + import tyro + + def checkout( + branch: Annotated[str, tyro.conf.arg(aliases=["-b"])], + ) -> None: + """Check out a branch.""" + print(f"{branch=}") + + if __name__ == "__main__": + tyro.cli(checkout) + + + + +.. raw:: html + + 
+    $ python ./10_aliases.py --help
+    usage: 10_aliases.py [-h] --branch STR
+    
+    Check out a branch.
+    
+    ╭─ options ───────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit │
+    │ --branch STR, -b STR    (required)                      │
+    ╰─────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./10_aliases.py --branch main
+    branch='main'
+
+ + + +.. raw:: html + + 
+    $ python ./10_aliases.py -b main
+    branch='main'
+
+.. _example-11_type_aliases_py312: + +Type Aliases (3.12+) +-------------------- + +In Python 3.12, the :code:`type` statement is introduced to create type aliases. + + +.. code-block:: python + :linenos: + + # 11_type_aliases_py312.py + import dataclasses + + import tyro + + # Lazily-evaluated type alias. + type Field1Type = Inner + + @dataclasses.dataclass + class Inner: + a: int + b: str + + @dataclasses.dataclass + class Args: + """Description. + This should show up in the helptext!""" + + field1: Field1Type + """A field.""" + + field2: int = 3 + """A numeric field, with a default value.""" + + if __name__ == "__main__": + args = tyro.cli(Args) + print(args) + + + + +.. raw:: html + + 
+    $ python ./11_type_aliases_py312.py --help
+    usage: 11_type_aliases_py312.py [-h] [--field2 INT] --field1.a INT --field1.b
+                                    STR
+    
+    Description. This should show up in the helptext!
+    
+    ╭─ options ─────────────────────────────────────────────────────────────────╮
+    │ -h, --help            show this help message and exit                     │
+    │ --field2 INT          A numeric field, with a default value. (default: 3) │
+    ╰───────────────────────────────────────────────────────────────────────────╯
+    ╭─ field1 options ──────────────────────────────────────────────────────────╮
+    │ A field.                                                                  │
+    │ ────────────────────────────────                                          │
+    │ --field1.a INT        (required)                                          │
+    │ --field1.b STR        (required)                                          │
+    ╰───────────────────────────────────────────────────────────────────────────╯
+
+.. _example-12_counters: + +Counters +-------- + +Repeatable 'counter' arguments can be specified via :data:`tyro.conf.UseCounterAction`. + + +.. code-block:: python + :linenos: + + # 12_counters.py + from typing_extensions import Annotated + + import tyro + from tyro.conf import UseCounterAction + + def main( + verbosity: UseCounterAction[int], + aliased_verbosity: Annotated[UseCounterAction[int], tyro.conf.arg(aliases=["-v"])], + ) -> None: + """Example showing how to use counter actions. + + Args: + verbosity: Verbosity level. + aliased_verbosity: Same as above, but can also be specified with -v, -vv, -vvv, etc. + """ + print("Verbosity level:", verbosity) + print("Verbosity level (aliased):", aliased_verbosity) + + if __name__ == "__main__": + tyro.cli(main) + + + + +.. raw:: html + + 
+    $ python ./12_counters.py --help
+    usage: 12_counters.py [-h] [--verbosity] [--aliased-verbosity]
+    
+    Example showing how to use counter actions.
+    
+    ╭─ options ──────────────────────────────────────────────────────────────────╮
+    │ -h, --help         show this help message and exit                         │
+    │ --verbosity        Verbosity level. (repeatable)                           │
+    │ --aliased-verbosity, -v                                                    │
+    │                    Same as above, but can also be specified with -v, -vv,  │
+    │                    -vvv, etc. (repeatable)                                 │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./12_counters.py --verbosity
+    Verbosity level: 1
+    Verbosity level (aliased): 0
+
+ + + +.. raw:: html + + 
+    $ python ./12_counters.py --verbosity --verbosity
+    Verbosity level: 2
+    Verbosity level (aliased): 0
+
+ + + +.. raw:: html + + 
+    $ python ./12_counters.py -vvv
+    Verbosity level: 0
+    Verbosity level (aliased): 3
+
\ No newline at end of file diff --git a/docs/source/examples/custom_constructors.rst b/docs/source/examples/custom_constructors.rst new file mode 100644 index 000000000..070cba69f --- /dev/null +++ b/docs/source/examples/custom_constructors.rst @@ -0,0 +1,190 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +.. _example-category-custom_constructors: + +Custom constructors +=================== + +In these examples, we show how custom types can be parsed by and registered with :func:`tyro.cli`. + + +.. _example-01_primitive_annotation: + +Custom Primitive +---------------- + +.. note:: + + This is an advanced feature, which should not be needed for the vast + majority of use cases. If :mod:`tyro` is missing support for a built-in + Python type, please open an issue on `GitHub `_. + +For additional flexibility, :mod:`tyro.constructors` exposes tyro's API for +defining behavior for different types. There are two categories of types: +primitive types can be instantiated from a single commandline argument, while +struct types are broken down into multiple. + +In this example, we attach a custom constructor via a runtime annotation. + + +.. code-block:: python + :linenos: + + # 01_primitive_annotation.py + import json + + from typing_extensions import Annotated + + import tyro + + # A dictionary type, but `tyro` will expect a JSON string from the CLI. + JsonDict = Annotated[ + dict, + tyro.constructors.PrimitiveConstructorSpec( + # Number of arguments to consume. + nargs=1, + # Argument name in usage messages. + metavar="JSON", + # Convert a list of strings to an instance. The length of the list + # should match `nargs`. + instance_from_str=lambda args: json.loads(args[0]), + # Check if an instance is of the expected type. This is only used for + # helptext formatting in the presence of union types. + is_instance=lambda instance: isinstance(instance, dict), + # Convert an instance to a list of strings. This is used for handling + # default values that are set in Python. The length of the list should + # match `nargs`. + str_from_instance=lambda instance: [json.dumps(instance)], + ), + ] + + def main( + dict1: JsonDict, + dict2: JsonDict = {"default": None}, + ) -> None: + print(f"{dict1=}") + print(f"{dict2=}") + + if __name__ == "__main__": + tyro.cli(main) + + + + +.. raw:: html + + 
+    $ python ./01_primitive_annotation.py --help
+    usage: 01_primitive_annotation.py [-h] --dict1 JSON [--dict2 JSON]
+    
+    ╭─ options ───────────────────────────────────────────╮
+    │ -h, --help          show this help message and exit │
+    │ --dict1 JSON        (required)                      │
+    │ --dict2 JSON        (default: '{"default": null}')  │
+    ╰─────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./01_primitive_annotation.py --dict1 '{"hello": "world"}'
+    dict1={'hello': 'world'}
+    dict2={'default': None}
+
+ + + +.. raw:: html + + 
+    $ python ./01_primitive_annotation.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
+    dict1={'hello': 'world'}
+    dict2={'hello': 'world'}
+
+.. _example-02_primitive_registry: + +Custom Primitive (Registry) +--------------------------- + +In this example, we use :class:`tyro.constructors.PrimitiveConstructorSpec` to +define a rule that applies to all types that match ``dict[str, Any]``. + + +.. code-block:: python + :linenos: + + # 02_primitive_registry.py + import json + from typing import Any + + import tyro + + custom_registry = tyro.constructors.ConstructorRegistry() + + @custom_registry.primitive_rule + def _( + type_info: tyro.constructors.PrimitiveTypeInfo, + ) -> tyro.constructors.PrimitiveConstructorSpec | None: + # We return `None` if the rule does not apply. + if type_info.type != dict[str, Any]: + return None + + # If the rule applies, we return the constructor spec. + return tyro.constructors.PrimitiveConstructorSpec( + nargs=1, + metavar="JSON", + instance_from_str=lambda args: json.loads(args[0]), + is_instance=lambda instance: isinstance(instance, dict), + str_from_instance=lambda instance: [json.dumps(instance)], + ) + + def main( + dict1: dict[str, Any], + dict2: dict[str, Any] = {"default": None}, + ) -> None: + print(f"{dict1=}") + print(f"{dict2=}") + + if __name__ == "__main__": + # The custom registry is used as a context. + with custom_registry: + tyro.cli(main) + + + + +.. raw:: html + + 
+    $ python ./02_primitive_registry.py --help
+    usage: 02_primitive_registry.py [-h] --dict1 JSON [--dict2 JSON]
+    
+    ╭─ options ───────────────────────────────────────────╮
+    │ -h, --help          show this help message and exit │
+    │ --dict1 JSON        (required)                      │
+    │ --dict2 JSON        (default: '{"default": null}')  │
+    ╰─────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./02_primitive_registry.py --dict1 '{"hello": "world"}'
+    dict1={'hello': 'world'}
+    dict2={'default': None}
+
+ + + +.. raw:: html + + 
+    $ python ./02_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
+    dict1={'hello': 'world'}
+    dict2={'hello': 'world'}
+
\ No newline at end of file diff --git a/docs/source/examples/generics.rst b/docs/source/examples/generics.rst new file mode 100644 index 000000000..22163858a --- /dev/null +++ b/docs/source/examples/generics.rst @@ -0,0 +1,157 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +.. _example-category-generics: + +Generics +======== + +:mod:`tyro`'s understanding of Python's type system includes user-defined +parameterized types, which can reduce boilerplate and improve type safety. + + +.. _example-01_generics_py312: + +Generics (3.12+) +---------------- + +This example uses syntax introduced in Python 3.12 (`PEP 695 `_). + +.. note:: + + If used in conjunction with :code:`from __future__ import annotations`, the updated type parameter syntax requires Python 3.12.4 or newer. For technical details, see `this CPython PR `_. + + +.. code-block:: python + :linenos: + + # 01_generics_py312.py + import dataclasses + + import tyro + + @dataclasses.dataclass + class Point3[ScalarType: (int, float)]: + x: ScalarType + y: ScalarType + z: ScalarType + frame_id: str + + @dataclasses.dataclass + class Triangle: + a: Point3[float] + b: Point3[float] + c: Point3[float] + + @dataclasses.dataclass + class Args[ShapeType]: + shape: ShapeType + + if __name__ == "__main__": + args = tyro.cli(Args[Triangle]) + print(args) + + + + +.. raw:: html + + 
+    $ python ./01_generics_py312.py --help
+    usage: 01_generics_py312.py [-h] [OPTIONS]
+    
+    ╭─ options ───────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ shape.a options ───────────────────────────────────────╮
+    │ --shape.a.x FLOAT       (required)                      │
+    │ --shape.a.y FLOAT       (required)                      │
+    │ --shape.a.z FLOAT       (required)                      │
+    │ --shape.a.frame-id STR  (required)                      │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ shape.b options ───────────────────────────────────────╮
+    │ --shape.b.x FLOAT       (required)                      │
+    │ --shape.b.y FLOAT       (required)                      │
+    │ --shape.b.z FLOAT       (required)                      │
+    │ --shape.b.frame-id STR  (required)                      │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ shape.c options ───────────────────────────────────────╮
+    │ --shape.c.x FLOAT       (required)                      │
+    │ --shape.c.y FLOAT       (required)                      │
+    │ --shape.c.z FLOAT       (required)                      │
+    │ --shape.c.frame-id STR  (required)                      │
+    ╰─────────────────────────────────────────────────────────╯
+
+.. _example-02_generics: + +Generics (Legacy) +----------------- + +The legacy :py:class:`typing.Generic` and :py:class:`typing.TypeVar` syntax for +generic types is also supported. + + +.. code-block:: python + :linenos: + + # 02_generics.py + import dataclasses + from typing import Generic, TypeVar + + import tyro + + ScalarType = TypeVar("ScalarType", int, float) + ShapeType = TypeVar("ShapeType") + + @dataclasses.dataclass(frozen=True) + class Point3(Generic[ScalarType]): + x: ScalarType + y: ScalarType + z: ScalarType + frame_id: str + + @dataclasses.dataclass(frozen=True) + class Triangle: + a: Point3[float] + b: Point3[float] + c: Point3[float] + + @dataclasses.dataclass(frozen=True) + class Args(Generic[ShapeType]): + shape: ShapeType + + if __name__ == "__main__": + args = tyro.cli(Args[Triangle]) + print(args) + + + + +.. raw:: html + + 
+    $ python ./02_generics.py --help
+    usage: 02_generics.py [-h] [OPTIONS]
+    
+    ╭─ options ───────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ shape.a options ───────────────────────────────────────╮
+    │ --shape.a.x FLOAT       (required)                      │
+    │ --shape.a.y FLOAT       (required)                      │
+    │ --shape.a.z FLOAT       (required)                      │
+    │ --shape.a.frame-id STR  (required)                      │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ shape.b options ───────────────────────────────────────╮
+    │ --shape.b.x FLOAT       (required)                      │
+    │ --shape.b.y FLOAT       (required)                      │
+    │ --shape.b.z FLOAT       (required)                      │
+    │ --shape.b.frame-id STR  (required)                      │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ shape.c options ───────────────────────────────────────╮
+    │ --shape.c.x FLOAT       (required)                      │
+    │ --shape.c.y FLOAT       (required)                      │
+    │ --shape.c.z FLOAT       (required)                      │
+    │ --shape.c.frame-id STR  (required)                      │
+    ╰─────────────────────────────────────────────────────────╯
+
\ No newline at end of file diff --git a/docs/source/examples/nested_structures.rst b/docs/source/examples/nested_structures.rst new file mode 100644 index 000000000..a64a046a0 --- /dev/null +++ b/docs/source/examples/nested_structures.rst @@ -0,0 +1,594 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +.. _example-category-nested_structures: + +Nested Structures +================= + +In these examples, we show how :func:`tyro.cli` can be used to instantiate +nested structures. This can enable modular, reusable, and composable CLI +interfaces. + + +.. _example-01_nesting: + +Nested Dataclasses +------------------ + +Structures (typically :py:func:`dataclasses.dataclass`) can be nested to build hierarchical configuration +objects. This helps with modularity and grouping in larger projects. + + +.. code-block:: python + :linenos: + + # 01_nesting.py + import dataclasses + + import tyro + + @dataclasses.dataclass + class OptimizerConfig: + learning_rate: float = 3e-4 + weight_decay: float = 1e-2 + + @dataclasses.dataclass + class Config: + # Optimizer options. + opt: OptimizerConfig + + # Random seed. + seed: int = 0 + + if __name__ == "__main__": + config = tyro.cli(Config) + print(dataclasses.asdict(config)) + + + + +.. raw:: html + + 
+    $ python ./01_nesting.py --help
+    usage: 01_nesting.py [-h] [--seed INT] [--opt.learning-rate FLOAT]
+                         [--opt.weight-decay FLOAT]
+    
+    ╭─ options ─────────────────────────────────────────╮
+    │ -h, --help        show this help message and exit │
+    │ --seed INT        Random seed. (default: 0)       │
+    ╰───────────────────────────────────────────────────╯
+    ╭─ opt options ─────────────────────────────────────╮
+    │ Optimizer options.                                │
+    │ ───────────────────────────────────               │
+    │ --opt.learning-rate FLOAT                         │
+    │                   (default: 0.0003)               │
+    │ --opt.weight-decay FLOAT                          │
+    │                   (default: 0.01)                 │
+    ╰───────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./01_nesting.py --opt.learning-rate 1e-3
+    {'opt': {'learning_rate': 0.001, 'weight_decay': 0.01}, 'seed': 0}
+
+ + + +.. raw:: html + + 
+    $ python ./01_nesting.py --seed 4
+    {'opt': {'learning_rate': 0.0003, 'weight_decay': 0.01}, 'seed': 4}
+
+.. _example-02_nesting_in_func: + +Structures as Function Arguments +-------------------------------- + +Structures can also be used as input to functions. + + +.. code-block:: python + :linenos: + + # 02_nesting_in_func.py + import dataclasses + import pathlib + + import tyro + + @dataclasses.dataclass + class OptimizerConfig: + learning_rate: float = 3e-4 + weight_decay: float = 1e-2 + + @dataclasses.dataclass + class Config: + # Optimizer options. + optimizer: OptimizerConfig + + # Random seed. + seed: int = 0 + + def train( + out_dir: pathlib.Path, + config: Config, + ) -> None: + """Train a model. + + Args: + out_dir: Where to save logs and checkpoints. + config: Experiment configuration. + """ + print(f"Saving to: {out_dir}") + print(f"Config: f{config}") + + if __name__ == "__main__": + tyro.cli(train) + + + + +.. raw:: html + + 
+    $ python ./02_nesting_in_func.py --help
+    usage: 02_nesting_in_func.py [-h] [OPTIONS]
+    
+    Train a model.
+    
+    ╭─ options ──────────────────────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit                │
+    │ --out-dir PATH          Where to save logs and checkpoints. (required) │
+    ╰────────────────────────────────────────────────────────────────────────╯
+    ╭─ config options ───────────────────────────────────────────────────────╮
+    │ Experiment configuration.                                              │
+    │ ─────────────────────────────────────────────────                      │
+    │ --config.seed INT       Random seed. (default: 0)                      │
+    ╰────────────────────────────────────────────────────────────────────────╯
+    ╭─ config.optimizer options ─────────────────────────────────────────────╮
+    │ Optimizer options.                                                     │
+    │ ─────────────────────────────────────────────────                      │
+    │ --config.optimizer.learning-rate FLOAT                                 │
+    │                         (default: 0.0003)                              │
+    │ --config.optimizer.weight-decay FLOAT                                  │
+    │                         (default: 0.01)                                │
+    ╰────────────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./02_nesting_in_func.py --out-dir /tmp/test1
+    Saving to: /tmp/test1
+    Config: fConfig(optimizer=OptimizerConfig(learning_rate=0.0003, weight_decay=0.01), seed=0)
+
+ + + +.. raw:: html + + 
+    $ python ./02_nesting_in_func.py --out-dir /tmp/test2 --config.seed 4
+    Saving to: /tmp/test2
+    Config: fConfig(optimizer=OptimizerConfig(learning_rate=0.0003, weight_decay=0.01), seed=4)
+
+.. _example-03_nesting_containers: + +Nesting in Containers +--------------------- + +Structures can be nested inside of standard containers. + +.. warning:: + + When placing structures inside of containers like lists or tuples, the + length of the container must be inferrable from the annotation or default + value. + + +.. code-block:: python + :linenos: + + # 03_nesting_containers.py + import dataclasses + + import tyro + + @dataclasses.dataclass + class RGB: + r: int + g: int + b: int + + @dataclasses.dataclass + class Args: + color_tuple: tuple[RGB, RGB] + color_dict: dict[str, RGB] = dataclasses.field( + # We can't use mutable values as defaults directly. + default_factory={ + "red": RGB(255, 0, 0), + "green": RGB(0, 255, 0), + "blue": RGB(0, 0, 255), + }.copy + ) + + if __name__ == "__main__": + args = tyro.cli(Args) + print(args) + + + + +.. raw:: html + + 
+    $ python ./03_nesting_containers.py --help
+    usage: 03_nesting_containers.py [-h] [OPTIONS]
+    
+    ╭─ options ───────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ color-tuple.0 options ─────────────────────────────────╮
+    │ --color-tuple.0.r INT   (required)                      │
+    │ --color-tuple.0.g INT   (required)                      │
+    │ --color-tuple.0.b INT   (required)                      │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ color-tuple.1 options ─────────────────────────────────╮
+    │ --color-tuple.1.r INT   (required)                      │
+    │ --color-tuple.1.g INT   (required)                      │
+    │ --color-tuple.1.b INT   (required)                      │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ color-dict.red options ────────────────────────────────╮
+    │ --color-dict.red.r INT  (default: 255)                  │
+    │ --color-dict.red.g INT  (default: 0)                    │
+    │ --color-dict.red.b INT  (default: 0)                    │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ color-dict.green options ──────────────────────────────╮
+    │ --color-dict.green.r INT                                │
+    │                         (default: 0)                    │
+    │ --color-dict.green.g INT                                │
+    │                         (default: 255)                  │
+    │ --color-dict.green.b INT                                │
+    │                         (default: 0)                    │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ color-dict.blue options ───────────────────────────────╮
+    │ --color-dict.blue.r INT                                 │
+    │                         (default: 0)                    │
+    │ --color-dict.blue.g INT                                 │
+    │                         (default: 0)                    │
+    │ --color-dict.blue.b INT                                 │
+    │                         (default: 255)                  │
+    ╰─────────────────────────────────────────────────────────╯
+
+.. _example-04_dictionaries: + +Dictionaries and TypedDict +-------------------------- + +Dictionary inputs can be specified using either a standard ``dict[K, V]`` +annotation, or a :code:`TypedDict` subclass. + +For configuring :code:`TypedDict`, we also support :code:`total={True/False}`, +:code:`typing.Required`, and :code:`typing.NotRequired`. See the `Python docs `_ for all :code:`TypedDict` features. + + +.. code-block:: python + :linenos: + + # 04_dictionaries.py + from typing import TypedDict + + from typing_extensions import NotRequired + + import tyro + + class DictionarySchemaA( + TypedDict, + # Setting `total=False` specifies that not all keys need to exist. + total=False, + ): + learning_rate: float + betas: tuple[float, float] + + class DictionarySchemaB(TypedDict): + learning_rate: NotRequired[float] + """NotRequired[] specifies that a particular key doesn't need to exist.""" + betas: tuple[float, float] + + def main( + typed_dict_a: DictionarySchemaA, + typed_dict_b: DictionarySchemaB, + standard_dict: dict[str, float] = { + "learning_rate": 3e-4, + "beta1": 0.9, + "beta2": 0.999, + }, + ) -> None: + assert isinstance(typed_dict_a, dict) + assert isinstance(typed_dict_b, dict) + assert isinstance(standard_dict, dict) + print("Typed dict A:", typed_dict_a) + print("Typed dict B:", typed_dict_b) + print("Standard dict:", standard_dict) + + if __name__ == "__main__": + tyro.cli(main) + + + + +.. raw:: html + + 
+    $ python ./04_dictionaries.py --help
+    usage: 04_dictionaries.py [-h] [OPTIONS]
+    
+    ╭─ options ──────────────────────────────────────────────────────────────────╮
+    │ -h, --help        show this help message and exit                          │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+    ╭─ typed-dict-a options ─────────────────────────────────────────────────────╮
+    │ --typed-dict-a.learning-rate FLOAT                                         │
+    │                   (unset by default)                                       │
+    │ --typed-dict-a.betas FLOAT FLOAT                                           │
+    │                   (unset by default)                                       │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+    ╭─ typed-dict-b options ─────────────────────────────────────────────────────╮
+    │ --typed-dict-b.learning-rate FLOAT                                         │
+    │                   NotRequired[] specifies that a particular key doesn't    │
+    │                   need to exist. (unset by default)                        │
+    │ --typed-dict-b.betas FLOAT FLOAT                                           │
+    │                   (required)                                               │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+    ╭─ standard-dict options ────────────────────────────────────────────────────╮
+    │ --standard-dict.learning-rate FLOAT                                        │
+    │                   (default: 0.0003)                                        │
+    │ --standard-dict.beta1 FLOAT                                                │
+    │                   (default: 0.9)                                           │
+    │ --standard-dict.beta2 FLOAT                                                │
+    │                   (default: 0.999)                                         │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./04_dictionaries.py --typed-dict-a.learning-rate 3e-4 --typed-dict-b.betas 0.9 0.999
+    Typed dict A: {'learning_rate': 0.0003}
+    Typed dict B: {'betas': (0.9, 0.999)}
+    Standard dict: {'learning_rate': 0.0003, 'beta1': 0.9, 'beta2': 0.999}
+
+ + + +.. raw:: html + + 
+    $ python ./04_dictionaries.py --typed-dict-b.betas 0.9 0.999
+    Typed dict A: {}
+    Typed dict B: {'betas': (0.9, 0.999)}
+    Standard dict: {'learning_rate': 0.0003, 'beta1': 0.9, 'beta2': 0.999}
+
+.. _example-05_tuples: + +Tuples and NamedTuple +--------------------- + +Example using :func:`tyro.cli()` to instantiate tuple types. :code:`tuple`, +:py:data:`typing.Tuple`, and :py:class:`typing.NamedTuple` are all supported. + + +.. code-block:: python + :linenos: + + # 05_tuples.py + from typing import NamedTuple + + import tyro + + # Named tuples are interpreted as nested structures. + class Color(NamedTuple): + r: int + g: int + b: int + + class TupleType(NamedTuple): + """Description. + This should show up in the helptext!""" + + # Tuple types can contain raw values. + color: tuple[int, int, int] = (255, 0, 0) + + # Tuple types can contain nested structures. + two_colors: tuple[Color, Color] = (Color(255, 0, 0), Color(0, 255, 0)) + + if __name__ == "__main__": + x = tyro.cli(TupleType) + assert isinstance(x, tuple) + print(x) + + + + +.. raw:: html + + 
+    $ python ./05_tuples.py --help
+    usage: 05_tuples.py [-h] [OPTIONS]
+    
+    Description. This should show up in the helptext!
+    
+    ╭─ options ──────────────────────────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit                    │
+    │ --color INT INT INT     Tuple types can contain raw values. (default: 255  │
+    │                         0 0)                                               │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+    ╭─ two-colors.0 options ─────────────────────────────────────────────────────╮
+    │ --two-colors.0.r INT    (default: 255)                                     │
+    │ --two-colors.0.g INT    (default: 0)                                       │
+    │ --two-colors.0.b INT    (default: 0)                                       │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+    ╭─ two-colors.1 options ─────────────────────────────────────────────────────╮
+    │ --two-colors.1.r INT    (default: 0)                                       │
+    │ --two-colors.1.g INT    (default: 255)                                     │
+    │ --two-colors.1.b INT    (default: 0)                                       │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./05_tuples.py --color 127 127 127
+    TupleType(color=(127, 127, 127), two_colors=(Color(r=255, g=0, b=0), Color(r=0, g=255, b=0)))
+
+ + + +.. raw:: html + + 
+    $ python ./05_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0
+    TupleType(color=(255, 0, 0), two_colors=(Color(r=255, g=0, b=0), Color(r=127, g=0, b=0)))
+
+.. _example-06_pydantic: + +Pydantic Integration +-------------------- + +In addition to standard dataclasses, :func:`tyro.cli()` also supports +`Pydantic `_ models. + + +.. code-block:: python + :linenos: + + # 06_pydantic.py + from pydantic import BaseModel, Field + + import tyro + + class Args(BaseModel): + """Description. + This should show up in the helptext!""" + + field1: str + field2: int = Field(3, description="An integer field.") + + if __name__ == "__main__": + args = tyro.cli(Args) + print(args) + + + + +.. raw:: html + + 
+    $ python ./06_pydantic.py --help
+    usage: 06_pydantic.py [-h] --field1 STR [--field2 INT]
+    
+    Description. This should show up in the helptext!
+    
+    ╭─ options ───────────────────────────────────────────╮
+    │ -h, --help          show this help message and exit │
+    │ --field1 STR        (required)                      │
+    │ --field2 INT        An integer field. (default: 3)  │
+    ╰─────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./06_pydantic.py --field1 hello
+    field1='hello' field2=3
+
+ + + +.. raw:: html + + 
+    $ python ./06_pydantic.py --field1 hello --field2 5
+    field1='hello' field2=5
+
+.. _example-07_attrs: + +Attrs Integration +----------------- + +In addition to standard dataclasses, :func:`tyro.cli()` also supports +`attrs `_ classes. + + +.. code-block:: python + :linenos: + + # 07_attrs.py + import attr + + import tyro + + @attr.s + class Args: + """Description. + This should show up in the helptext!""" + + field1: str = attr.ib() + """A string field.""" + + field2: int = attr.ib(factory=lambda: 5) + """A required integer field.""" + + if __name__ == "__main__": + args = tyro.cli(Args) + print(args) + + + + +.. raw:: html + + 
+    $ python ./07_attrs.py --help
+    usage: 07_attrs.py [-h] --field1 STR [--field2 INT]
+    
+    Description. This should show up in the helptext!
+    
+    ╭─ options ──────────────────────────────────────────────────╮
+    │ -h, --help          show this help message and exit        │
+    │ --field1 STR        A string field. (required)             │
+    │ --field2 INT        A required integer field. (default: 5) │
+    ╰────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./07_attrs.py --field1 hello
+    Args(field1='hello', field2=5)
+
+ + + +.. raw:: html + + 
+    $ python ./07_attrs.py --field1 hello --field2 5
+    Args(field1='hello', field2=5)
+
\ No newline at end of file diff --git a/docs/source/examples/overriding_configs.rst b/docs/source/examples/overriding_configs.rst new file mode 100644 index 000000000..3f28f2aca --- /dev/null +++ b/docs/source/examples/overriding_configs.rst @@ -0,0 +1,375 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +.. _example-category-overriding_configs: + +Overriding Configs +================== + +In these examples, we show how :func:`tyro.cli` can be used to override values +in pre-instantiated configuration objects. + + +.. _example-01_dataclasses_defaults: + +Dataclasses + Defaults +---------------------- + +The :code:`default=` argument can be used to override default values in dataclass +types. + + +.. note:: + + When ``default=`` is used, we advise against mutation of configuration + objects from a dataclass's :code:`__post_init__` method [#f1]_. In the + example below, :code:`__post_init__` would be called twice: once for the + :code:`Args()` object provided as a default value and another time for the + :code:`Args()` objected instantiated by :func:`tyro.cli()`. This can cause + confusing behavior! Instead, we show below one example of how derived + fields can be defined immutably. + + .. [#f1] Official Python docs for ``__post_init__`` can be found `here `_. + + +.. code-block:: python + :linenos: + + # 01_dataclasses_defaults.py + import dataclasses + + import tyro + + @dataclasses.dataclass + class Args: + """Description. + This should show up in the helptext!""" + + string: str + """A string field.""" + + reps: int = 3 + """A numeric field, with a default value.""" + + @property + def derived_field(self) -> str: + return ", ".join([self.string] * self.reps) + + if __name__ == "__main__": + args = tyro.cli( + Args, + default=Args( + string="default string", + reps=tyro.MISSING, + ), + ) + print(args.derived_field) + + + + +.. raw:: html + + 
+    $ python ./01_dataclasses_defaults.py --help
+    usage: 01_dataclasses_defaults.py [-h] [--string STR] --reps INT
+    
+    Description. This should show up in the helptext!
+    
+    ╭─ options ─────────────────────────────────────────────────────────────╮
+    │ -h, --help          show this help message and exit                   │
+    │ --string STR        A string field. (default: 'default string')       │
+    │ --reps INT          A numeric field, with a default value. (required) │
+    ╰───────────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./01_dataclasses_defaults.py --reps 3
+    default string, default string, default string
+
+ + + +.. raw:: html + + 
+    $ python ./01_dataclasses_defaults.py --string hello --reps 5
+    hello, hello, hello, hello, hello
+
+.. _example-02_overriding_yaml: + +Overriding YAML Configs +----------------------- + +:mod:`tyro` understands a wide range of data structures, including standard dictionaries +and lists. + +If you have a library of existing YAML files that you want to use, `tyro` can +help override values within them. + +.. note:: + + We recommend dataclass configs for new projects. + + +.. code-block:: python + :linenos: + + # 02_overriding_yaml.py + import yaml + + import tyro + + # YAML configuration. Note that this could also be loaded from a file! Environment + # variables are an easy way to select between different YAML files. + default_yaml = r""" + exp_name: test + optimizer: + learning_rate: 0.0001 + type: adam + training: + batch_size: 32 + num_steps: 10000 + checkpoint_steps: + - 500 + - 1000 + - 1500 + """.strip() + + if __name__ == "__main__": + # Convert our YAML config into a nested dictionary. + default_config = yaml.safe_load(default_yaml) + + # Override fields in the dictionary. + overridden_config = tyro.cli(dict, default=default_config) + + # Print the overridden config. + overridden_yaml = yaml.safe_dump(overridden_config) + print(overridden_yaml) + + + + +.. raw:: html + + 
+    $ python ./02_overriding_yaml.py --help
+    usage: 02_overriding_yaml.py [-h] [OPTIONS]
+    
+    ╭─ options ───────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit │
+    │ --exp-name STR          (default: test)                 │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ optimizer options ─────────────────────────────────────╮
+    │ --optimizer.learning-rate FLOAT                         │
+    │                         (default: 0.0001)               │
+    │ --optimizer.type STR    (default: adam)                 │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ training options ──────────────────────────────────────╮
+    │ --training.batch-size INT                               │
+    │                         (default: 32)                   │
+    │ --training.num-steps INT                                │
+    │                         (default: 10000)                │
+    │ --training.checkpoint-steps [INT [INT ...]]             │
+    │                         (default: 500 1000 1500)        │
+    ╰─────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./02_overriding_yaml.py --training.checkpoint-steps 300 1000 9000
+    exp_name: test
+    optimizer:
+      learning_rate: 0.0001
+      type: adam
+    training:
+      batch_size: 32
+      checkpoint_steps:
+      - 300
+      - 1000
+      - 9000
+      num_steps: 10000
+    
+
+.. _example-03_choosing_base_configs: + +Choosing Base Configs +--------------------- + +One common pattern is to have a set of "base" configurations, which can be +selected from and then overridden. + +This is often implemented with a set of configuration files (e.g., YAML files). +With :mod:`tyro`, we can instead define each base configuration as a separate +Python object. + +After creating the base configurations, we can use the CLI to select one of +them and then override (existing) or fill in (missing) values. + +The helper function used here, :func:`tyro.extras.overridable_config_cli()`, is +a lightweight wrapper over :func:`tyro.cli()` and its Union-based subcommand +syntax. + + +.. code-block:: python + :linenos: + + # 03_choosing_base_configs.py + from dataclasses import dataclass + from typing import Callable, Literal + + from torch import nn + + import tyro + + @dataclass(frozen=True) + class ExperimentConfig: + # Dataset to run experiment on. + dataset: Literal["mnist", "imagenet-50"] + + # Model size. + num_layers: int + units: int + + # Batch size. + batch_size: int + + # Total number of training steps. + train_steps: int + + # Random seed. This is helpful for making sure that our experiments are all + # reproducible! + seed: int + + # Activation to use. Not specifiable via the commandline. + activation: Callable[[], nn.Module] + + # Note that we could also define this library using separate YAML files (similar to + # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type + # checking + IDE support. + default_configs = { + "small": ( + "Small experiment.", + ExperimentConfig( + dataset="mnist", + batch_size=2048, + num_layers=4, + units=64, + train_steps=30_000, + seed=0, + activation=nn.ReLU, + ), + ), + "big": ( + "Big experiment.", + ExperimentConfig( + dataset="imagenet-50", + batch_size=32, + num_layers=8, + units=256, + train_steps=100_000, + seed=0, + activation=nn.GELU, + ), + ), + } + if __name__ == "__main__": + config = tyro.extras.overridable_config_cli(default_configs) + print(config) + + +Overall helptext: + +.. raw:: html + + 
+    $ python ./03_choosing_base_configs.py --help
+    usage: 03_choosing_base_configs.py [-h] {small,big}
+    
+    ╭─ options ──────────────────────────────────────────╮
+    │ -h, --help         show this help message and exit │
+    ╰────────────────────────────────────────────────────╯
+    ╭─ subcommands ──────────────────────────────────────╮
+    │ {small,big}                                        │
+    │     small          Small experiment.               │
+    │     big            Big experiment.                 │
+    ╰────────────────────────────────────────────────────╯
+
+ +The "small" subcommand: + +.. raw:: html + + 
+    $ python ./03_choosing_base_configs.py small --help
+    usage: 03_choosing_base_configs.py small [-h] [SMALL OPTIONS]
+    
+    Small experiment.
+    
+    ╭─ options ──────────────────────────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit                    │
+    │ --dataset {mnist,imagenet-50}                                              │
+    │                         Dataset to run experiment on. (default: mnist)     │
+    │ --num-layers INT        Model size. (default: 4)                           │
+    │ --units INT             Model size. (default: 64)                          │
+    │ --batch-size INT        Batch size. (default: 2048)                        │
+    │ --train-steps INT       Total number of training steps. (default: 30000)   │
+    │ --seed INT              Random seed. This is helpful for making sure that  │
+    │                         our experiments are all reproducible! (default: 0) │
+    │ --activation {fixed}    Activation to use. Not specifiable via the         │
+    │                         commandline. (fixed to: <class                     │
+    │                         'torch.nn.modules.activation.ReLU'>)               │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./03_choosing_base_configs.py small --seed 94720
+    ExperimentConfig(dataset='mnist', num_layers=4, units=64, batch_size=2048, train_steps=30000, seed=94720, activation=<class 'torch.nn.modules.activation.ReLU'>)
+
+ +The "big" subcommand: + +.. raw:: html + + 
+    $ python ./03_choosing_base_configs.py big --help
+    usage: 03_choosing_base_configs.py big [-h] [BIG OPTIONS]
+    
+    Big experiment.
+    
+    ╭─ options ──────────────────────────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit                    │
+    │ --dataset {mnist,imagenet-50}                                              │
+    │                         Dataset to run experiment on. (default:            │
+    │                         imagenet-50)                                       │
+    │ --num-layers INT        Model size. (default: 8)                           │
+    │ --units INT             Model size. (default: 256)                         │
+    │ --batch-size INT        Batch size. (default: 32)                          │
+    │ --train-steps INT       Total number of training steps. (default: 100000)  │
+    │ --seed INT              Random seed. This is helpful for making sure that  │
+    │                         our experiments are all reproducible! (default: 0) │
+    │ --activation {fixed}    Activation to use. Not specifiable via the         │
+    │                         commandline. (fixed to: <class                     │
+    │                         'torch.nn.modules.activation.GELU'>)               │
+    ╰────────────────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./03_choosing_base_configs.py big --seed 94720
+    ExperimentConfig(dataset='imagenet-50', num_layers=8, units=256, batch_size=32, train_steps=100000, seed=94720, activation=<class 'torch.nn.modules.activation.GELU'>)
+
\ No newline at end of file diff --git a/docs/source/examples/pytorch_jax.rst b/docs/source/examples/pytorch_jax.rst new file mode 100644 index 000000000..fe9735c61 --- /dev/null +++ b/docs/source/examples/pytorch_jax.rst @@ -0,0 +1,159 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +.. _example-category-pytorch_jax: + +PyTorch / JAX +============= + +In these examples, we show some patterns for using :func:`tyro.cli` with PyTorch and JAX. + + +.. _example-01_pytorch_parallelism: + +PyTorch Parallelism +------------------- + +The :code:`console_outputs=` argument can be set to :code:`False` to suppress helptext and +error message printing. + +This is useful in PyTorch for distributed training scripts, where you only want +to print the helptext from the main process: + + +.. code-block:: python + + # HuggingFace Accelerate. + args = tyro.cli(Args, console_outputs=accelerator.is_main_process) + + # PyTorch DDP. + args = tyro.cli(Args, console_outputs=(rank == 0)) + + # PyTorch Lightning. + args = tyro.cli(Args, console_outputs=trainer.is_global_zero) + + +.. code-block:: python + :linenos: + + # 01_pytorch_parallelism.py + import dataclasses + + import tyro + + @dataclasses.dataclass + class Args: + """Description. + This should show up in the helptext!""" + + field1: int + """A field.""" + + field2: int = 3 + """A numeric field, with a default value.""" + + if __name__ == "__main__": + args = tyro.cli(Args, console_outputs=False) + print(args) + + + + +.. raw:: html + + 
+    $ python ./01_pytorch_parallelism.py --help
+
+.. _example-02_flax: + +JAX/Flax Integration +-------------------- + +If you use `flax.linen `_, modules can be instantiated +directly from :func:`tyro.cli()`. + + +.. code-block:: python + :linenos: + + # 02_flax.py + from flax import linen as nn + from jax import numpy as jnp + + import tyro + + class Classifier(nn.Module): + layers: int + """Layers in our network.""" + units: int = 32 + """Hidden unit count.""" + output_dim: int = 10 + """Number of classes.""" + + @nn.compact + def __call__(self, x: jnp.ndarray) -> jnp.ndarray: # type: ignore + for i in range(self.layers - 1): + x = nn.Dense( + self.units, + kernel_init=nn.initializers.kaiming_normal(), + )(x) + x = nn.relu(x) + + x = nn.Dense( + self.output_dim, + kernel_init=nn.initializers.xavier_normal(), + )(x) + x = nn.sigmoid(x) + return x + + def train(model: Classifier, num_iterations: int = 1000) -> None: + """Train a model. + + Args: + model: Model to train. + num_iterations: Number of training iterations. + """ + print(f"{model=}") + print(f"{num_iterations=}") + + if __name__ == "__main__": + tyro.cli(train) + + + + +.. raw:: html + + 
+    $ python ./02_flax.py --help
+    usage: 02_flax.py [-h] [OPTIONS]
+    
+    Train a model.
+    
+    ╭─ options ──────────────────────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit                │
+    │ --num-iterations INT    Number of training iterations. (default: 1000) │
+    ╰────────────────────────────────────────────────────────────────────────╯
+    ╭─ model options ────────────────────────────────────────────────────────╮
+    │ Model to train.                                                        │
+    │ ─────────────────────────────────────────────────────────              │
+    │ --model.layers INT      Layers in our network. (required)              │
+    │ --model.units INT       Hidden unit count. (default: 32)               │
+    │ --model.output-dim INT  Number of classes. (default: 10)               │
+    ╰────────────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./02_flax.py --model.layers 4
+    model=Classifier(
+        # attributes
+        layers = 4
+        units = 32
+        output_dim = 10
+    )
+    num_iterations=1000
+
\ No newline at end of file diff --git a/docs/source/examples/subcommands.rst b/docs/source/examples/subcommands.rst new file mode 100644 index 000000000..e99d5a762 --- /dev/null +++ b/docs/source/examples/subcommands.rst @@ -0,0 +1,573 @@ +.. Comment: this file is automatically generated by `update_example_docs.py`. + It should not be modified manually. + +.. _example-category-subcommands: + +Subcommands +=========== + +In these examples, we show how :func:`tyro.cli` can be used to create CLI +interfaces with subcommands. + + +.. _example-01_subcommands: + +Subcommands are Unions +---------------------- + +All of :mod:`tyro`'s subcommand features are built using unions over struct +types (typically dataclasses). Subcommands are used to choose between types in +the union; arguments are then populated from the chosen type. + +.. note:: + + For configuring subcommands beyond what can be expressed with type annotations, see + :func:`tyro.conf.subcommand()`. + + +.. code-block:: python + :linenos: + + # 01_subcommands.py + from __future__ import annotations + + import dataclasses + + import tyro + + @dataclasses.dataclass(frozen=True) + class Checkout: + """Checkout a branch.""" + + branch: str + + @dataclasses.dataclass(frozen=True) + class Commit: + """Commit changes.""" + + message: str + + if __name__ == "__main__": + cmd = tyro.cli(Checkout | Commit) + print(cmd) + + +Print the helptext. This will show the available subcommands: + +.. raw:: html + + 
+    $ python ./01_subcommands.py --help
+    usage: 01_subcommands.py [-h] {checkout,commit}
+    
+    ╭─ options ───────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ subcommands ───────────────────────────────────────────╮
+    │ {checkout,commit}                                       │
+    │     checkout            Checkout a branch.              │
+    │     commit              Commit changes.                 │
+    ╰─────────────────────────────────────────────────────────╯
+
+ +The `commit` subcommand: + +.. raw:: html + + 
+    $ python ./01_subcommands.py commit --help
+    usage: 01_subcommands.py commit [-h] --message STR
+    
+    Commit changes.
+    
+    ╭─ options ────────────────────────────────────────────╮
+    │ -h, --help           show this help message and exit │
+    │ --message STR        (required)                      │
+    ╰──────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./01_subcommands.py commit --message hello
+    Commit(message='hello')
+
+ +The `checkout` subcommand: + +.. raw:: html + + 
+    $ python ./01_subcommands.py checkout --help
+    usage: 01_subcommands.py checkout [-h] --branch STR
+    
+    Checkout a branch.
+    
+    ╭─ options ───────────────────────────────────────────╮
+    │ -h, --help          show this help message and exit │
+    │ --branch STR        (required)                      │
+    ╰─────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./01_subcommands.py checkout --branch main
+    Checkout(branch='main')
+
+.. _example-02_subcommands_in_func: + +Subcommands as Function Arguments +--------------------------------- + +A subcommand will be created for each input annotated with a union over +struct types. + +.. note:: + + To prevent :func:`tyro.cli()` from converting a Union type into a subcommand, + use :class:`tyro.conf.AvoidSubcommands`. + +.. note:: + + Argument ordering for subcommands can be tricky. In the example below, + ``--shared-arg`` must always come *before* the subcommand. As an option for + alleviating this, see :class:`tyro.conf.ConsolidateSubcommandArgs`. + + +.. code-block:: python + :linenos: + + # 02_subcommands_in_func.py + from __future__ import annotations + + import dataclasses + + import tyro + + @dataclasses.dataclass(frozen=True) + class Checkout: + """Checkout a branch.""" + + branch: str + + @dataclasses.dataclass(frozen=True) + class Commit: + """Commit changes.""" + + message: str + + def main( + shared_arg: int, + cmd: Checkout | Commit = Checkout(branch="default"), + ): + print(f"{shared_arg=}") + print(cmd) + + if __name__ == "__main__": + tyro.cli(main) + + +Print the helptext. This will show the available subcommands: + +.. raw:: html + + 
+    $ python ./02_subcommands_in_func.py --help
+    usage: 02_subcommands_in_func.py [-h] --shared-arg INT
+                                     [{cmd:checkout,cmd:commit}]
+    
+    ╭─ options ───────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit │
+    │ --shared-arg INT        (required)                      │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ optional subcommands ──────────────────────────────────╮
+    │ (default: cmd:checkout)                                 │
+    │ ──────────────────────────────────────────              │
+    │ [{cmd:checkout,cmd:commit}]                             │
+    │     cmd:checkout        Checkout a branch.              │
+    │     cmd:commit          Commit changes.                 │
+    ╰─────────────────────────────────────────────────────────╯
+
+ +Using the default subcommand: + +.. raw:: html + + 
+    $ python ./02_subcommands_in_func.py --shared-arg 100
+    shared_arg=100
+    Checkout(branch='default')
+
+ +Choosing a different subcommand: + +.. raw:: html + + 
+    $ python ./02_subcommands_in_func.py --shared-arg 100 cmd:commit --cmd.message 'Hello!'
+    shared_arg=100
+    Commit(message='Hello!')
+
+.. _example-03_multiple_subcommands: + +Sequenced Subcommands +--------------------- + +Multiple unions over struct types are populated using a series of subcommands. + + +.. code-block:: python + :linenos: + + # 03_multiple_subcommands.py + from __future__ import annotations + + import dataclasses + from typing import Literal + + import tyro + + # Possible dataset configurations. + + @dataclasses.dataclass + class Mnist: + binary: bool = False + """Set to load binary version of MNIST dataset.""" + + @dataclasses.dataclass + class ImageNet: + subset: Literal[50, 100, 1000] + """Choose between ImageNet-50, ImageNet-100, ImageNet-1000, etc.""" + + # Possible optimizer configurations. + + @dataclasses.dataclass + class Adam: + learning_rate: float = 1e-3 + betas: tuple[float, float] = (0.9, 0.999) + + @dataclasses.dataclass + class Sgd: + learning_rate: float = 3e-4 + + # Train script. + + def train( + dataset: Mnist | ImageNet = Mnist(), + optimizer: Adam | Sgd = Adam(), + ) -> None: + """Example training script. + + Args: + dataset: Dataset to train on. + optimizer: Optimizer to train with. + + Returns: + None: + """ + print(dataset) + print(optimizer) + + if __name__ == "__main__": + tyro.cli(train, config=(tyro.conf.ConsolidateSubcommandArgs,)) + + +Note that we apply the :class:`tyro.conf.ConsolidateSubcommandArgs` flag. +This pushes all arguments to the end of the command: + +.. raw:: html + + 
+    $ python ./03_multiple_subcommands.py --help
+    usage: 03_multiple_subcommands.py [-h] {dataset:mnist,dataset:image-net}
+    
+    Example training script.
+    
+    ╭─ options ─────────────────────────────────────────╮
+    │ -h, --help        show this help message and exit │
+    ╰───────────────────────────────────────────────────╯
+    ╭─ subcommands ─────────────────────────────────────╮
+    │ Dataset to train on.                              │
+    │ ─────────────────────────────────                 │
+    │ {dataset:mnist,dataset:image-net}                 │
+    │     dataset:mnist                                 │
+    │     dataset:image-net                             │
+    ╰───────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./03_multiple_subcommands.py dataset:mnist --help
+    usage: 03_multiple_subcommands.py dataset:mnist [-h]
+                                                    {optimizer:adam,optimizer:sgd}
+    
+    ╭─ options ─────────────────────────────────────────╮
+    │ -h, --help        show this help message and exit │
+    ╰───────────────────────────────────────────────────╯
+    ╭─ subcommands ─────────────────────────────────────╮
+    │ Optimizer to train with.                          │
+    │ ──────────────────────────────                    │
+    │ {optimizer:adam,optimizer:sgd}                    │
+    │     optimizer:adam                                │
+    │     optimizer:sgd                                 │
+    ╰───────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --help
+    usage: 03_multiple_subcommands.py dataset:mnist optimizer:adam
+           [-h] [--optimizer.learning-rate FLOAT] [--optimizer.betas FLOAT FLOAT]
+           [--dataset.binary | --dataset.no-binary]
+    
+    ╭─ options ────────────────────────────────────────────────────────╮
+    │ -h, --help                                                       │
+    │     show this help message and exit                              │
+    ╰──────────────────────────────────────────────────────────────────╯
+    ╭─ optimizer options ──────────────────────────────────────────────╮
+    │ --optimizer.learning-rate FLOAT                                  │
+    │     (default: 0.001)                                             │
+    │ --optimizer.betas FLOAT FLOAT                                    │
+    │     (default: 0.9 0.999)                                         │
+    ╰──────────────────────────────────────────────────────────────────╯
+    ╭─ dataset options ────────────────────────────────────────────────╮
+    │ --dataset.binary, --dataset.no-binary                            │
+    │     Set to load binary version of MNIST dataset. (default: None) │
+    
+    ╰──────────────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 --dataset.binary
+    Mnist(binary=True)
+    Adam(learning_rate=0.0003, betas=(0.9, 0.999))
+
+.. _example-04_decorator_subcommands: + +Decorator-based Subcommands +--------------------------- + +:func:`tyro.extras.SubcommandApp()` provides a decorator-based API for +subcommands, which is inspired by `click `_. + + +.. code-block:: python + :linenos: + + # 04_decorator_subcommands.py + from tyro.extras import SubcommandApp + + app = SubcommandApp() + + @app.command + def greet(name: str, loud: bool = False) -> None: + """Greet someone.""" + greeting = f"Hello, {name}!" + if loud: + greeting = greeting.upper() + print(greeting) + + @app.command(name="addition") + def add(a: int, b: int) -> None: + """Add two numbers.""" + print(f"{a} + {b} = {a + b}") + + if __name__ == "__main__": + app.cli() + + + + +.. raw:: html + + 
+    $ python 04_decorator_subcommands.py --help
+    usage: 04_decorator_subcommands.py [-h] {greet,addition}
+    
+    ╭─ options ───────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ subcommands ───────────────────────────────────────────╮
+    │ {greet,addition}                                        │
+    │     greet               Greet someone.                  │
+    │     addition            Add two numbers.                │
+    ╰─────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python 04_decorator_subcommands.py greet --help
+    usage: 04_decorator_subcommands.py greet [-h] --name STR [--loud | --no-loud]
+    
+    Greet someone.
+    
+    ╭─ options ───────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit │
+    │ --name STR              (required)                      │
+    │ --loud, --no-loud       (default: None)                 │
+    ╰─────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python 04_decorator_subcommands.py greet --name Alice
+    Hello, Alice!
+
+ + + +.. raw:: html + + 
+    $ python 04_decorator_subcommands.py greet --name Bob --loud
+    HELLO, BOB!
+
+ + + +.. raw:: html + + 
+    $ python 04_decorator_subcommands.py addition --help
+    usage: 04_decorator_subcommands.py addition [-h] --a INT --b INT
+    
+    Add two numbers.
+    
+    ╭─ options ─────────────────────────────────────────╮
+    │ -h, --help        show this help message and exit │
+    │ --a INT           (required)                      │
+    │ --b INT           (required)                      │
+    ╰───────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python 04_decorator_subcommands.py addition --a 5 --b 3
+    5 + 3 = 8
+
+.. _example-05_subcommands_func: + +Subcommands from Functions +-------------------------- + +We provide a shorthand for generating a subcommand CLI from a dictionary. This +is a thin wrapper around :func:`tyro.cli()`'s more verbose, type-based API. If +more generality is needed, the internal working are explained in the docs for +:func:`tyro.extras.subcommand_cli_from_dict()`. + + +.. code-block:: python + :linenos: + + # 05_subcommands_func.py + import tyro + + def checkout(branch: str) -> None: + """Check out a branch.""" + print(f"{branch=}") + + def commit(message: str, all: bool = False) -> None: + """Make a commit.""" + print(f"{message=} {all=}") + + if __name__ == "__main__": + tyro.extras.subcommand_cli_from_dict( + { + "checkout": checkout, + "commit": commit, + } + ) + + + + +.. raw:: html + + 
+    $ python ./05_subcommands_func.py --help
+    usage: 05_subcommands_func.py [-h] {checkout,commit}
+    
+    ╭─ options ───────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ subcommands ───────────────────────────────────────────╮
+    │ {checkout,commit}                                       │
+    │     checkout            Check out a branch.             │
+    │     commit              Make a commit.                  │
+    ╰─────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./05_subcommands_func.py commit --help
+    usage: 05_subcommands_func.py commit [-h] --message STR [--all | --no-all]
+    
+    Make a commit.
+    
+    ╭─ options ──────────────────────────────────────────────╮
+    │ -h, --help             show this help message and exit │
+    │ --message STR          (required)                      │
+    │ --all, --no-all        (default: None)                 │
+    ╰────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./05_subcommands_func.py commit --message hello --all
+    message='hello' all=True
+
+ + + +.. raw:: html + + 
+    $ python ./05_subcommands_func.py checkout --help
+    usage: 05_subcommands_func.py checkout [-h] --branch STR
+    
+    Check out a branch.
+    
+    ╭─ options ───────────────────────────────────────────╮
+    │ -h, --help          show this help message and exit │
+    │ --branch STR        (required)                      │
+    ╰─────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./05_subcommands_func.py checkout --branch main
+    branch='main'
+
\ No newline at end of file diff --git a/docs/source/goals_and_alternatives.md b/docs/source/goals_and_alternatives.md index b8dc54843..8ab178078 100644 --- a/docs/source/goals_and_alternatives.md +++ b/docs/source/goals_and_alternatives.md @@ -14,7 +14,7 @@ Usage distinctions are the result of two API goals: - In contrast, similar libraries have more expansive APIs , and require more library-specific structures, decorators, or metadata formats for configuring parsing behavior. -- **Strict typing.** Any type that can be annotated and unambiguously parsed +- **Types.** Any type that can be annotated and unambiguously parsed with an `argparse`-style CLI interface should work out-of-the-box; any public API that isn't statically analyzable should be avoided. - In contrast, many similar libraries implement features that depend on diff --git a/docs/source/helptext_generation.md b/docs/source/helptext_generation.md index 8b46e5882..5efffdef4 100644 --- a/docs/source/helptext_generation.md +++ b/docs/source/helptext_generation.md @@ -12,10 +12,7 @@ and Epydoc docstrings are supported as well. Under the hood, all of these options use [docstring_parser](https://github.com/rr-/docstring_parser). ```python -def main( - field1: str, - field2: int = 3, -) -> None: +def main(field1: str, field2: int = 3) -> None: """Function, whose arguments will be populated from a CLI interface. Args: diff --git a/docs/source/index.md b/docs/source/index.md index dbcbae069..cb95fefdc 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -2,8 +2,7 @@ |build| |nbsp| |ruff| |nbsp| |mypy| |nbsp| |pyright| |nbsp| |coverage| |nbsp| |versions| -:func:`tyro.cli()` is a tool for generating CLI -interfaces. +:func:`tyro.cli()` is a tool for generating CLI interfaces in Python. We can define configurable scripts using functions: @@ -93,53 +92,17 @@ shell completion. supported_types .. toctree:: - :caption: Basics + :caption: Examples :hidden: - :maxdepth: 1 - :titlesonly: - :glob: - - examples/01_basics/* - - -.. toctree:: - :caption: Hierarchies - :hidden: - :maxdepth: 1 - :titlesonly: - :glob: - - examples/02_nesting/* - - -.. toctree:: - :caption: Config Management - :hidden: - :maxdepth: 1 :titlesonly: - :glob: - - examples/03_config_systems/* - - -.. toctree:: - :caption: Additional Features - :hidden: - :maxdepth: 1 - :titlesonly: - :glob: - - examples/04_additional/* - - -.. toctree:: - :caption: Custom Constructors - :hidden: - :maxdepth: 1 - :titlesonly: - :glob: - examples/05_custom_constructors/* + ./examples/basics.rst + ./examples/nested_structures.rst + ./examples/subcommands.rst + ./examples/overriding_configs.rst + ./examples/generics.rst + ./examples/custom_constructors.rst + ./examples/pytorch_jax.rst .. toctree:: diff --git a/docs/source/supported_types.rst b/docs/source/supported_types.rst index 3fdc1707a..772d1e95f 100644 --- a/docs/source/supported_types.rst +++ b/docs/source/supported_types.rst @@ -29,7 +29,7 @@ What's not supported -------------------- -There are some limitations. We currently _do not_ support: +There are some limitations. We currently *do not* support: - Variable-length sequences over nested structures, unless a default is provided. For types like ``list[Dataclass]``, we require a default value to @@ -41,6 +41,5 @@ There are some limitations. We currently _do not_ support: ambiguous to parse and not supported. - Self-referential types, like ``type RecursiveList[T] = T | list[RecursiveList[T]]``. -In each of these cases, a `custom -constructor `_ -can be defined as a workaround. +In each of these cases, a :ref:`custom constructor +` can be defined as a workaround. diff --git a/docs/update_example_docs.py b/docs/update_example_docs.py index 475b9dbe2..a89bf4d21 100644 --- a/docs/update_example_docs.py +++ b/docs/update_example_docs.py @@ -1,23 +1,121 @@ +# mypy: ignore-errors """Helper script for updating the auto-generated examples pages in the documentation.""" from __future__ import annotations import dataclasses +import io +import os import pathlib +import pty +import re import shlex import shutil -from typing import Iterable +import subprocess +from pathlib import Path +from typing import Iterable, Optional, Tuple, Union + +from tqdm import tqdm import tyro +def command_to_rst( + command: list[str], + cwd: Optional[Union[str, Path]] = None, + shell: bool = False, +) -> Tuple[str, int]: + """ + Run a command and format its output (including ANSI codes) as RST with HTML colors. + Uses a pseudo-terminal to ensure ANSI color codes are output. + + Args: + command: The command to run (string or list of strings) + cwd: Working directory to run the command in (str or Path) + shell: Whether to run the command through the shell + + Returns: + Tuple of (formatted RST string, return code) + """ + # Convert cwd to Path if provided + if cwd is not None: + cwd = Path(cwd).expanduser().resolve() + if not cwd.exists(): + raise FileNotFoundError(f"Working directory does not exist: {cwd}") + if not cwd.is_dir(): + raise NotADirectoryError( + f"Working directory path is not a directory: {cwd}" + ) + + # Create a pseudo-terminal to capture colored output + m, s = pty.openpty() + + # Set terminal type to ensure color support + env = os.environ.copy() + env["TERM"] = "xterm-256color" + process = subprocess.Popen( + command, + stdout=s, + stderr=s, + cwd=cwd, + shell=shell, + env=env, + close_fds=True, + ) + + os.close(s) + + # Read output from the master end of the pseudo-terminal + output = io.BytesIO() + while True: + try: + data = os.read(m, 1024) + if not data: + break + output.write(data) + except OSError: + break + + os.close(m) + process.wait() + + # Decode the captured output + output_str = output.getvalue().decode("utf-8", errors="replace") + + from ansi2html import Ansi2HTMLConverter + + # Create an Ansi2HTMLConverter instance + converter = Ansi2HTMLConverter(inline=True, scheme="osx-basic") + + # Convert ANSI codes to HTML + html_output = converter.convert(output_str, full=False) + + # Clean up any remaining ANSI codes (just in case) + output_str = re.sub("\x1b\\[[0-9;]*[mGKH]", "", html_output) + + # Format as RST code block with HTML + rst_output = ".. raw:: html\n\n" + rst_output += ' 
\n'
+    rst_output += f'    $ {shlex.join(command)}\n'
+
+    # Indent and HTML-escape the content
+    for line in output_str.splitlines():
+        # We don't escape < and > because we want to preserve HTML tags
+        # but we do escape & to prevent XML entities from being interpreted
+        rst_output += f"    {line}\n"
+
+    rst_output += "
\n" + + return rst_output, process.returncode + + @dataclasses.dataclass class ExampleMetadata: index: str index_with_zero: str source: str title: str - usages: Iterable[str] + usages: tuple[tuple[str, str]] # (comment, command) description: str @staticmethod @@ -37,19 +135,24 @@ def from_path(path: pathlib.Path) -> ExampleMetadata: title, _, description = docstring.partition("\n") description, _, usage_text = description.partition("Usage:") - example_usages = map( - lambda x: x[1:-1], - filter( - lambda line: line.startswith("`") and line.endswith("`"), - usage_text.split("\n"), - ), - ) + example_usages: list[tuple[str, str]] = [] + comment = "" + for usage_line in usage_text.splitlines(): + usage_line = usage_line.strip() + if usage_line.startswith("#"): + comment += usage_line.strip().lstrip("#").strip() + "\n" + elif usage_line.startswith("python"): + example_usages.append((comment.strip(), usage_line)) + comment = "" + # else: + # assert len(usage_line) == 0, usage_line + return ExampleMetadata( index=index, index_with_zero=index_with_zero, source=source.partition('"""')[2].partition('"""')[2].strip(), title=title, - usages=example_usages, + usages=tuple(example_usages), description=description.strip(), ) @@ -67,46 +170,97 @@ def main( examples_dir: pathlib.Path = REPO_ROOT / "examples", sphinx_source_dir: pathlib.Path = REPO_ROOT / "docs" / "source", ) -> None: + print("\nStarting documentation update...") + print(f"Examples directory: {examples_dir}") + print(f"Sphinx source directory: {sphinx_source_dir}") + example_doc_dir = sphinx_source_dir / "examples" + print(f"Cleaning up old docs directory: {example_doc_dir}") shutil.rmtree(example_doc_dir) example_doc_dir.mkdir() - for path in get_example_paths(examples_dir): - ex = ExampleMetadata.from_path(path) - path_for_sphinx = pathlib.Path("..") / ".." / path.relative_to(REPO_ROOT) + category_set: set[str] = set() - usage_lines = [] - for usage in ex.usages: - args = shlex.split(usage) - python_index = args.index("python") - sphinx_usage = shlex.join( - args[:python_index] - + ["python", path_for_sphinx.as_posix()] - + args[python_index + 2 :] - ) + from concurrent.futures import ThreadPoolExecutor - # Note that :kbd: in Sphinx does unnecessary stuff we want to avoid, see: - # https://github.com/sphinx-doc/sphinx/issues/7530 - # - # Instead, we just use raw HTML. - assert "../../examples/" in sphinx_usage - command = sphinx_usage.replace("../../examples/", "") + def process_example(path, examples_dir, REPO_ROOT): + print(f"Processing example: {path.name}") + ex = ExampleMetadata.from_path(path) + usage_lines = [] + for comment, usage in ex.usages: usage_lines += [ - "------------", "", - ".. raw:: html", + f"{comment}", + "", + ] + command_to_rst(shlex.split(usage), cwd=path.parent)[0].splitlines() + + category = path.parent.name + + example_content = "\n".join( + [ + f".. _example-{path.stem}:", "", - f" {command}", + f"{ex.title}", + "-" * len(ex.title), "", - f".. program-output:: {sphinx_usage}", + ex.description, + "", + "", + ".. code-block:: python", + " :linenos:", + "", + f" # {path.name}", + "\n".join( + f" {line}".rstrip() + for line in ex.source.replace("\n\n\n", "\n\n").splitlines() + ), "", ] + + usage_lines + ) + return category, example_content + + with ThreadPoolExecutor() as executor: + futures = [] + for path in get_example_paths(examples_dir): + futures.append( + executor.submit(process_example, path, examples_dir, REPO_ROOT) + ) - relative_dir = path.parent.relative_to(examples_dir) - target_dir = example_doc_dir / relative_dir - target_dir.mkdir(exist_ok=True, parents=True) + category_set = set() + example_contents: dict[str, str] = {} + for future in tqdm(futures, total=len(futures)): + category, content = future.result() + category_set.add(category) + if category not in example_contents: + example_contents[category] = [] + example_contents[category].append(content) - (target_dir / f"{path.stem}.rst").write_text( + print(f"\nProcessing {len(category_set)} categories...") + for category in category_set: + print(f"\nProcessing category: {category}") + + example_category_dir = examples_dir / category + readme_path = example_category_dir / "README.rst" + if readme_path.exists(): + print("Found", readme_path) + readme_content = readme_path.read_text() + else: + category_title = " ".join(category.split("_")[1:]).title() + readme_content = "\n".join( + [ + f"{category_title}", + "=" * len(category_title), + ] + ) + + # 0_basics => basics + number, _, category_clean = category.partition("_") + int(number) + + output_file = example_doc_dir / f"{category_clean}.rst" + print(f"Writing documentation to: {output_file}") + output_file.write_text( "\n".join( [ ( @@ -115,25 +269,19 @@ def main( ), " It should not be modified manually.", "", - f"{ex.title}", - "==========================================", - "", - ex.description, + f".. _example-category-{category_clean}:", "", - "", - ".. code-block:: python", - " :linenos:", - "", - "", - "\n".join( - f" {line}".rstrip() for line in ex.source.split("\n") - ), + readme_content, "", ] - + usage_lines - ) + + example_contents[category] + ), + encoding="utf-8", ) + output_file.parent.mkdir(parents=True, exist_ok=True) if __name__ == "__main__": + print("Starting example documentation generator...") tyro.cli(main, description=__doc__) + print("\nDocumentation generation complete!") diff --git a/examples/01_basics/01_functions.py b/examples/01_basics/01_functions.py index d9fe17413..b12013bc9 100644 --- a/examples/01_basics/01_functions.py +++ b/examples/01_basics/01_functions.py @@ -4,18 +4,19 @@ arguments populated from the CLI. Usage: -`python ./01_functions.py --help` -`python ./01_functions.py --field1 hello` -`python ./01_functions.py --field1 hello --field2 10` + + # We can use ``--help`` to show the help message, or ``--field1`` and + # ``--field2`` to set the arguments: + python ./01_functions.py --help + python ./01_functions.py --field1 hello + python ./01_functions.py --field1 hello --field2 10 + """ import tyro -def main( - field1: str, - field2: int = 3, -) -> None: +def main(field1: str, field2: int = 3) -> None: """Function, whose arguments will be populated from a CLI interface. Args: diff --git a/examples/01_basics/02_dataclasses.py b/examples/01_basics/02_dataclasses.py index 09e94bbfc..4c41da278 100644 --- a/examples/01_basics/02_dataclasses.py +++ b/examples/01_basics/02_dataclasses.py @@ -1,19 +1,24 @@ """Dataclasses -Common pattern: use :func:`tyro.cli()` to instantiate a dataclass. +In addition to functions, :func:`tyro.cli()` can also take dataclasses as input. Usage: -`python ./02_dataclasses.py --help` -`python ./02_dataclasses.py --field1 hello` -`python ./02_dataclasses.py --field1 hello --field2 5` + + # To show the help message, we can use the ``--help`` flag: + python ./02_dataclasses.py --help + + # We can override ``field1`` and ``field2``: + python ./02_dataclasses.py --field1 hello + python ./02_dataclasses.py --field1 hello --field2 5 """ -import dataclasses +from dataclasses import dataclass +from pprint import pprint import tyro -@dataclasses.dataclass +@dataclass class Args: """Description. This should show up in the helptext!""" @@ -27,4 +32,4 @@ class Args: if __name__ == "__main__": args = tyro.cli(Args) - print(args) + pprint(args) diff --git a/examples/01_basics/03_dataclasses_defaults.py b/examples/01_basics/03_dataclasses_defaults.py deleted file mode 100644 index 9b207e8c4..000000000 --- a/examples/01_basics/03_dataclasses_defaults.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Dataclasses + Defaults - -The :code:`default=` argument can be used to override default values in dataclass -types. - - -.. warning:: - - We advise against mutation of configuration objects from a dataclass's - :code:`__post_init__` method [#f1]_. In the example below, - :code:`__post_init__` would be called twice: once for the :code:`Args()` - object provided as a default value and another time for the :code:`Args()` - objected instantiated by :func:`tyro.cli()`. This can cause confusing - behavior! Instead, we show below one example of how derived fields can be - defined immutably. - - .. [#f1] Official Python docs for ``__post_init__`` can be found `here `_. - - -Usage: -`python ./03_dataclasses_defaults.py --help` -`python ./03_dataclasses_defaults.py --field2 3` -`python ./03_dataclasses_defaults.py --field1 hello --field2 5` -""" - -import dataclasses - -import tyro - - -@dataclasses.dataclass -class Args: - """Description. - This should show up in the helptext!""" - - field1: str - """A string field.""" - - field2: int = 3 - """A numeric field, with a default value.""" - - @property - def derived_field(self) -> str: - return ", ".join([self.field1] * self.field2) - - -if __name__ == "__main__": - args = tyro.cli( - Args, - default=Args( - field1="default string", - field2=tyro.MISSING, - ), - ) - print(args.derived_field) diff --git a/examples/01_basics/03_multivalue.py b/examples/01_basics/03_multivalue.py new file mode 100644 index 000000000..10d05b3b8 --- /dev/null +++ b/examples/01_basics/03_multivalue.py @@ -0,0 +1,39 @@ +"""Multi-value Arguments + +Arguments of both fixed and variable lengths can be annotated with standard +Python collection types. For Python 3.7 and 3.8, we can use either ``from +__future__ import annotations`` to support ``list[T]`` and ``tuple[T]``, +or the older :py:class:`typing.List` and :py:data:`typing.Tuple`. + +Usage: + + # To print help: + python ./03_multivalue.py --help + + # We can override arguments: + python ./03_multivalue.py --source-paths ./data --dimensions 16 16 + python ./03_multivalue.py --source-paths ./data1 ./data2 +""" + +import pathlib +from dataclasses import dataclass +from pprint import pprint + +import tyro + + +@dataclass +class Config: + # Example of a variable-length tuple. `list[T]`, `set[T]`, + # `dict[K, V]`, etc are supported as well. + source_paths: tuple[pathlib.Path, ...] + """This can be multiple!""" + + # Fixed-length tuples are also okay. + dimensions: tuple[int, int] = (32, 32) + """Height and width.""" + + +if __name__ == "__main__": + config = tyro.cli(Config) + pprint(config) diff --git a/examples/04_additional/04_classes.py b/examples/01_basics/04_classes.py similarity index 86% rename from examples/04_additional/04_classes.py rename to examples/01_basics/04_classes.py index 5dbae4639..5c977e14d 100644 --- a/examples/04_additional/04_classes.py +++ b/examples/01_basics/04_classes.py @@ -4,8 +4,9 @@ constructors of standard Python classes. Usage: -`python ./04_classes.py --help` -`python ./04_classes.py --field1 hello --field2 7` + + python ./04_classes.py --help + python ./04_classes.py --field1 hello --field2 7 """ import tyro diff --git a/examples/01_basics/04_collections.py b/examples/01_basics/04_collections.py deleted file mode 100644 index a3f3b836e..000000000 --- a/examples/01_basics/04_collections.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Multi-value Arguments - -Arguments of both fixed and variable lengths can be annotated with standard -Python collection types. For Python 3.7 and 3.8, we can use either :code:`from -__future__ import annotations` to support :code:`list[T]` and :code:`tuple[T]`, -or the older API :code:`typing.List[T]` and :code:`typing.Tuple[T1, T2]`. - -Usage: -`python ./03_collections.py --help` -`python ./03_collections.py --dataset-sources ./data --image-dimensions 16 16` -`python ./03_collections.py --dataset-sources ./data` -""" - -import dataclasses -import pathlib - -import tyro - - -@dataclasses.dataclass(frozen=True) -class TrainConfig: - # Example of a variable-length tuple. `list[T]`, `set[T]`, - # `dict[K, V]`, etc are supported as well. - dataset_sources: tuple[pathlib.Path, ...] - """Paths to load training data from. This can be multiple!""" - - # Fixed-length tuples are also okay. - image_dimensions: tuple[int, int] = (32, 32) - """Height and width of some image data.""" - - -if __name__ == "__main__": - config = tyro.cli(TrainConfig) - print(config) diff --git a/examples/01_basics/05_flags.py b/examples/01_basics/04_flags.py similarity index 70% rename from examples/01_basics/05_flags.py rename to examples/01_basics/04_flags.py index 11c07b8bf..ff5f45ca6 100644 --- a/examples/01_basics/05_flags.py +++ b/examples/01_basics/04_flags.py @@ -6,18 +6,20 @@ To turn off conversion, see :class:`tyro.conf.FlagConversionOff`. Usage: -`python ./05_flags.py --help` -`python ./05_flags.py --boolean True` -`python ./05_flags.py --boolean False --flag-a` -`python ./05_flags.py --boolean False --no-flag-b` + + python ./04_flags.py --help + python ./04_flags.py --boolean True + python ./04_flags.py --boolean False --flag-a + python ./04_flags.py --boolean False --no-flag-b """ -import dataclasses +from dataclasses import dataclass +from pprint import pprint import tyro -@dataclasses.dataclass +@dataclass class Args: # Boolean. This expects an explicit "True" or "False". boolean: bool @@ -34,4 +36,4 @@ class Args: if __name__ == "__main__": args = tyro.cli(Args) - print(args) + pprint(args) diff --git a/examples/01_basics/06_literals.py b/examples/01_basics/05_choices.py similarity index 61% rename from examples/01_basics/06_literals.py rename to examples/01_basics/05_choices.py index 5f3017d85..602eaef9a 100644 --- a/examples/01_basics/06_literals.py +++ b/examples/01_basics/05_choices.py @@ -3,25 +3,29 @@ :code:`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices. Usage: -`python ./06_literals.py --help` + + python ./05_choices.py --help + python ./05_choices.py --string red + python ./05_choices.py --string blue """ import dataclasses +from pprint import pprint from typing import Literal import tyro -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass class Args: # We can use Literal[] to restrict the set of allowable inputs, for example, over # a set of strings. - strings: Literal["red", "green"] = "red" + string: Literal["red", "green"] = "red" # Integers also work. (as well as booleans, enums, etc) - numbers: Literal[0, 1, 2] = 0 + number: Literal[0, 1, 2] = 0 if __name__ == "__main__": args = tyro.cli(Args) - print(args) + pprint(args) diff --git a/examples/01_basics/06_enums.py b/examples/01_basics/06_enums.py new file mode 100644 index 000000000..749e26bf4 --- /dev/null +++ b/examples/01_basics/06_enums.py @@ -0,0 +1,36 @@ +"""Enums + +In addition to literals, enums can also be used to provide a fixed set of +choices. + +Usage: + + python ./06_enums.py --help + python ./06_enums.py --color RED + python ./06_enums.py --color BLUE --opacity 0.75 +""" + +import enum +from dataclasses import dataclass +from pprint import pprint + +import tyro + + +class Color(enum.Enum): + RED = enum.auto() + BLUE = enum.auto() + + +@dataclass +class Config: + color: Color = Color.RED + """Color argument.""" + + opacity: float = 0.5 + """Opacity argument.""" + + +if __name__ == "__main__": + config = tyro.cli(Config) + pprint(config) diff --git a/examples/01_basics/07_unions.py b/examples/01_basics/07_unions.py index f99ffb04d..3289d0e05 100644 --- a/examples/01_basics/07_unions.py +++ b/examples/01_basics/07_unions.py @@ -4,11 +4,17 @@ multiple types. Usage: -`python ./07_unions.py --help` + + python ./07_unions.py --help + python ./07_unions.py --union-over-types 3 + python ./07_unions.py --union-over-types three + python ./07_unions.py --integer None + python ./07_unions.py --integer 0 """ import dataclasses import enum +from pprint import pprint from typing import Literal, Optional import tyro @@ -41,4 +47,4 @@ class Args: if __name__ == "__main__": args = tyro.cli(Args) - print(args) + pprint(args) diff --git a/examples/01_basics/08_enums.py b/examples/01_basics/08_enums.py deleted file mode 100644 index 0fa47eed4..000000000 --- a/examples/01_basics/08_enums.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Enums - -In addition to literals, enums can also be used to provide a fixed set of -choices. - -Usage: -`python ./04_enums.py --help` -`python ./04_enums.py --optimizer-type SGD` -`python ./04_enums.py --optimizer-type ADAM --learning-rate 3e-4` -""" - -import dataclasses -import enum - -import tyro - - -class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - -@dataclasses.dataclass(frozen=True) -class TrainConfig: - # Enums are handled seamlessly. - optimizer_type: OptimizerType = OptimizerType.ADAM - """Gradient-based optimizer to use.""" - - learning_rate: float = 1e-4 - """Learning rate for optimizer.""" - - -if __name__ == "__main__": - config = tyro.cli(TrainConfig) - print(config) diff --git a/examples/01_basics/08_positional.py b/examples/01_basics/08_positional.py new file mode 100644 index 000000000..6380b90b0 --- /dev/null +++ b/examples/01_basics/08_positional.py @@ -0,0 +1,39 @@ +"""Positional Arguments + +Positional-only arguments in functions are converted to positional CLI arguments. + +For more general positional arguments, see :class:`tyro.conf.Positional`. + +Usage: + + python 08_positional.py --help + python 08_positional.py ./a ./b + python 08_positional.py ./test1 ./test2 --verbose +""" + +from __future__ import annotations + +import pathlib + +import tyro + + +def main( + source: pathlib.Path, + dest: pathlib.Path, + /, # Mark the end of positional arguments. + verbose: bool = False, +) -> None: + """Command-line interface defined using a function signature. Note that this + docstring is parsed to generate helptext. + + Args: + source: Source path. + dest: Destination path. + verbose: Explain what is being done. + """ + print(f"{source=}\n{dest=}\n{verbose=}") + + +if __name__ == "__main__": + tyro.cli(main) diff --git a/examples/04_additional/07_conf.py b/examples/01_basics/09_conf.py similarity index 81% rename from examples/04_additional/07_conf.py rename to examples/01_basics/09_conf.py index 479a888a9..d87f88095 100644 --- a/examples/04_additional/07_conf.py +++ b/examples/01_basics/09_conf.py @@ -1,13 +1,15 @@ """Configuration via typing.Annotated[] -The :mod:`tyro.conf` module contains utilities that can be used to configure -command-line interfaces beyond what is expressible via static type annotations. +The :mod:`tyro.conf` module contains utilities that can be used in conjunction +with :py:data:`typing.Annotated` to configure command-line interfaces beyond +what is expressible via static type annotations. Features here are supported, but generally unnecessary and should be used sparingly. Usage: -`python ./06_conf.py --help` -`python ./06_conf.py 5 --boolean True` + + python ./09_conf.py --help + python ./09_conf.py 5 --boolean True """ import dataclasses diff --git a/examples/01_basics/10_aliases.py b/examples/01_basics/10_aliases.py new file mode 100644 index 000000000..3c534be98 --- /dev/null +++ b/examples/01_basics/10_aliases.py @@ -0,0 +1,25 @@ +"""Argument Aliases + +:func:`tyro.conf.arg()` can be used to attach aliases to arguments. + +Usage: + + python ./10_aliases.py --help + python ./10_aliases.py --branch main + python ./10_aliases.py -b main +""" + +from typing import Annotated + +import tyro + + +def checkout( + branch: Annotated[str, tyro.conf.arg(aliases=["-b"])], +) -> None: + """Check out a branch.""" + print(f"{branch=}") + + +if __name__ == "__main__": + tyro.cli(checkout) diff --git a/examples/04_additional/12_type_statement.py b/examples/01_basics/11_type_aliases_py312.py similarity index 89% rename from examples/04_additional/12_type_statement.py rename to examples/01_basics/11_type_aliases_py312.py index fa8140297..c8daad52a 100644 --- a/examples/04_additional/12_type_statement.py +++ b/examples/01_basics/11_type_aliases_py312.py @@ -1,12 +1,13 @@ # mypy: ignore-errors # # PEP 695 isn't yet supported in mypy. (April 4, 2024) -"""Type Aliases (Python 3.12+) +"""Type Aliases (3.12+) In Python 3.12, the :code:`type` statement is introduced to create type aliases. Usage: -`python ./12_type_statement.py --help` + + python ./11_type_aliases_py312.py --help """ import dataclasses diff --git a/examples/04_additional/13_counters.py b/examples/01_basics/12_counters.py similarity index 81% rename from examples/04_additional/13_counters.py rename to examples/01_basics/12_counters.py index 83562b1ac..7d9b6374c 100644 --- a/examples/04_additional/13_counters.py +++ b/examples/01_basics/12_counters.py @@ -3,10 +3,11 @@ Repeatable 'counter' arguments can be specified via :data:`tyro.conf.UseCounterAction`. Usage: -`python ./13_counters.py --help` -`python ./13_counters.py --verbosity` -`python ./13_counters.py --verbosity --verbosity` -`python ./13_counters.py -vvv` + + python ./12_counters.py --help + python ./12_counters.py --verbosity + python ./12_counters.py --verbosity --verbosity + python ./12_counters.py -vvv """ from typing_extensions import Annotated diff --git a/examples/01_basics/README.rst b/examples/01_basics/README.rst new file mode 100644 index 000000000..bc3b1d23f --- /dev/null +++ b/examples/01_basics/README.rst @@ -0,0 +1,5 @@ +Basics +====== + +In these examples, we show basic examples of using :func:`tyro.cli`: functions, +dataclasses, supported annotations, and configuration. diff --git a/examples/02_nested_structures/01_nesting.py b/examples/02_nested_structures/01_nesting.py new file mode 100644 index 000000000..32128d296 --- /dev/null +++ b/examples/02_nested_structures/01_nesting.py @@ -0,0 +1,35 @@ +"""Nested Dataclasses + +Structures (typically :py:func:`dataclasses.dataclass`) can be nested to build hierarchical configuration +objects. This helps with modularity and grouping in larger projects. + +Usage: + + python ./01_nesting.py --help + python ./01_nesting.py --opt.learning-rate 1e-3 + python ./01_nesting.py --seed 4 +""" + +import dataclasses + +import tyro + + +@dataclasses.dataclass +class OptimizerConfig: + learning_rate: float = 3e-4 + weight_decay: float = 1e-2 + + +@dataclasses.dataclass +class Config: + # Optimizer options. + opt: OptimizerConfig + + # Random seed. + seed: int = 0 + + +if __name__ == "__main__": + config = tyro.cli(Config) + print(dataclasses.asdict(config)) diff --git a/examples/02_nested_structures/02_nesting_in_func.py b/examples/02_nested_structures/02_nesting_in_func.py new file mode 100644 index 000000000..da0c81f09 --- /dev/null +++ b/examples/02_nested_structures/02_nesting_in_func.py @@ -0,0 +1,48 @@ +"""Structures as Function Arguments + +Structures can also be used as input to functions. + +Usage: + + python ./02_nesting_in_func.py --help + python ./02_nesting_in_func.py --out-dir /tmp/test1 + python ./02_nesting_in_func.py --out-dir /tmp/test2 --config.seed 4 +""" + +import dataclasses +import pathlib + +import tyro + + +@dataclasses.dataclass +class OptimizerConfig: + learning_rate: float = 3e-4 + weight_decay: float = 1e-2 + + +@dataclasses.dataclass +class Config: + # Optimizer options. + optimizer: OptimizerConfig + + # Random seed. + seed: int = 0 + + +def train( + out_dir: pathlib.Path, + config: Config, +) -> None: + """Train a model. + + Args: + out_dir: Where to save logs and checkpoints. + config: Experiment configuration. + """ + print(f"Saving to: {out_dir}") + print(f"Config: f{config}") + + +if __name__ == "__main__": + tyro.cli(train) diff --git a/examples/02_nested_structures/03_nesting_containers.py b/examples/02_nested_structures/03_nesting_containers.py new file mode 100644 index 000000000..9f3ecd08d --- /dev/null +++ b/examples/02_nested_structures/03_nesting_containers.py @@ -0,0 +1,44 @@ +"""Nesting in Containers + +Structures can be nested inside of standard containers. + +.. warning:: + + When placing structures inside of containers like lists or tuples, the + length of the container must be inferrable from the annotation or default + value. + + +Usage: + + python ./03_nesting_containers.py --help +""" + +import dataclasses + +import tyro + + +@dataclasses.dataclass +class RGB: + r: int + g: int + b: int + + +@dataclasses.dataclass +class Args: + color_tuple: tuple[RGB, RGB] + color_dict: dict[str, RGB] = dataclasses.field( + # We can't use mutable values as defaults directly. + default_factory={ + "red": RGB(255, 0, 0), + "green": RGB(0, 255, 0), + "blue": RGB(0, 0, 255), + }.copy + ) + + +if __name__ == "__main__": + args = tyro.cli(Args) + print(args) diff --git a/examples/04_additional/02_dictionaries.py b/examples/02_nested_structures/04_dictionaries.py similarity index 82% rename from examples/04_additional/02_dictionaries.py rename to examples/02_nested_structures/04_dictionaries.py index 9b0bac6f0..b1fa51086 100644 --- a/examples/04_additional/02_dictionaries.py +++ b/examples/02_nested_structures/04_dictionaries.py @@ -1,15 +1,16 @@ """Dictionaries and TypedDict -Dictionary inputs can be specified using either a standard `Dict[K, V]` +Dictionary inputs can be specified using either a standard ``dict[K, V]`` annotation, or a :code:`TypedDict` subclass. For configuring :code:`TypedDict`, we also support :code:`total={True/False}`, :code:`typing.Required`, and :code:`typing.NotRequired`. See the `Python docs `_ for all :code:`TypedDict` features. Usage: -`python ./02_dictionaries.py --help` -`python ./02_dictionaries.py --typed-dict-a.learning-rate 3e-4 --typed-dict-b.betas 0.9 0.999` -`python ./02_dictionaries.py --typed-dict-b.betas 0.9 0.999` + + python ./04_dictionaries.py --help + python ./04_dictionaries.py --typed-dict-a.learning-rate 3e-4 --typed-dict-b.betas 0.9 0.999 + python ./04_dictionaries.py --typed-dict-b.betas 0.9 0.999 """ from typing import TypedDict diff --git a/examples/04_additional/03_tuples.py b/examples/02_nested_structures/05_tuples.py similarity index 71% rename from examples/04_additional/03_tuples.py rename to examples/02_nested_structures/05_tuples.py index ef40280c8..d9e839813 100644 --- a/examples/04_additional/03_tuples.py +++ b/examples/02_nested_structures/05_tuples.py @@ -1,12 +1,13 @@ -"""Tuples +"""Tuples and NamedTuple Example using :func:`tyro.cli()` to instantiate tuple types. :code:`tuple`, -:code:`typing.Tuple`, and :code:`NamedTuple` are all supported. +:py:data:`typing.Tuple`, and :py:class:`typing.NamedTuple` are all supported. Usage: -`python ./03_tuples.py --help` -`python ./03_tuples.py --color 127 127 127` -`python ./03_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0` + + python ./05_tuples.py --help + python ./05_tuples.py --color 127 127 127 + python ./05_tuples.py --two-colors.1.r 127 --two-colors.1.g 0 --two-colors.1.b 0 """ from typing import NamedTuple diff --git a/examples/04_additional/08_pydantic.py b/examples/02_nested_structures/06_pydantic.py similarity index 77% rename from examples/04_additional/08_pydantic.py rename to examples/02_nested_structures/06_pydantic.py index e9c5ee457..740eeac42 100644 --- a/examples/04_additional/08_pydantic.py +++ b/examples/02_nested_structures/06_pydantic.py @@ -4,9 +4,10 @@ `Pydantic `_ models. Usage: -`python ./08_pydantic.py --help` -`python ./08_pydantic.py --field1 hello` -`python ./08_pydantic.py --field1 hello --field2 5` + + python ./06_pydantic.py --help + python ./06_pydantic.py --field1 hello + python ./06_pydantic.py --field1 hello --field2 5 """ from pydantic import BaseModel, Field diff --git a/examples/04_additional/09_attrs.py b/examples/02_nested_structures/07_attrs.py similarity index 79% rename from examples/04_additional/09_attrs.py rename to examples/02_nested_structures/07_attrs.py index effe864e9..30d93488e 100644 --- a/examples/04_additional/09_attrs.py +++ b/examples/02_nested_structures/07_attrs.py @@ -4,9 +4,10 @@ `attrs `_ classes. Usage: -`python ./09_attrs.py --help` -`python ./09_attrs.py --field1 hello` -`python ./09_attrs.py --field1 hello --field2 5` + + python ./07_attrs.py --help + python ./07_attrs.py --field1 hello + python ./07_attrs.py --field1 hello --field2 5 """ import attr diff --git a/examples/02_nested_structures/README.rst b/examples/02_nested_structures/README.rst new file mode 100644 index 000000000..8ec6ad9b9 --- /dev/null +++ b/examples/02_nested_structures/README.rst @@ -0,0 +1,6 @@ +Nested Structures +================= + +In these examples, we show how :func:`tyro.cli` can be used to instantiate +nested structures. This can enable modular, reusable, and composable CLI +interfaces. diff --git a/examples/02_nesting/01_nesting.py b/examples/02_nesting/01_nesting.py deleted file mode 100644 index b6e673056..000000000 --- a/examples/02_nesting/01_nesting.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Hierarchical Configs - -Structures (typically dataclasses) can be nested to build hierarchical configuration -objects. This helps with modularity and grouping in larger projects. - -Usage: -`python ./01_nesting.py --help` -`python ./01_nesting.py --out-dir . --config.optimizer.algorithm SGD` -`python ./01_nesting.py --out-dir . --restore-checkpoint` -""" - -import dataclasses -import enum -import pathlib - -import tyro - - -class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - -@dataclasses.dataclass -class OptimizerConfig: - # Gradient-based optimizer to use. - algorithm: OptimizerType = OptimizerType.ADAM - - # Learning rate to use. - learning_rate: float = 3e-4 - - # Coefficient for L2 regularization. - weight_decay: float = 1e-2 - - -@dataclasses.dataclass -class ExperimentConfig: - # Various configurable options for our optimizer. - optimizer: OptimizerConfig - - # Batch size. - batch_size: int = 32 - - # Total number of training steps. - train_steps: int = 100_000 - - # Random seed. This is helpful for making sure that our experiments are all - # reproducible! - seed: int = 0 - - -def train( - out_dir: pathlib.Path, - config: ExperimentConfig, - restore_checkpoint: bool = False, - checkpoint_interval: int = 1000, -) -> None: - """Train a model. - - Args: - out_dir: Where to save logs and checkpoints. - config: Experiment configuration. - restore_checkpoint: Set to restore an existing checkpoint. - checkpoint_interval: Training steps between each checkpoint save. - """ - print(f"{out_dir=}, {restore_checkpoint=}, {checkpoint_interval=}") - print() - print(f"{config=}") - - -if __name__ == "__main__": - tyro.cli(train) diff --git a/examples/02_nesting/02_subcommands.py b/examples/02_nesting/02_subcommands.py deleted file mode 100644 index 5ccc26610..000000000 --- a/examples/02_nesting/02_subcommands.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Subcommands - -Unions over nested types (classes or dataclasses) are populated using subcommands. - -For configuring subcommands beyond what can be expressed with type annotations, see -:func:`tyro.conf.subcommand()`. - -Usage: -`python ./02_subcommands.py --help` -`python ./02_subcommands.py cmd:commit --help` -`python ./02_subcommands.py cmd:commit --cmd.message hello --cmd.all` -`python ./02_subcommands.py cmd:checkout --help` -`python ./02_subcommands.py cmd:checkout --cmd.branch main` -""" - -from __future__ import annotations - -import dataclasses - -import tyro - - -@dataclasses.dataclass(frozen=True) -class Checkout: - """Checkout a branch.""" - - branch: str - - -@dataclasses.dataclass(frozen=True) -class Commit: - """Commit changes.""" - - message: str - all: bool = False - - -def main(cmd: Checkout | Commit) -> None: - print(cmd) - - -if __name__ == "__main__": - # Note that we can also pass `Checkout | Command` directly into - # `tyro.cli()`; this is understood by tyro and pyright, but unfortunately not by - # mypy. - tyro.cli(main) diff --git a/examples/02_nesting/04_nesting_in_containers.py b/examples/02_nesting/04_nesting_in_containers.py deleted file mode 100644 index 18892696d..000000000 --- a/examples/02_nesting/04_nesting_in_containers.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Nesting in Containers - -Structures can be nested inside of standard containers. - -Note that lengths must be inferable, either via a fixed-length tuple annotation or by -parsing default values. - - -Usage: -`python ./04_nesting_in_containers.py.py --help` -""" - -import dataclasses - -import tyro - - -class Color: - pass - - -@dataclasses.dataclass -class RGB(Color): - r: int - g: int - b: int - - -@dataclasses.dataclass -class HSL(Color): - h: int - s: int - l: int - - -@dataclasses.dataclass -class Args: - # Example of specifying nested structures via a fixed-length tuple. - color_tuple: tuple[RGB, HSL] - - # Examples of nested structures in variable-length containers. These need a default - # provided for length inference; we don't currently support specifying dynamic - # container lengths directly from the commandline. - color_tuple_alt: tuple[Color, ...] = ( - RGB(255, 0, 0), - HSL(0, 255, 0), - ) - color_map: dict[str, RGB] = dataclasses.field( - # We can't use mutable values as defaults directly. - default_factory={ - "red": RGB(255, 0, 0), - "green": RGB(0, 255, 0), - "blue": RGB(0, 0, 255), - }.copy - ) - - -if __name__ == "__main__": - args = tyro.cli(Args) - print(args) diff --git a/examples/03_subcommands/01_subcommands.py b/examples/03_subcommands/01_subcommands.py new file mode 100644 index 000000000..0f184d347 --- /dev/null +++ b/examples/03_subcommands/01_subcommands.py @@ -0,0 +1,53 @@ +# mypy: ignore-errors +# +# Passing a Union type directly to tyro.cli() doesn't type-check correctly in +# mypy. This will be fixed by `typing.TypeForm`: https://peps.python.org/pep-0747/ +"""Subcommands are Unions + +All of :mod:`tyro`'s subcommand features are built using unions over struct +types (typically dataclasses). Subcommands are used to choose between types in +the union; arguments are then populated from the chosen type. + +.. note:: + + For configuring subcommands beyond what can be expressed with type annotations, see + :func:`tyro.conf.subcommand()`. + +Usage: + + # Print the helptext. This will show the available subcommands: + python ./01_subcommands.py --help + + # The `commit` subcommand: + python ./01_subcommands.py commit --help + python ./01_subcommands.py commit --message hello + + # The `checkout` subcommand: + python ./01_subcommands.py checkout --help + python ./01_subcommands.py checkout --branch main +""" + +from __future__ import annotations + +import dataclasses + +import tyro + + +@dataclasses.dataclass(frozen=True) +class Checkout: + """Checkout a branch.""" + + branch: str + + +@dataclasses.dataclass(frozen=True) +class Commit: + """Commit changes.""" + + message: str + + +if __name__ == "__main__": + cmd = tyro.cli(Checkout | Commit) + print(cmd) diff --git a/examples/03_subcommands/02_subcommands_in_func.py b/examples/03_subcommands/02_subcommands_in_func.py new file mode 100644 index 000000000..727d336eb --- /dev/null +++ b/examples/03_subcommands/02_subcommands_in_func.py @@ -0,0 +1,60 @@ +"""Subcommands as Function Arguments + +A subcommand will be created for each input annotated with a union over +struct types. + +.. note:: + + To prevent :func:`tyro.cli()` from converting a Union type into a subcommand, + use :class:`tyro.conf.AvoidSubcommands`. + +.. note:: + + Argument ordering for subcommands can be tricky. In the example below, + ``--shared-arg`` must always come *before* the subcommand. As an option for + alleviating this, see :class:`tyro.conf.ConsolidateSubcommandArgs`. + + +Usage: + + # Print the helptext. This will show the available subcommands: + python ./02_subcommands_in_func.py --help + + # Using the default subcommand: + python ./02_subcommands_in_func.py --shared-arg 100 + + # Choosing a different subcommand: + python ./02_subcommands_in_func.py --shared-arg 100 cmd:commit --cmd.message Hello! +""" + +from __future__ import annotations + +import dataclasses + +import tyro + + +@dataclasses.dataclass(frozen=True) +class Checkout: + """Checkout a branch.""" + + branch: str + + +@dataclasses.dataclass(frozen=True) +class Commit: + """Commit changes.""" + + message: str + + +def main( + shared_arg: int, + cmd: Checkout | Commit = Checkout(branch="default"), +): + print(f"{shared_arg=}") + print(cmd) + + +if __name__ == "__main__": + tyro.cli(main) diff --git a/examples/02_nesting/03_multiple_subcommands.py b/examples/03_subcommands/03_multiple_subcommands.py similarity index 68% rename from examples/02_nesting/03_multiple_subcommands.py rename to examples/03_subcommands/03_multiple_subcommands.py index 561232b54..86cfcd5c8 100644 --- a/examples/02_nesting/03_multiple_subcommands.py +++ b/examples/03_subcommands/03_multiple_subcommands.py @@ -1,12 +1,15 @@ """Sequenced Subcommands -Multiple unions over nested types are populated using a series of subcommands. +Multiple unions over struct types are populated using a series of subcommands. Usage: -`python ./03_multiple_subcommands.py --help` -`python ./03_multiple_subcommands.py dataset:mnist --help` -`python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --help` -`python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 --dataset.binary` + + # Note that we apply the :class:`tyro.conf.ConsolidateSubcommandArgs` flag. + # This pushes all arguments to the end of the command: + python ./03_multiple_subcommands.py --help + python ./03_multiple_subcommands.py dataset:mnist --help + python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --help + python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --optimizer.learning-rate 3e-4 --dataset.binary """ from __future__ import annotations diff --git a/examples/04_additional/15_decorator_subcommands.py b/examples/03_subcommands/04_decorator_subcommands.py similarity index 64% rename from examples/04_additional/15_decorator_subcommands.py rename to examples/03_subcommands/04_decorator_subcommands.py index e9fc77ccb..297a6ab99 100644 --- a/examples/04_additional/15_decorator_subcommands.py +++ b/examples/03_subcommands/04_decorator_subcommands.py @@ -4,12 +4,13 @@ subcommands, which is inspired by `click `_. Usage: -`python my_script.py --help` -`python my_script.py greet --help` -`python my_script.py greet --name Alice` -`python my_script.py greet --name Bob --loud` -`python my_script.py addition --help` -`python my_script.py addition --a 5 --b 3` + + python 04_decorator_subcommands.py --help + python 04_decorator_subcommands.py greet --help + python 04_decorator_subcommands.py greet --name Alice + python 04_decorator_subcommands.py greet --name Bob --loud + python 04_decorator_subcommands.py addition --help + python 04_decorator_subcommands.py addition --a 5 --b 3 """ from tyro.extras import SubcommandApp diff --git a/examples/02_nesting/05_subcommands_func.py b/examples/03_subcommands/05_subcommands_func.py similarity index 72% rename from examples/02_nesting/05_subcommands_func.py rename to examples/03_subcommands/05_subcommands_func.py index ca2f202f9..a65c0bb53 100644 --- a/examples/02_nesting/05_subcommands_func.py +++ b/examples/03_subcommands/05_subcommands_func.py @@ -7,11 +7,12 @@ Usage: -`python ./05_subcommands_func.py --help` -`python ./05_subcommands_func.py commit --help` -`python ./05_subcommands_func.py commit --message hello --all` -`python ./05_subcommands_func.py checkout --help` -`python ./05_subcommands_func.py checkout --branch main` + + python ./05_subcommands_func.py --help + python ./05_subcommands_func.py commit --help + python ./05_subcommands_func.py commit --message hello --all + python ./05_subcommands_func.py checkout --help + python ./05_subcommands_func.py checkout --branch main """ import tyro diff --git a/examples/03_subcommands/README.rst b/examples/03_subcommands/README.rst new file mode 100644 index 000000000..156c89fe8 --- /dev/null +++ b/examples/03_subcommands/README.rst @@ -0,0 +1,5 @@ +Subcommands +=========== + +In these examples, we show how :func:`tyro.cli` can be used to create CLI +interfaces with subcommands. diff --git a/examples/04_additional/01_positional_args.py b/examples/04_additional/01_positional_args.py deleted file mode 100644 index 66048cd53..000000000 --- a/examples/04_additional/01_positional_args.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Positional Arguments - -Positional-only arguments in functions are converted to positional CLI arguments. - -For more general positional arguments, see :class:`tyro.conf.Positional`. - -Usage: -`python ./01_positional_args.py --help` -`python ./01_positional_args.py ./a ./b --optimizer.learning-rate 1e-5` -""" - -from __future__ import annotations - -import dataclasses -import enum -import pathlib -from typing import Tuple - -import tyro - - -def main( - source: pathlib.Path, - dest: pathlib.Path, - /, # Mark the end of positional arguments. - optimizer: OptimizerConfig, - force: bool = False, - verbose: bool = False, - background_rgb: Tuple[float, float, float] = (1.0, 0.0, 0.0), -) -> None: - """Command-line interface defined using a function signature. Note that this - docstring is parsed to generate helptext. - - Args: - source: Source path. - dest: Destination path. - optimizer: Configuration for our optimizer object. - force: Do not prompt before overwriting. - verbose: Explain what is being done. - background_rgb: Background color. Red by default. - """ - print(f"{source=}\n{dest=}\n{optimizer=}\n{force=}\n{verbose=}\n{background_rgb=}") - - -class OptimizerType(enum.Enum): - ADAM = enum.auto() - SGD = enum.auto() - - -@dataclasses.dataclass(frozen=True) -class OptimizerConfig: - algorithm: OptimizerType = OptimizerType.ADAM - """Gradient-based optimizer to use.""" - - learning_rate: float = 3e-4 - """Learning rate to use.""" - - weight_decay: float = 1e-2 - """Coefficient for L2 regularization.""" - - -if __name__ == "__main__": - tyro.cli(main) diff --git a/examples/04_additional/11_aliases.py b/examples/04_additional/11_aliases.py deleted file mode 100644 index 5ee33de9c..000000000 --- a/examples/04_additional/11_aliases.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Argument Aliases - -:func:`tyro.conf.arg()` can be used to attach aliases to arguments. - -Usage: -`python ./11_aliases.py --help` -`python ./11_aliases.py commit --help` -`python ./11_aliases.py commit --message hello --all` -`python ./11_aliases.py commit -m hello -a` -`python ./11_aliases.py checkout --help` -`python ./11_aliases.py checkout --branch main` -`python ./11_aliases.py checkout -b main` -""" - -from typing_extensions import Annotated - -import tyro - - -def checkout( - branch: Annotated[str, tyro.conf.arg(aliases=["-b"])], -) -> None: - """Check out a branch.""" - print(f"{branch=}") - - -def commit( - message: Annotated[str, tyro.conf.arg(aliases=["-m"])], - all: Annotated[bool, tyro.conf.arg(aliases=["-a"])] = False, -) -> None: - """Make a commit.""" - print(f"{message=} {all=}") - - -if __name__ == "__main__": - tyro.extras.subcommand_cli_from_dict( - { - "checkout": checkout, - "commit": commit, - } - ) diff --git a/examples/04_overriding_configs/01_dataclasses_defaults.py b/examples/04_overriding_configs/01_dataclasses_defaults.py new file mode 100644 index 000000000..c52aa66fc --- /dev/null +++ b/examples/04_overriding_configs/01_dataclasses_defaults.py @@ -0,0 +1,56 @@ +"""Dataclasses + Defaults + +The :code:`default=` argument can be used to override default values in dataclass +types. + + +.. note:: + + When ``default=`` is used, we advise against mutation of configuration + objects from a dataclass's :code:`__post_init__` method [#f1]_. In the + example below, :code:`__post_init__` would be called twice: once for the + :code:`Args()` object provided as a default value and another time for the + :code:`Args()` objected instantiated by :func:`tyro.cli()`. This can cause + confusing behavior! Instead, we show below one example of how derived + fields can be defined immutably. + + .. [#f1] Official Python docs for ``__post_init__`` can be found `here `_. + + +Usage: + + python ./01_dataclasses_defaults.py --help + python ./01_dataclasses_defaults.py --reps 3 + python ./01_dataclasses_defaults.py --string hello --reps 5 +""" + +import dataclasses + +import tyro + + +@dataclasses.dataclass +class Args: + """Description. + This should show up in the helptext!""" + + string: str + """A string field.""" + + reps: int = 3 + """A numeric field, with a default value.""" + + @property + def derived_field(self) -> str: + return ", ".join([self.string] * self.reps) + + +if __name__ == "__main__": + args = tyro.cli( + Args, + default=Args( + string="default string", + reps=tyro.MISSING, + ), + ) + print(args.derived_field) diff --git a/examples/03_config_systems/02_overriding_yaml.py b/examples/04_overriding_configs/02_overriding_yaml.py similarity index 72% rename from examples/03_config_systems/02_overriding_yaml.py rename to examples/04_overriding_configs/02_overriding_yaml.py index 5db0cfd86..6ef7bcce7 100644 --- a/examples/03_config_systems/02_overriding_yaml.py +++ b/examples/04_overriding_configs/02_overriding_yaml.py @@ -1,12 +1,19 @@ """Overriding YAML Configs +:mod:`tyro` understands a wide range of data structures, including standard dictionaries +and lists. + If you have a library of existing YAML files that you want to use, `tyro` can -be used to override values in them. We generally recommend dataclass configs -for new projects. +help override values within them. + +.. note:: + + We recommend dataclass configs for new projects. Usage: -`python ./02_overriding_yaml.py --help` -`python ./02_overriding_yaml.py --training.checkpoint-steps 300 1000 9000` + + python ./02_overriding_yaml.py --help + python ./02_overriding_yaml.py --training.checkpoint-steps 300 1000 9000 """ import yaml diff --git a/examples/03_config_systems/01_base_configs.py b/examples/04_overriding_configs/03_choosing_base_configs.py similarity index 63% rename from examples/03_config_systems/01_base_configs.py rename to examples/04_overriding_configs/03_choosing_base_configs.py index 50ee71fa4..fb26e0689 100644 --- a/examples/03_config_systems/01_base_configs.py +++ b/examples/04_overriding_configs/03_choosing_base_configs.py @@ -1,19 +1,32 @@ -"""Base Configurations +"""Choosing Base Configs -We can integrate `tyro` into common configuration patterns: here, we select -one of multiple possible base configurations, create a subcommand for each one, and then -use the CLI to either override (existing) or fill in (missing) values. +One common pattern is to have a set of "base" configurations, which can be +selected from and then overridden. -The helper function used here, :func:`tyro.extras.overridable_config_cli()`, is a -lightweight wrapper over :func:`tyro.cli()`. +This is often implemented with a set of configuration files (e.g., YAML files). +With :mod:`tyro`, we can instead define each base configuration as a separate +Python object. + +After creating the base configurations, we can use the CLI to select one of +them and then override (existing) or fill in (missing) values. + +The helper function used here, :func:`tyro.extras.overridable_config_cli()`, is +a lightweight wrapper over :func:`tyro.cli()` and its Union-based subcommand +syntax. Usage: -`python ./01_base_configs.py --help` -`python ./01_base_configs.py small --help` -`python ./01_base_configs.py small --seed 94720` -`python ./01_base_configs.py big --help` -`python ./01_base_configs.py big --seed 94720` + + # Overall helptext: + python ./03_choosing_base_configs.py --help + + # The "small" subcommand: + python ./03_choosing_base_configs.py small --help + python ./03_choosing_base_configs.py small --seed 94720 + + # The "big" subcommand: + python ./03_choosing_base_configs.py big --help + python ./03_choosing_base_configs.py big --seed 94720 """ from dataclasses import dataclass @@ -24,20 +37,11 @@ import tyro -@dataclass(frozen=True) -class AdamOptimizer: - learning_rate: float = 1e-3 - betas: tuple[float, float] = (0.9, 0.999) - - @dataclass(frozen=True) class ExperimentConfig: # Dataset to run experiment on. dataset: Literal["mnist", "imagenet-50"] - # Optimizer parameters. - optimizer: AdamOptimizer - # Model size. num_layers: int units: int @@ -64,7 +68,6 @@ class ExperimentConfig: "Small experiment.", ExperimentConfig( dataset="mnist", - optimizer=AdamOptimizer(), batch_size=2048, num_layers=4, units=64, @@ -77,7 +80,6 @@ class ExperimentConfig: "Big experiment.", ExperimentConfig( dataset="imagenet-50", - optimizer=AdamOptimizer(), batch_size=32, num_layers=8, units=256, diff --git a/examples/04_overriding_configs/README.rst b/examples/04_overriding_configs/README.rst new file mode 100644 index 000000000..2ea9e6906 --- /dev/null +++ b/examples/04_overriding_configs/README.rst @@ -0,0 +1,5 @@ +Overriding Configs +================== + +In these examples, we show how :func:`tyro.cli` can be used to override values +in pre-instantiated configuration objects. diff --git a/examples/04_additional/06_generics_py312.py b/examples/05_generics/01_generics_py312.py similarity index 68% rename from examples/04_additional/06_generics_py312.py rename to examples/05_generics/01_generics_py312.py index 416d75e25..bb4ef8688 100644 --- a/examples/04_additional/06_generics_py312.py +++ b/examples/05_generics/01_generics_py312.py @@ -1,17 +1,17 @@ # mypy: ignore-errors # # PEP 695 isn't yet supported in mypy. (April 4, 2024) -"""Generic Types (Python 3.12+) +"""Generics (3.12+) -Example of parsing for generic dataclasses using syntax introduced in Python -3.12 (`PEP 695 `_). +This example uses syntax introduced in Python 3.12 (`PEP 695 `_). -.. warning:: +.. note:: If used in conjunction with :code:`from __future__ import annotations`, the updated type parameter syntax requires Python 3.12.4 or newer. For technical details, see `this CPython PR `_. Usage: -`python ./05_generics.py --help` + + python ./01_generics_py312.py --help """ import dataclasses @@ -19,7 +19,7 @@ import tyro -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass class Point3[ScalarType: (int, float)]: x: ScalarType y: ScalarType @@ -27,14 +27,14 @@ class Point3[ScalarType: (int, float)]: frame_id: str -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass class Triangle: a: Point3[float] b: Point3[float] c: Point3[float] -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass class Args[ShapeType]: shape: ShapeType diff --git a/examples/04_additional/05_generics.py b/examples/05_generics/02_generics.py similarity index 77% rename from examples/04_additional/05_generics.py rename to examples/05_generics/02_generics.py index 89f96b242..698d15e8c 100644 --- a/examples/04_additional/05_generics.py +++ b/examples/05_generics/02_generics.py @@ -1,9 +1,11 @@ -"""Generic Types +"""Generics (Legacy) -Example of parsing for generic dataclasses. +The legacy :py:class:`typing.Generic` and :py:class:`typing.TypeVar` syntax for +generic types is also supported. Usage: -`python ./05_generics.py --help` + + python ./02_generics.py --help """ import dataclasses diff --git a/examples/05_generics/README.rst b/examples/05_generics/README.rst new file mode 100644 index 000000000..b6e388af6 --- /dev/null +++ b/examples/05_generics/README.rst @@ -0,0 +1,5 @@ +Generics +======== + +:mod:`tyro`'s understanding of Python's type system includes user-defined +parameterized types, which can reduce boilerplate and improve type safety. diff --git a/examples/05_custom_constructors/01_primitive_annotation.py b/examples/06_custom_constructors/01_primitive_annotation.py similarity index 50% rename from examples/05_custom_constructors/01_primitive_annotation.py rename to examples/06_custom_constructors/01_primitive_annotation.py index 505e8b9dd..e22fbca42 100644 --- a/examples/05_custom_constructors/01_primitive_annotation.py +++ b/examples/06_custom_constructors/01_primitive_annotation.py @@ -1,5 +1,11 @@ """Custom Primitive +.. note:: + + This is an advanced feature, which should not be needed for the vast + majority of use cases. If :mod:`tyro` is missing support for a built-in + Python type, please open an issue on `GitHub `_. + For additional flexibility, :mod:`tyro.constructors` exposes tyro's API for defining behavior for different types. There are two categories of types: primitive types can be instantiated from a single commandline argument, while @@ -8,9 +14,10 @@ In this example, we attach a custom constructor via a runtime annotation. Usage: -`python ./01_primitive_annotation.py --help` -`python ./01_primitive_annotation.py --dict1 '{"hello": "world"}'` -`python ./01_primitive_annotation.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'` + + python ./01_primitive_annotation.py --help + python ./01_primitive_annotation.py --dict1 '{"hello": "world"}' + python ./01_primitive_annotation.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}' """ import json @@ -23,10 +30,19 @@ JsonDict = Annotated[ dict, tyro.constructors.PrimitiveConstructorSpec( + # Number of arguments to consume. nargs=1, + # Argument name in usage messages. metavar="JSON", + # Convert a list of strings to an instance. The length of the list + # should match `nargs`. instance_from_str=lambda args: json.loads(args[0]), + # Check if an instance is of the expected type. This is only used for + # helptext formatting in the presence of union types. is_instance=lambda instance: isinstance(instance, dict), + # Convert an instance to a list of strings. This is used for handling + # default values that are set in Python. The length of the list should + # match `nargs`. str_from_instance=lambda instance: [json.dumps(instance)], ), ] diff --git a/examples/05_custom_constructors/02_primitive_registry.py b/examples/06_custom_constructors/02_primitive_registry.py similarity index 62% rename from examples/05_custom_constructors/02_primitive_registry.py rename to examples/06_custom_constructors/02_primitive_registry.py index 767765f0b..43589c235 100644 --- a/examples/05_custom_constructors/02_primitive_registry.py +++ b/examples/06_custom_constructors/02_primitive_registry.py @@ -1,17 +1,13 @@ """Custom Primitive (Registry) -For additional flexibility, :mod:`tyro.constructors` exposes tyro's API for -defining behavior for different types. There are two categories of types: -primitive types can be instantiated from a single commandline argument, while -struct types are broken down into multiple. - -In this example, we attach a custom constructor by defining a rule that applies -to all types that match ``dict[str, Any]``. +In this example, we use :class:`tyro.constructors.PrimitiveConstructorSpec` to +define a rule that applies to all types that match ``dict[str, Any]``. Usage: -`python ./02_primitive_registry.py --help` -`python ./02_primitive_registry.py --dict1 '{"hello": "world"}'` -`python ./02_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'` + + python ./02_primitive_registry.py --help + python ./02_primitive_registry.py --dict1 '{"hello": "world"}' + python ./02_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}' """ import json @@ -49,5 +45,6 @@ def main( if __name__ == "__main__": + # The custom registry is used as a context. with custom_registry: tyro.cli(main) diff --git a/examples/06_custom_constructors/README.rst b/examples/06_custom_constructors/README.rst new file mode 100644 index 000000000..f954f2cea --- /dev/null +++ b/examples/06_custom_constructors/README.rst @@ -0,0 +1,4 @@ +Custom constructors +=================== + +In these examples, we show how custom types can be parsed by and registered with :func:`tyro.cli`. diff --git a/examples/04_additional/14_suppress_console_outputs.py b/examples/07_pytorch_jax/01_pytorch_parallelism.py similarity index 88% rename from examples/04_additional/14_suppress_console_outputs.py rename to examples/07_pytorch_jax/01_pytorch_parallelism.py index c8b48840a..97ef6a74a 100644 --- a/examples/04_additional/14_suppress_console_outputs.py +++ b/examples/07_pytorch_jax/01_pytorch_parallelism.py @@ -1,4 +1,4 @@ -"""Cleaner Console Outputs for Scripts with Multiple Workers +"""PyTorch Parallelism The :code:`console_outputs=` argument can be set to :code:`False` to suppress helptext and error message printing. @@ -20,7 +20,8 @@ Usage: -`python ./14_suppress_console_outputs.py --help` + + python ./01_pytorch_parallelism.py --help """ import dataclasses diff --git a/examples/04_additional/10_flax.py b/examples/07_pytorch_jax/02_flax.py similarity index 94% rename from examples/04_additional/10_flax.py rename to examples/07_pytorch_jax/02_flax.py index ecc5c50fc..10bb0131f 100644 --- a/examples/04_additional/10_flax.py +++ b/examples/07_pytorch_jax/02_flax.py @@ -4,8 +4,9 @@ directly from :func:`tyro.cli()`. Usage: -`python ./07_flax.py --help` -`python ./07_flax.py --model.layers 4` + + python ./02_flax.py --help + python ./02_flax.py --model.layers 4 """ from flax import linen as nn diff --git a/examples/07_pytorch_jax/README.rst b/examples/07_pytorch_jax/README.rst new file mode 100644 index 000000000..f7bb835f0 --- /dev/null +++ b/examples/07_pytorch_jax/README.rst @@ -0,0 +1,4 @@ +PyTorch / JAX +============= + +In these examples, we show some patterns for using :func:`tyro.cli` with PyTorch and JAX. diff --git a/src/tyro/conf/_markers.py b/src/tyro/conf/_markers.py index f96aa68e5..5ac78ef99 100644 --- a/src/tyro/conf/_markers.py +++ b/src/tyro/conf/_markers.py @@ -68,17 +68,22 @@ By default, :mod:`tyro` will generate a traditional CLI interface where args are applied to the directly preceding subcommand. When we have two subcommands ``s1`` and ``s2``: -`` -python x.py {--root options} s1 {--s1 options} s2 {--s2 options} -`` + + +.. code-block:: bash + + python x.py {--root options} s1 {--s1 options} s2 {--s2 options} This can be frustrating because the resulting CLI is sensitive to the positioning of options. To consolidate subcommands, we push arguments to the end, after all subcommands: -`` -python x.py s1 s2 {--root, s1, and s2 options} -`` + + +.. code-block:: bash + + python x.py s1 s2 {--root, s1, and s2 options} + This is more robust to reordering of options, ensuring that any new options can simply be placed at the end of the command. @@ -103,6 +108,8 @@ If we have a structure with the field: +.. code-block:: python + cmd: NestedType By default, ``--cmd.arg`` may be generated as a flag. If prefixes are omitted, we would diff --git a/src/tyro/extras/_subcommand_app.py b/src/tyro/extras/_subcommand_app.py index 21e72be4e..7ce21494b 100644 --- a/src/tyro/extras/_subcommand_app.py +++ b/src/tyro/extras/_subcommand_app.py @@ -8,7 +8,9 @@ class SubcommandApp: - """This module provides a decorator-based API for subcommands in :mod:`tyro`, inspired by click. + """This class provides a decorator-based API for subcommands in + :mod:`tyro`, inspired by click. Under-the-hood, this is a light wrapper + over :func:`tyro.cli`. Example: From 10651352af9e7a785c10339b868bd43f35b3192f Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Thu, 7 Nov 2024 04:50:48 -0800 Subject: [PATCH 475/491] Helptext, docs fixes (#199) * Helptext, docs fixes * refine --- README.md | 3 +- docs/source/examples/basics.rst | 152 +++++++++--------- docs/source/examples/overriding_configs.rst | 37 ++--- docs/source/examples/subcommands.rst | 37 +++-- docs/source/goals_and_alternatives.md | 12 +- docs/source/tab_completion.md | 4 +- examples/01_basics/08_positional.py | 2 +- .../{04_classes.py => 13_classes.py} | 4 +- .../03_subcommands/03_multiple_subcommands.py | 4 +- .../02_overriding_yaml.py | 10 +- .../03_choosing_base_configs.py | 11 +- src/tyro/_arguments.py | 15 +- src/tyro/_cli.py | 13 +- src/tyro/_docstrings.py | 6 +- src/tyro/_fields.py | 10 +- src/tyro/_parsers.py | 5 +- src/tyro/_singleton.py | 2 +- src/tyro/conf/_markers.py | 2 +- src/tyro/constructors/_primitive_spec.py | 2 +- src/tyro/constructors/_struct_spec.py | 4 +- tests/test_base_configs_nested.py | 2 +- tests/test_boolean_optional.py | 14 ++ tests/test_helptext.py | 2 +- tests/test_mixed_unions.py | 4 +- .../test_base_configs_nested_generated.py | 2 +- .../test_boolean_optional_generated.py | 14 ++ .../test_helptext_generated.py | 2 +- .../test_mixed_unions_generated.py | 4 +- 28 files changed, 200 insertions(+), 179 deletions(-) rename examples/01_basics/{04_classes.py => 13_classes.py} (86%) diff --git a/README.md b/README.md index a2455b4f0..fd03bf0e3 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@

diff --git a/docs/source/examples/basics.rst b/docs/source/examples/basics.rst index ffa143219..e701e6b7d 100644 --- a/docs/source/examples/basics.rst +++ b/docs/source/examples/basics.rst @@ -212,68 +212,6 @@ We can override arguments: Config(source_paths=(PosixPath('data1'), PosixPath('data2')), dimensions=(32, 32)) -.. _example-04_classes: - -Instantiating Classes ---------------------- - -In addition to functions and dataclasses, we can also generate CLIs from the -constructors of standard Python classes. - - -.. code-block:: python - :linenos: - - # 04_classes.py - import tyro - - class Args: - def __init__( - self, - field1: str, - field2: int, - flag: bool = False, - ): - """Arguments. - - Args: - field1: A string field. - field2: A numeric field. - flag: A boolean flag. - """ - self.data = [field1, field2, flag] - - if __name__ == "__main__": - args = tyro.cli(Args) - print(args.data) - - - - -.. raw:: html - - 

-    $ python ./04_classes.py --help
-    usage: 04_classes.py [-h] --field1 STR --field2 INT [--flag | --no-flag]
-    
-    Arguments.
-    
-    ╭─ options ───────────────────────────────────────────────╮
-    │ -h, --help              show this help message and exit │
-    │ --field1 STR            A string field. (required)      │
-    │ --field2 INT            A numeric field. (required)     │
-    │ --flag, --no-flag       A boolean flag. (default: None) │
-    ╰─────────────────────────────────────────────────────────╯
-
- - - -.. raw:: html - - 
-    $ python ./04_classes.py --field1 hello --field2 7
-    ['hello', 7, False]
-
.. _example-04_flags: Booleans and Flags @@ -329,9 +267,9 @@ To turn off conversion, see :class:`tyro.conf.FlagConversionOff`. │ --optional-boolean {None,True,False} │ │ Optional boolean. Same as above, but can be omitted. (default: None) │ │ --flag-a, --no-flag-a │ - │ Pass --flag-a in to set this value to True. (default: None) │ + │ Pass --flag-a in to set this value to True. (default: False) │ │ --flag-b, --no-flag-b │ - │ Pass --no-flag-b in to set this value to False. (default: None) │ + │ Pass --no-flag-b in to set this value to False. (default: True) │ ╰──────────────────────────────────────────────────────────────────────────╯ @@ -650,7 +588,7 @@ For more general positional arguments, see :class:`tyro.conf.Positional`. /, # Mark the end of positional arguments. verbose: bool = False, ) -> None: - """Command-line interface defined using a function signature. Note that this + """Command-line interface defined using a function signature. This docstring is parsed to generate helptext. Args: @@ -672,18 +610,18 @@ For more general positional arguments, see :class:`tyro.conf.Positional`. $ python 08_positional.py --help usage: 08_positional.py [-h] [--verbose | --no-verbose] PATH PATH - Command-line interface defined using a function signature. Note that this - docstring is parsed to generate helptext. + Command-line interface defined using a function signature. This docstring is + parsed to generate helptext. - ╭─ positional arguments ────────────────────────────────────────╮ - │ PATH Source path. (required) │ - │ PATH Destination path. (required) │ - ╰───────────────────────────────────────────────────────────────╯ - ╭─ options ─────────────────────────────────────────────────────╮ - │ -h, --help show this help message and exit │ - │ --verbose, --no-verbose │ - │ Explain what is being done. (default: None) │ - ╰───────────────────────────────────────────────────────────────╯ + ╭─ positional arguments ─────────────────────────────────────────╮ + │ PATH Source path. (required) │ + │ PATH Destination path. (required) │ + ╰────────────────────────────────────────────────────────────────╯ + ╭─ options ──────────────────────────────────────────────────────╮ + │ -h, --help show this help message and exit │ + │ --verbose, --no-verbose │ + │ Explain what is being done. (default: False) │ + ╰────────────────────────────────────────────────────────────────╯ @@ -987,4 +925,66 @@ Repeatable 'counter' arguments can be specified via :data:`tyro.conf.UseCounterA $ python ./12_counters.py -vvv Verbosity level: 0 Verbosity level (aliased): 3 + +.. _example-13_classes: + +Instantiating Classes +--------------------- + +In addition to functions and dataclasses, we can also generate CLIs from the +constructors of standard Python classes. + + +.. code-block:: python + :linenos: + + # 13_classes.py + import tyro + + class Args: + def __init__( + self, + field1: str, + field2: int, + flag: bool = False, + ): + """Arguments. + + Args: + field1: A string field. + field2: A numeric field. + flag: A boolean flag. + """ + self.data = [field1, field2, flag] + + if __name__ == "__main__": + args = tyro.cli(Args) + print(args.data) + + + + +.. raw:: html + + 
+    $ python ./13_classes.py --help
+    usage: 13_classes.py [-h] --field1 STR --field2 INT [--flag | --no-flag]
+    
+    Arguments.
+    
+    ╭─ options ────────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit  │
+    │ --field1 STR            A string field. (required)       │
+    │ --field2 INT            A numeric field. (required)      │
+    │ --flag, --no-flag       A boolean flag. (default: False) │
+    ╰──────────────────────────────────────────────────────────╯
+
+ + + +.. raw:: html + + 
+    $ python ./13_classes.py --field1 hello --field2 7
+    ['hello', 7, False]
\ No newline at end of file diff --git a/docs/source/examples/overriding_configs.rst b/docs/source/examples/overriding_configs.rst index 3f28f2aca..32e3c25b4 100644 --- a/docs/source/examples/overriding_configs.rst +++ b/docs/source/examples/overriding_configs.rst @@ -105,11 +105,11 @@ types. Overriding YAML Configs ----------------------- -:mod:`tyro` understands a wide range of data structures, including standard dictionaries -and lists. +:mod:`tyro` understands a wide range of data structures, including standard +dictionaries and lists. -If you have a library of existing YAML files that you want to use, `tyro` can -help override values within them. +If you have a library of existing YAML files that you want to use, +:func:`tyro.cli` can help override values within them. .. note:: @@ -124,7 +124,7 @@ help override values within them. import tyro - # YAML configuration. Note that this could also be loaded from a file! Environment + # YAML configuration. This could also be loaded from a file! Environment # variables are an easy way to select between different YAML files. default_yaml = r""" exp_name: test @@ -244,16 +244,15 @@ syntax. # Total number of training steps. train_steps: int - # Random seed. This is helpful for making sure that our experiments are all - # reproducible! + # Random seed. seed: int - # Activation to use. Not specifiable via the commandline. + # Not specifiable via the commandline. activation: Callable[[], nn.Module] - # Note that we could also define this library using separate YAML files (similar to - # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type - # checking + IDE support. + # We could also define this library using separate YAML files (similar to + # `config_path`/`config_name` in Hydra), but staying in Python enables seamless + # type checking + IDE support. default_configs = { "small": ( "Small experiment.", @@ -321,11 +320,9 @@ The "small" subcommand: │ --units INT Model size. (default: 64) │ │ --batch-size INT Batch size. (default: 2048) │ │ --train-steps INT Total number of training steps. (default: 30000) │ - │ --seed INT Random seed. This is helpful for making sure that │ - │ our experiments are all reproducible! (default: 0) │ - │ --activation {fixed} Activation to use. Not specifiable via the │ - │ commandline. (fixed to: <class │ - │ 'torch.nn.modules.activation.ReLU'>) │ + │ --seed INT Random seed. (default: 0) │ + │ --activation {fixed} Not specifiable via the commandline. (fixed to: │ + │ <class 'torch.nn.modules.activation.ReLU'>) │ ╰────────────────────────────────────────────────────────────────────────────╯ @@ -357,11 +354,9 @@ The "big" subcommand: │ --units INT Model size. (default: 256) │ │ --batch-size INT Batch size. (default: 32) │ │ --train-steps INT Total number of training steps. (default: 100000) │ - │ --seed INT Random seed. This is helpful for making sure that │ - │ our experiments are all reproducible! (default: 0) │ - │ --activation {fixed} Activation to use. Not specifiable via the │ - │ commandline. (fixed to: <class │ - │ 'torch.nn.modules.activation.GELU'>) │ + │ --seed INT Random seed. (default: 0) │ + │ --activation {fixed} Not specifiable via the commandline. (fixed to: │ + │ <class 'torch.nn.modules.activation.GELU'>) │ ╰────────────────────────────────────────────────────────────────────────────╯ diff --git a/docs/source/examples/subcommands.rst b/docs/source/examples/subcommands.rst index e99d5a762..3d1d6aa57 100644 --- a/docs/source/examples/subcommands.rst +++ b/docs/source/examples/subcommands.rst @@ -277,8 +277,8 @@ Multiple unions over struct types are populated using a series of subcommands. tyro.cli(train, config=(tyro.conf.ConsolidateSubcommandArgs,)) -Note that we apply the :class:`tyro.conf.ConsolidateSubcommandArgs` flag. -This pushes all arguments to the end of the command: +We apply the :class:`tyro.conf.ConsolidateSubcommandArgs` flag. This +pushes all arguments to the end of the command: .. raw:: html @@ -331,21 +331,20 @@ This pushes all arguments to the end of the command: [-h] [--optimizer.learning-rate FLOAT] [--optimizer.betas FLOAT FLOAT] [--dataset.binary | --dataset.no-binary] - ╭─ options ────────────────────────────────────────────────────────╮ - │ -h, --help │ - │ show this help message and exit │ - ╰──────────────────────────────────────────────────────────────────╯ - ╭─ optimizer options ──────────────────────────────────────────────╮ - │ --optimizer.learning-rate FLOAT │ - │ (default: 0.001) │ - │ --optimizer.betas FLOAT FLOAT │ - │ (default: 0.9 0.999) │ - ╰──────────────────────────────────────────────────────────────────╯ - ╭─ dataset options ────────────────────────────────────────────────╮ - │ --dataset.binary, --dataset.no-binary │ - │ Set to load binary version of MNIST dataset. (default: None) │ - - ╰──────────────────────────────────────────────────────────────────╯ + ╭─ options ─────────────────────────────────────────────────────────╮ + │ -h, --help │ + │ show this help message and exit │ + ╰───────────────────────────────────────────────────────────────────╯ + ╭─ optimizer options ───────────────────────────────────────────────╮ + │ --optimizer.learning-rate FLOAT │ + │ (default: 0.001) │ + │ --optimizer.betas FLOAT FLOAT │ + │ (default: 0.9 0.999) │ + ╰───────────────────────────────────────────────────────────────────╯ + ╭─ dataset options ─────────────────────────────────────────────────╮ + │ --dataset.binary, --dataset.no-binary │ + │ Set to load binary version of MNIST dataset. (default: False) │ + ╰───────────────────────────────────────────────────────────────────╯ @@ -422,7 +421,7 @@ subcommands, which is inspired by `click `_. ╭─ options ───────────────────────────────────────────────╮ │ -h, --help show this help message and exit │ │ --name STR (required) │ - │ --loud, --no-loud (default: None) │ + │ --loud, --no-loud (default: False) │ ╰─────────────────────────────────────────────────────────╯ @@ -534,7 +533,7 @@ more generality is needed, the internal working are explained in the docs for ╭─ options ──────────────────────────────────────────────╮ │ -h, --help show this help message and exit │ │ --message STR (required) │ - │ --all, --no-all (default: None) │ + │ --all, --no-all (default: False) │ ╰────────────────────────────────────────────────────────╯ diff --git a/docs/source/goals_and_alternatives.md b/docs/source/goals_and_alternatives.md index 8ab178078..be4b3af9b 100644 --- a/docs/source/goals_and_alternatives.md +++ b/docs/source/goals_and_alternatives.md @@ -76,12 +76,12 @@ More concretely, we can also compare specific features. A noncomprehensive set: -Note that other libraries are generally aimed specifically at only one of -dataclasses (`datargs`, `simple-parsing`, `argparse-dataclass`, -`argparse-dataclasses`, `dataclass-cli`, `clout`, `hf_argparser`, `pyrallis`, -`yahp`), custom structures (`tap`), or functions (`typer`) rather than general -types and callables, but offer other features that you might find critical, such -as registration for custom types (`pyrallis`), built-in approaches for +Other libraries are generally aimed specifically at only one of dataclasses +(`datargs`, `simple-parsing`, `argparse-dataclass`, `argparse-dataclasses`, +`dataclass-cli`, `clout`, `hf_argparser`, `pyrallis`, `yahp`), custom +structures (`tap`), or functions (`typer`, `defopt`) rather than general types +and callables, but offer other features that you might find critical, such as +registration for custom types (`pyrallis`), built-in approaches for serialization and config files (`tap`, `pyrallis`, `yahp`), and opportunities for integration with fields generated using standard argparse definitions (`simple-parsing`). diff --git a/docs/source/tab_completion.md b/docs/source/tab_completion.md index a7ac735f0..30938afd0 100644 --- a/docs/source/tab_completion.md +++ b/docs/source/tab_completion.md @@ -64,8 +64,8 @@ Borrowing from the `bash-completion` source[^bash_completion], we can run: completion_dir=${BASH_COMPLETION_USER_DIR:-${XDG_DATA_HOME:-$HOME/.local/share}/bash-completion}/completions/ mkdir -p $completion_dir -# (2) Write completion scripts. Note that the name of the completion script must -# match the name of the file. +# (2) Write completion scripts. The name of the completion script must match +# the name of the file. python 01_functions.py --tyro-write-completion bash ${completion_dir}/01_functions.py ``` diff --git a/examples/01_basics/08_positional.py b/examples/01_basics/08_positional.py index 6380b90b0..089342cce 100644 --- a/examples/01_basics/08_positional.py +++ b/examples/01_basics/08_positional.py @@ -24,7 +24,7 @@ def main( /, # Mark the end of positional arguments. verbose: bool = False, ) -> None: - """Command-line interface defined using a function signature. Note that this + """Command-line interface defined using a function signature. This docstring is parsed to generate helptext. Args: diff --git a/examples/01_basics/04_classes.py b/examples/01_basics/13_classes.py similarity index 86% rename from examples/01_basics/04_classes.py rename to examples/01_basics/13_classes.py index 5c977e14d..c40b37080 100644 --- a/examples/01_basics/04_classes.py +++ b/examples/01_basics/13_classes.py @@ -5,8 +5,8 @@ Usage: - python ./04_classes.py --help - python ./04_classes.py --field1 hello --field2 7 + python ./13_classes.py --help + python ./13_classes.py --field1 hello --field2 7 """ import tyro diff --git a/examples/03_subcommands/03_multiple_subcommands.py b/examples/03_subcommands/03_multiple_subcommands.py index 86cfcd5c8..91da881d2 100644 --- a/examples/03_subcommands/03_multiple_subcommands.py +++ b/examples/03_subcommands/03_multiple_subcommands.py @@ -4,8 +4,8 @@ Usage: - # Note that we apply the :class:`tyro.conf.ConsolidateSubcommandArgs` flag. - # This pushes all arguments to the end of the command: + # We apply the :class:`tyro.conf.ConsolidateSubcommandArgs` flag. This + # pushes all arguments to the end of the command: python ./03_multiple_subcommands.py --help python ./03_multiple_subcommands.py dataset:mnist --help python ./03_multiple_subcommands.py dataset:mnist optimizer:adam --help diff --git a/examples/04_overriding_configs/02_overriding_yaml.py b/examples/04_overriding_configs/02_overriding_yaml.py index 6ef7bcce7..42023cdf6 100644 --- a/examples/04_overriding_configs/02_overriding_yaml.py +++ b/examples/04_overriding_configs/02_overriding_yaml.py @@ -1,10 +1,10 @@ """Overriding YAML Configs -:mod:`tyro` understands a wide range of data structures, including standard dictionaries -and lists. +:mod:`tyro` understands a wide range of data structures, including standard +dictionaries and lists. -If you have a library of existing YAML files that you want to use, `tyro` can -help override values within them. +If you have a library of existing YAML files that you want to use, +:func:`tyro.cli` can help override values within them. .. note:: @@ -20,7 +20,7 @@ import tyro -# YAML configuration. Note that this could also be loaded from a file! Environment +# YAML configuration. This could also be loaded from a file! Environment # variables are an easy way to select between different YAML files. default_yaml = r""" exp_name: test diff --git a/examples/04_overriding_configs/03_choosing_base_configs.py b/examples/04_overriding_configs/03_choosing_base_configs.py index fb26e0689..fd532b677 100644 --- a/examples/04_overriding_configs/03_choosing_base_configs.py +++ b/examples/04_overriding_configs/03_choosing_base_configs.py @@ -52,17 +52,16 @@ class ExperimentConfig: # Total number of training steps. train_steps: int - # Random seed. This is helpful for making sure that our experiments are all - # reproducible! + # Random seed. seed: int - # Activation to use. Not specifiable via the commandline. + # Not specifiable via the commandline. activation: Callable[[], nn.Module] -# Note that we could also define this library using separate YAML files (similar to -# `config_path`/`config_name` in Hydra), but staying in Python enables seamless type -# checking + IDE support. +# We could also define this library using separate YAML files (similar to +# `config_path`/`config_name` in Hydra), but staying in Python enables seamless +# type checking + IDE support. default_configs = { "small": ( "Small experiment.", diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py index d0b4aef9b..34198ff84 100644 --- a/src/tyro/_arguments.py +++ b/src/tyro/_arguments.py @@ -136,7 +136,7 @@ def add_argument( kwargs["default"] = [] # Apply overrides in our arg configuration object. - # Note that the `name` field is applied when the field object is instantiated! + # The `name` field is applied when the field object is instantiated! if self.field.argconf.metavar is not None: kwargs["metavar"] = self.field.argconf.metavar @@ -250,6 +250,7 @@ def _rule_handle_boolean_flags( lowered.instance_from_str = ( lambda x: x ) # argparse will directly give us a bool! + lowered.default = arg.field.default return assert False, ( @@ -518,7 +519,7 @@ def _rule_generate_helptext( else: help_parts.append(_rich_tag_if_enabled("(required)", "helptext_required")) - # Note that the percent symbol needs some extra handling in argparse. + # The percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string lowered.help = " ".join([p for p in help_parts if len(p) > 0]).replace("%", "%%") return @@ -546,10 +547,10 @@ def _rule_set_name_or_flag_and_dest( _markers.OmitSubcommandPrefixes in arg.field.markers and arg.subcommand_prefix != "" ): - # Strip subcommand prefixes, but keep following prefixes. Note that - # `extern_prefix` can start with the prefix corresponding to the parent - # subcommand, but end with other prefixes correspondeding to nested - # structures within the subcommand. + # Strip subcommand prefixes, but keep following + # prefixes.`extern_prefix` can start with the prefix corresponding to + # the parent subcommand, but end with other prefixes correspondeding to + # nested structures within the subcommand. name_or_flag = _strings.make_field_name( [arg.extern_prefix, arg.field.extern_name] ) @@ -584,7 +585,7 @@ def _rule_positional_special_handling( if metavar is not None: metavar = "[" + metavar + "]" if lowered.nargs == 1: - # Optional positional arguments. Note that this needs to be special-cased in + # Optional positional arguments. This needs to be special-cased in # _calling.py. nargs = "?" else: diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py index ad203dcd4..a25fb17b3 100644 --- a/src/tyro/_cli.py +++ b/src/tyro/_cli.py @@ -28,8 +28,9 @@ OutT = TypeVar("OutT") -# Note that the overload here is necessary for pyright and pylance due to special-casing -# related to using typing.Type[] as a temporary replacement for typing.TypeForm[]. +# The overload here is necessary for pyright and pylance due to special-casing +# related to using typing.Type[] as a temporary replacement for +# typing.TypeForm[]. # # https://github.com/microsoft/pyright/issues/4298 @@ -455,10 +456,10 @@ def _cli_impl( ) except _calling.InstantiationError as e: # Print prettier errors. - # Note that this doesn't catch errors raised directly by get_out(), - # since that's called later! This is intentional, because we do less - # error handling for the root callable. Relevant: the - # `field_name_prefix == ""` condition in `callable_with_args()`! + # This doesn't catch errors raised directly by get_out(), since that's + # called later! This is intentional, because we do less error handling + # for the root callable. Relevant: the `field_name_prefix == ""` + # condition in `callable_with_args()`! # Emulate argparse's error behavior when invalid arguments are passed in. from rich.console import Console, Group diff --git a/src/tyro/_docstrings.py b/src/tyro/_docstrings.py index 6eadae785..a5918bd6e 100644 --- a/src/tyro/_docstrings.py +++ b/src/tyro/_docstrings.py @@ -298,9 +298,9 @@ def get_field_docstring(cls: Type, field_name: str) -> Optional[str]: def get_callable_description(f: Callable) -> str: """Get description associated with a callable via docstring parsing. - Note that the `dataclasses.dataclass` will automatically populate __doc__ based on - the fields of the class if a docstring is not specified; this helper will ignore - these docstrings.""" + `dataclasses.dataclass` will automatically populate __doc__ based on the + fields of the class if a docstring is not specified; this helper will + ignore these docstrings.""" f, _ = _resolver.resolve_generic_types(f) f = _resolver.unwrap_origin_strip_extras(f) diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 2cc973c93..4e786c5f4 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -385,10 +385,10 @@ def _field_list_from_function( # # This will create a `--kwargs STR T [STR T ...]` CLI argument. # - # Note that it would be straightforward to make both this and *args truly - # positional, omitting the --args/--kwargs prefix, but we are choosing not - # to because it would make *args and **kwargs difficult to use in - # conjunction. + # It would be straightforward to make both this and *args truly + # positional, omitting the --args/--kwargs prefix, but we are + # choosing not to because it would make *args and **kwargs + # difficult to use in conjunction. markers = (_markers._UnpackKwargsCall,) typ = Dict.__getitem__((str, typ)) # type: ignore default = {} @@ -397,7 +397,7 @@ def _field_list_from_function( field_list.append( FieldDefinition.make( name=param.name, - # Note that param.annotation doesn't resolve forward references. + # param.annotation doesn't resolve forward references. typ=typ if default_instance in MISSING_SINGLETONS else Annotated.__class_getitem__((typ, _markers._OPTIONAL_GROUP)), # type: ignore diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index 06c7cb79b..edfceb72a 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -99,8 +99,7 @@ def from_callable_or_type( # Cycle detection. # - # Note that 'parent' here refers to in the nesting hierarchy, not the - # superclass. + # 'parent' here refers to in the nesting hierarchy, not the superclass. if f in parent_classes and f is not dict: raise UnsupportedTypeAnnotationError( f"Found a cyclic dependency with type {f}." @@ -270,7 +269,7 @@ def group_name_from_arg(arg: _arguments.ArgumentDefinition) -> str: positional_group = parser._action_groups[0] assert positional_group.title == "positional arguments" - # Add each argument group. Note that groups with only suppressed arguments won't + # Add each argument group. Groups with only suppressed arguments won't # be added. for arg in self.args: group_name = group_name_from_arg(arg) diff --git a/src/tyro/_singleton.py b/src/tyro/_singleton.py index c947966ad..80d367d9a 100644 --- a/src/tyro/_singleton.py +++ b/src/tyro/_singleton.py @@ -46,7 +46,7 @@ class NotRequiredButWeDontKnowTheValueType(Singleton): EXCLUDE_FROM_CALL = ExcludeFromCallType() -# Note that our "public" missing API will always be the propagating missing sentinel. +# Our "public" missing API will always be the propagating missing sentinel. MISSING: Any = MISSING_PROP """Sentinel value to mark fields as missing. Can be used to mark fields passed in via `default=` for `tyro.cli()` as required.""" diff --git a/src/tyro/conf/_markers.py b/src/tyro/conf/_markers.py index 5ac78ef99..4e6bf1b0b 100644 --- a/src/tyro/conf/_markers.py +++ b/src/tyro/conf/_markers.py @@ -7,7 +7,7 @@ # Current design issue: markers are applied recursively to nested structures, but can't # be unapplied. -# Note that all Annotated[T, None] values are just for static checkers. The real marker +# All Annotated[T, None] values are just for static checkers. The real marker # singletons are instantiated dynamically below. # # An alias could ideally be made, but SpecialForm aliases aren't well supported by diff --git a/src/tyro/constructors/_primitive_spec.py b/src/tyro/constructors/_primitive_spec.py index 56a1d4d7b..5cf4567b0 100644 --- a/src/tyro/constructors/_primitive_spec.py +++ b/src/tyro/constructors/_primitive_spec.py @@ -404,7 +404,7 @@ def tuple_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None: if type_info.type_origin is not tuple: return None types = get_args(type_info.type) - typeset = set(types) # Note that sets are unordered. + typeset = set(types) # Sets are unordered. typeset_no_ellipsis = typeset - {Ellipsis} # type: ignore if typeset_no_ellipsis != typeset: diff --git a/src/tyro/constructors/_struct_spec.py b/src/tyro/constructors/_struct_spec.py index 8f54c21c2..1da5502da 100644 --- a/src/tyro/constructors/_struct_spec.py +++ b/src/tyro/constructors/_struct_spec.py @@ -150,7 +150,7 @@ def dataclass_rule(info: StructTypeInfo) -> StructConstructorSpec | None: helptext = dc_field.metadata.get("help", None) assert isinstance(helptext, (str, type(None))) - # Try to get helptext from docstrings. Note that this can't be generated + # Try to get helptext from docstrings. This can't be generated # dynamically. if helptext is None: helptext = _docstrings.get_field_docstring(info.type, dc_field.name) @@ -589,7 +589,7 @@ def _get_dataclass_field_default( # Try grabbing default from dataclass field. if field.default not in MISSING_SINGLETONS: default = field.default - # Note that dataclasses.is_dataclass() will also return true for dataclass + # dataclasses.is_dataclass() will also return true for dataclass # _types_, not just instances. if type(default) is not type and dataclasses.is_dataclass(default): _ensure_dataclass_instance_used_as_default_is_frozen(field, default) diff --git a/tests/test_base_configs_nested.py b/tests/test_base_configs_nested.py index f9dca8f03..fb738e053 100644 --- a/tests/test_base_configs_nested.py +++ b/tests/test_base_configs_nested.py @@ -63,7 +63,7 @@ class ExperimentConfig: ) -# Note that we could also define this library using separate YAML files (similar to +# We could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. ExperimentConfigDataParserUnion = tyro.extras.subcommand_type_from_defaults( diff --git a/tests/test_boolean_optional.py b/tests/test_boolean_optional.py index ba48bc9f3..a7faca410 100644 --- a/tests/test_boolean_optional.py +++ b/tests/test_boolean_optional.py @@ -1,5 +1,7 @@ import dataclasses +from helptext_utils import get_helptext_with_checks + import tyro @@ -69,3 +71,15 @@ class A: args=["--x", "True"], default=A(False), # type: ignore ) == A(True) + + +def test_flag_default_true_helptext() -> None: + """Test for argparse.BooleanOptionalAction-style usage.""" + + @dataclasses.dataclass + class A: + x: bool = True + + assert "(default: True)" in get_helptext_with_checks(A) + assert "(default: False)" not in get_helptext_with_checks(A) + assert "(default: None)" not in get_helptext_with_checks(A) diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 588018f29..2301285c2 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -291,7 +291,7 @@ class HelptextHardString: """Helptext. 2% milk.""" # fmt: on - # Note that the percent symbol needs some extra handling in argparse. + # The percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string helptext = get_helptext_with_checks(HelptextHardString) diff --git a/tests/test_mixed_unions.py b/tests/test_mixed_unions.py index d6444f9bd..ec335a380 100644 --- a/tests/test_mixed_unions.py +++ b/tests/test_mixed_unions.py @@ -25,7 +25,7 @@ class DefaultSMTPServer: @dataclasses.dataclass class DefaultSubparser: x: int - # Note that we add [int, str] to the annotation here... this should be ignored. + # We add [int, str] to the annotation here... this should be ignored. bc: Union[int, str, DefaultHTTPServer, DefaultSMTPServer] = dataclasses.field( default_factory=lambda: DefaultHTTPServer(5) ) @@ -70,7 +70,7 @@ class DefaultSMTPServer: @dataclasses.dataclass class DefaultSubparser: x: int - # Note that we add [int, str] to the annotation here... this should be ignored. + # We add [int, str] to the annotation here... this should be ignored. bc: Union[int, str, DefaultHTTPServer, DefaultSMTPServer] = 5 assert ( diff --git a/tests/test_py311_generated/test_base_configs_nested_generated.py b/tests/test_py311_generated/test_base_configs_nested_generated.py index 9ce369220..6b1ff72cf 100644 --- a/tests/test_py311_generated/test_base_configs_nested_generated.py +++ b/tests/test_py311_generated/test_base_configs_nested_generated.py @@ -62,7 +62,7 @@ class ExperimentConfig: ) -# Note that we could also define this library using separate YAML files (similar to +# We could also define this library using separate YAML files (similar to # `config_path`/`config_name` in Hydra), but staying in Python enables seamless type # checking + IDE support. ExperimentConfigDataParserUnion = tyro.extras.subcommand_type_from_defaults( diff --git a/tests/test_py311_generated/test_boolean_optional_generated.py b/tests/test_py311_generated/test_boolean_optional_generated.py index ba48bc9f3..a7faca410 100644 --- a/tests/test_py311_generated/test_boolean_optional_generated.py +++ b/tests/test_py311_generated/test_boolean_optional_generated.py @@ -1,5 +1,7 @@ import dataclasses +from helptext_utils import get_helptext_with_checks + import tyro @@ -69,3 +71,15 @@ class A: args=["--x", "True"], default=A(False), # type: ignore ) == A(True) + + +def test_flag_default_true_helptext() -> None: + """Test for argparse.BooleanOptionalAction-style usage.""" + + @dataclasses.dataclass + class A: + x: bool = True + + assert "(default: True)" in get_helptext_with_checks(A) + assert "(default: False)" not in get_helptext_with_checks(A) + assert "(default: None)" not in get_helptext_with_checks(A) diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py index 7ef04eca7..3d0f8fb80 100644 --- a/tests/test_py311_generated/test_helptext_generated.py +++ b/tests/test_py311_generated/test_helptext_generated.py @@ -303,7 +303,7 @@ class HelptextHardString: """Helptext. 2% milk.""" # fmt: on - # Note that the percent symbol needs some extra handling in argparse. + # The percent symbol needs some extra handling in argparse. # https://stackoverflow.com/questions/21168120/python-argparse-errors-with-in-help-string helptext = get_helptext_with_checks(HelptextHardString) diff --git a/tests/test_py311_generated/test_mixed_unions_generated.py b/tests/test_py311_generated/test_mixed_unions_generated.py index c361de491..07674671c 100644 --- a/tests/test_py311_generated/test_mixed_unions_generated.py +++ b/tests/test_py311_generated/test_mixed_unions_generated.py @@ -25,7 +25,7 @@ class DefaultSMTPServer: @dataclasses.dataclass class DefaultSubparser: x: int - # Note that we add [int, str] to the annotation here... this should be ignored. + # We add [int, str] to the annotation here... this should be ignored. bc: int | str | DefaultHTTPServer | DefaultSMTPServer = dataclasses.field( default_factory=lambda: DefaultHTTPServer(5) ) @@ -70,7 +70,7 @@ class DefaultSMTPServer: @dataclasses.dataclass class DefaultSubparser: x: int - # Note that we add [int, str] to the annotation here... this should be ignored. + # We add [int, str] to the annotation here... this should be ignored. bc: int | str | DefaultHTTPServer | DefaultSMTPServer = 5 assert ( From f150c737aef19666519a93ab9b49eb3c7884da4b Mon Sep 17 00:00:00 2001 From: brentyi Date: Sat, 9 Nov 2024 23:19:35 -0800 Subject: [PATCH 476/491] Add test for `typing.Text` --- tests/test_dcargs.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index f9c604971..e63f11077 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -13,6 +13,7 @@ Dict, List, Optional, + Text, Tuple, TypeVar, Union, @@ -20,9 +21,8 @@ import pytest import torch -from typing_extensions import Annotated, Final, Literal, TypeAlias - import tyro +from typing_extensions import Annotated, Final, Literal, TypeAlias def test_no_args() -> None: @@ -573,6 +573,15 @@ def main(x: AnyStr) -> AnyStr: assert tyro.cli(main, args=["--x", "hello„"]) == "hello„" +def test_text() -> None: + # `Text` is an alias for `str` in Python 3. + def main(x: Text) -> Text: + return x + + assert tyro.cli(main, args=["--x", "hello"]) == "hello" + assert tyro.cli(main, args=["--x", "hello„"]) == "hello„" + + def test_fixed() -> None: def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]: return x From b9e72b7cb7d4eab4eb8d1459a74eeb2e89dc62a9 Mon Sep 17 00:00:00 2001 From: brentyi Date: Mon, 11 Nov 2024 14:59:21 -0800 Subject: [PATCH 477/491] Docs nits --- ...ed_structures.rst => hierarchical_structures.rst} | 12 ++++++------ docs/source/index.md | 2 +- .../01_nesting.py | 0 .../02_nesting_in_func.py | 2 +- .../03_nesting_containers.py | 0 .../04_dictionaries.py | 0 .../05_tuples.py | 0 .../06_pydantic.py | 0 .../07_attrs.py | 0 .../README.rst | 4 ++-- tests/test_dcargs.py | 3 ++- 11 files changed, 12 insertions(+), 11 deletions(-) rename docs/source/examples/{nested_structures.rst => hierarchical_structures.rst} (99%) rename examples/{02_nested_structures => 02_hierarchical_structures}/01_nesting.py (100%) rename examples/{02_nested_structures => 02_hierarchical_structures}/02_nesting_in_func.py (96%) rename examples/{02_nested_structures => 02_hierarchical_structures}/03_nesting_containers.py (100%) rename examples/{02_nested_structures => 02_hierarchical_structures}/04_dictionaries.py (100%) rename examples/{02_nested_structures => 02_hierarchical_structures}/05_tuples.py (100%) rename examples/{02_nested_structures => 02_hierarchical_structures}/06_pydantic.py (100%) rename examples/{02_nested_structures => 02_hierarchical_structures}/07_attrs.py (100%) rename examples/{02_nested_structures => 02_hierarchical_structures}/README.rst (50%) diff --git a/docs/source/examples/nested_structures.rst b/docs/source/examples/hierarchical_structures.rst similarity index 99% rename from docs/source/examples/nested_structures.rst rename to docs/source/examples/hierarchical_structures.rst index a64a046a0..9c920f0eb 100644 --- a/docs/source/examples/nested_structures.rst +++ b/docs/source/examples/hierarchical_structures.rst @@ -1,13 +1,13 @@ .. Comment: this file is automatically generated by `update_example_docs.py`. It should not be modified manually. -.. _example-category-nested_structures: +.. _example-category-hierarchical_structures: -Nested Structures +Hierarchical Structures ================= In these examples, we show how :func:`tyro.cli` can be used to instantiate -nested structures. This can enable modular, reusable, and composable CLI +hierarchical structures. This can enable modular, reusable, and composable CLI interfaces. @@ -127,7 +127,7 @@ Structures can also be used as input to functions. config: Experiment configuration. """ print(f"Saving to: {out_dir}") - print(f"Config: f{config}") + print(f"Config: {config}") if __name__ == "__main__": tyro.cli(train) @@ -169,7 +169,7 @@ Structures can also be used as input to functions. 
     $ python ./02_nesting_in_func.py --out-dir /tmp/test1
     Saving to: /tmp/test1
-    Config: fConfig(optimizer=OptimizerConfig(learning_rate=0.0003, weight_decay=0.01), seed=0)
+    Config: Config(optimizer=OptimizerConfig(learning_rate=0.0003, weight_decay=0.01), seed=0)
@@ -179,7 +179,7 @@ Structures can also be used as input to functions. 
     $ python ./02_nesting_in_func.py --out-dir /tmp/test2 --config.seed 4
     Saving to: /tmp/test2
-    Config: fConfig(optimizer=OptimizerConfig(learning_rate=0.0003, weight_decay=0.01), seed=4)
+    Config: Config(optimizer=OptimizerConfig(learning_rate=0.0003, weight_decay=0.01), seed=4)
.. _example-03_nesting_containers: diff --git a/docs/source/index.md b/docs/source/index.md index cb95fefdc..e95497ef1 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -97,7 +97,7 @@ shell completion. :titlesonly: ./examples/basics.rst - ./examples/nested_structures.rst + ./examples/hierarchical_structures.rst ./examples/subcommands.rst ./examples/overriding_configs.rst ./examples/generics.rst diff --git a/examples/02_nested_structures/01_nesting.py b/examples/02_hierarchical_structures/01_nesting.py similarity index 100% rename from examples/02_nested_structures/01_nesting.py rename to examples/02_hierarchical_structures/01_nesting.py diff --git a/examples/02_nested_structures/02_nesting_in_func.py b/examples/02_hierarchical_structures/02_nesting_in_func.py similarity index 96% rename from examples/02_nested_structures/02_nesting_in_func.py rename to examples/02_hierarchical_structures/02_nesting_in_func.py index da0c81f09..893994463 100644 --- a/examples/02_nested_structures/02_nesting_in_func.py +++ b/examples/02_hierarchical_structures/02_nesting_in_func.py @@ -41,7 +41,7 @@ def train( config: Experiment configuration. """ print(f"Saving to: {out_dir}") - print(f"Config: f{config}") + print(f"Config: {config}") if __name__ == "__main__": diff --git a/examples/02_nested_structures/03_nesting_containers.py b/examples/02_hierarchical_structures/03_nesting_containers.py similarity index 100% rename from examples/02_nested_structures/03_nesting_containers.py rename to examples/02_hierarchical_structures/03_nesting_containers.py diff --git a/examples/02_nested_structures/04_dictionaries.py b/examples/02_hierarchical_structures/04_dictionaries.py similarity index 100% rename from examples/02_nested_structures/04_dictionaries.py rename to examples/02_hierarchical_structures/04_dictionaries.py diff --git a/examples/02_nested_structures/05_tuples.py b/examples/02_hierarchical_structures/05_tuples.py similarity index 100% rename from examples/02_nested_structures/05_tuples.py rename to examples/02_hierarchical_structures/05_tuples.py diff --git a/examples/02_nested_structures/06_pydantic.py b/examples/02_hierarchical_structures/06_pydantic.py similarity index 100% rename from examples/02_nested_structures/06_pydantic.py rename to examples/02_hierarchical_structures/06_pydantic.py diff --git a/examples/02_nested_structures/07_attrs.py b/examples/02_hierarchical_structures/07_attrs.py similarity index 100% rename from examples/02_nested_structures/07_attrs.py rename to examples/02_hierarchical_structures/07_attrs.py diff --git a/examples/02_nested_structures/README.rst b/examples/02_hierarchical_structures/README.rst similarity index 50% rename from examples/02_nested_structures/README.rst rename to examples/02_hierarchical_structures/README.rst index 8ec6ad9b9..d4ffcfa5d 100644 --- a/examples/02_nested_structures/README.rst +++ b/examples/02_hierarchical_structures/README.rst @@ -1,6 +1,6 @@ -Nested Structures +Hierarchical Structures ================= In these examples, we show how :func:`tyro.cli` can be used to instantiate -nested structures. This can enable modular, reusable, and composable CLI +hierarchical structures. This can enable modular, reusable, and composable CLI interfaces. diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index e63f11077..317ad1b79 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -21,9 +21,10 @@ import pytest import torch -import tyro from typing_extensions import Annotated, Final, Literal, TypeAlias +import tyro + def test_no_args() -> None: def main() -> int: From 13d622ebb69f693a66ab500a238112e476abd71d Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Tue, 12 Nov 2024 20:43:08 -0800 Subject: [PATCH 478/491] Support Python 3.13 (#200) * Support Python 3.13 * Update pyproject.toml * Test annotated struct narrowing * ruff * mypy * Add narrow_subtype test * ruff --- pyproject.toml | 5 +- src/tyro/_fields.py | 10 ++-- src/tyro/_parsers.py | 10 ++-- src/tyro/_resolver.py | 20 ++++---- src/tyro/conf/_markers.py | 2 +- src/tyro/constructors/_struct_spec.py | 4 +- src/tyro/extras/_base_configs.py | 4 +- tests/conftest.py | 14 +++--- ...test_base_configs_nested_exclude_py313.py} | 0 tests/test_dcargs.py | 20 -------- ...py => test_flax_min_py38_exclude_py313.py} | 0 tests/test_helptext.py | 22 --------- tests/test_nested.py | 28 +++++++++++ ...configs_nested_exclude_py313_generated.py} | 0 .../test_dcargs_generated.py | 30 ++++-------- ..._flax_min_py38_exclude_py313_generated.py} | 0 .../test_helptext_generated.py | 22 --------- .../test_nested_generated.py | 28 +++++++++++ .../test_torch_exclude_py313_generated.py | 46 +++++++++++++++++++ tests/test_torch_exclude_py313.py | 46 +++++++++++++++++++ 20 files changed, 189 insertions(+), 122 deletions(-) rename tests/{test_base_configs_nested.py => test_base_configs_nested_exclude_py313.py} (100%) rename tests/{test_flax_min_py38.py => test_flax_min_py38_exclude_py313.py} (100%) rename tests/test_py311_generated/{test_base_configs_nested_generated.py => test_base_configs_nested_exclude_py313_generated.py} (100%) rename tests/test_py311_generated/{test_flax_min_py38_generated.py => test_flax_min_py38_exclude_py313_generated.py} (100%) create mode 100644 tests/test_py311_generated/test_torch_exclude_py313_generated.py create mode 100644 tests/test_torch_exclude_py313.py diff --git a/pyproject.toml b/pyproject.toml index 5ba0d3cae..fffa4ffe4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent" ] @@ -45,14 +46,14 @@ dev = [ "pytest-cov>=3.0.0", "omegaconf>=2.2.2", "attrs>=21.4.0", - "torch>=1.10.0", + "torch>=1.10.0;python_version<='3.12'", "pyright>=1.1.349,!=1.1.379", "ruff>=0.1.13", "mypy>=1.4.1", "numpy>=1.20.0", # As of 7/27/2023, flax install fails for Python 3.7 without pinning to an # old version. But doing so breaks other Python versions. - "flax>=0.6.9;python_version>='3.8'", + "flax>=0.6.9;python_version>='3.8' and python_version<='3.12'", "pydantic>=2.5.2", "coverage[toml]>=6.5.0", "eval_type_backport>=0.1.3", diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py index 4e786c5f4..03e7d1a93 100644 --- a/src/tyro/_fields.py +++ b/src/tyro/_fields.py @@ -186,14 +186,12 @@ def with_new_type_stripped( self, new_type_stripped: TypeForm[Any] | Callable ) -> FieldDefinition: if get_origin(self.type) is Annotated: - new_type = Annotated.__class_getitem__( # type: ignore - (new_type_stripped, *get_args(self.type)[1:]) - ) + new_type = Annotated[(new_type_stripped, *get_args(self.type)[1:])] # type: ignore else: - new_type = new_type_stripped + new_type = new_type_stripped # type: ignore return dataclasses.replace( self, - type=new_type, + type=new_type, # type: ignore type_stripped=new_type_stripped, ) @@ -400,7 +398,7 @@ def _field_list_from_function( # param.annotation doesn't resolve forward references. typ=typ if default_instance in MISSING_SINGLETONS - else Annotated.__class_getitem__((typ, _markers._OPTIONAL_GROUP)), # type: ignore + else Annotated[(typ, _markers._OPTIONAL_GROUP)], # type: ignore default=default, is_default_from_default_instance=False, helptext=helptext, diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py index edfceb72a..41a9d84b7 100644 --- a/src/tyro/_parsers.py +++ b/src/tyro/_parsers.py @@ -434,12 +434,12 @@ def from_field( len(found_subcommand_configs) > 0 and found_subcommand_configs[0].constructor_factory is not None ): - options[i] = Annotated.__class_getitem__( # type: ignore + options[i] = Annotated[ # type: ignore ( found_subcommand_configs[0].constructor_factory(), *_resolver.unwrap_annotated(option, "all")[1], ) - ) + ] # Exit if we don't contain any nested types. if not any( @@ -541,13 +541,11 @@ def from_field( if len(annotations) == 0: option = option_origin else: - option = Annotated.__class_getitem__( # type: ignore - (option_origin,) + annotations - ) + option = Annotated[(option_origin,) + annotations] # type: ignore with _fields.FieldDefinition.marker_context(tuple(field.markers)): subparser = ParserSpecification.from_callable_or_type( - option, + option, # type: ignore markers=field.markers, description=subcommand_config.description, parent_classes=parent_classes, diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py index 2473447fe..f045a0cfc 100644 --- a/src/tyro/_resolver.py +++ b/src/tyro/_resolver.py @@ -137,12 +137,12 @@ def resolve_newtype_and_aliases( ) -> TypeOrCallableOrNone: # Handle type aliases, eg via the `type` statement in Python 3.12. if isinstance(typ, TypeAliasType): - return Annotated.__class_getitem__( + return Annotated[ ( cast(Any, resolve_newtype_and_aliases(typ.__value__)), TyroTypeAliasBreadCrumb(typ.__name__), ) - ) + ] # We'll unwrap NewType annotations here; this is needed before issubclass # checks! @@ -156,7 +156,7 @@ def resolve_newtype_and_aliases( typ = resolve_newtype_and_aliases(getattr(typ, "__supertype__")) if return_name is not None: - typ = Annotated.__class_getitem__((typ, TyroTypeAliasBreadCrumb(return_name))) # type: ignore + typ = Annotated[(typ, TyroTypeAliasBreadCrumb(return_name))] # type: ignore return cast(TypeOrCallableOrNone, typ) @@ -192,9 +192,7 @@ def narrow_subtypes( if superclass is Any or issubclass(potential_subclass, superclass): # type: ignore if get_origin(typ) is Annotated: - return Annotated.__class_getitem__( # type: ignore - (potential_subclass,) + get_args(typ)[1:] - ) + return Annotated[(potential_subclass,) + get_args(typ)[1:]] # type: ignore typ = cast(TypeOrCallable, potential_subclass) except TypeError: # TODO: document where this TypeError can be raised, and reduce the amount of @@ -221,9 +219,7 @@ def swap_type_using_confstruct(typ: TypeOrCallable) -> TypeOrCallable: ) and anno.constructor_factory is not None ): - return Annotated.__class_getitem__( # type: ignore - (anno.constructor_factory(),) + annotations - ) + return Annotated[(anno.constructor_factory(),) + annotations] # type: ignore return typ @@ -589,7 +585,7 @@ def resolve_generic_types( return typ, type_from_typevar else: return ( - Annotated.__class_getitem__((typ, *annotations)), # type: ignore + Annotated[(typ, *annotations)], # type: ignore type_from_typevar, ) @@ -625,12 +621,12 @@ def get_type_hints_with_backported_syntax( non_none = args[1] if args[0] is NoneType else args[0] if get_origin(non_none) is Annotated: annotated_args = get_args(non_none) - out[k] = Annotated.__class_getitem__( # type: ignore + out[k] = Annotated[ # type: ignore ( Union.__getitem__((annotated_args[0], None)), # type: ignore *annotated_args[1:], ) - ) + ] return out except TypeError as e: # pragma: no cover diff --git a/src/tyro/conf/_markers.py b/src/tyro/conf/_markers.py index 4e6bf1b0b..227da7d46 100644 --- a/src/tyro/conf/_markers.py +++ b/src/tyro/conf/_markers.py @@ -170,7 +170,7 @@ class Args: class _Marker(_singleton.Singleton): def __getitem__(self, key): - return Annotated.__class_getitem__((key, self)) # type: ignore + return Annotated[(key, self)] # type: ignore Marker = Any diff --git a/src/tyro/constructors/_struct_spec.py b/src/tyro/constructors/_struct_spec.py index 1da5502da..d826aef29 100644 --- a/src/tyro/constructors/_struct_spec.py +++ b/src/tyro/constructors/_struct_spec.py @@ -531,9 +531,9 @@ def pydantic_rule(info: StructTypeInfo) -> StructConstructorSpec | None: StructFieldSpec( name=name, type=( - Annotated.__class_getitem__( # type: ignore + Annotated[ # type: ignore (pd2_field.annotation,) + tuple(pd2_field.metadata) - ) + ] if len(pd2_field.metadata) > 0 else pd2_field.annotation ), diff --git a/src/tyro/extras/_base_configs.py b/src/tyro/extras/_base_configs.py index 44f90efa5..51b8bbe81 100644 --- a/src/tyro/extras/_base_configs.py +++ b/src/tyro/extras/_base_configs.py @@ -134,7 +134,7 @@ def subcommand_type_from_defaults( assert len(defaults) >= 2, "At least two subcommands are required." return Union.__getitem__( # type: ignore tuple( - Annotated.__class_getitem__( # type: ignore + Annotated[ # type: ignore ( type(v), tyro.conf.subcommand( @@ -144,7 +144,7 @@ def subcommand_type_from_defaults( prefix_name=prefix_names, ), ) - ) + ] for k, v in defaults.items() ) ) diff --git a/tests/conftest.py b/tests/conftest.py index 23d01aae4..b4605fec8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,20 +4,20 @@ collect_ignore_glob: List[str] = [] if not sys.version_info >= (3, 8): - collect_ignore_glob.append("*_min_py38.py") - collect_ignore_glob.append("*_min_py38_generated.py") + collect_ignore_glob.append("*min_py38*.py") if not sys.version_info >= (3, 9): - collect_ignore_glob.append("*_min_py39.py") - collect_ignore_glob.append("*_min_py39_generated.py") + collect_ignore_glob.append("*min_py39*.py") if not sys.version_info >= (3, 10): - collect_ignore_glob.append("*_min_py310.py") + collect_ignore_glob.append("*min_py310*.py") collect_ignore_glob.append("*_min_py310_generated.py") if not sys.version_info >= (3, 12): - collect_ignore_glob.append("*_min_py312.py") - collect_ignore_glob.append("*_min_py312_generated.py") + collect_ignore_glob.append("*min_py312*.py") if not sys.version_info >= (3, 11): collect_ignore_glob.append("test_py311_generated/*.py") + +if sys.version_info >= (3, 13): + collect_ignore_glob.append("*_exclude_py313*.py") diff --git a/tests/test_base_configs_nested.py b/tests/test_base_configs_nested_exclude_py313.py similarity index 100% rename from tests/test_base_configs_nested.py rename to tests/test_base_configs_nested_exclude_py313.py diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py index 317ad1b79..b82543d1c 100644 --- a/tests/test_dcargs.py +++ b/tests/test_dcargs.py @@ -20,7 +20,6 @@ ) import pytest -import torch from typing_extensions import Annotated, Final, Literal, TypeAlias import tyro @@ -608,25 +607,6 @@ def test_missing_singleton() -> None: assert tyro.MISSING is copy.deepcopy(tyro.MISSING) -def test_torch_device() -> None: - def main(device: torch.device) -> torch.device: - return device - - assert tyro.cli(main, args=["--device", "cpu"]) == torch.device("cpu") - - -def test_supports_inference_mode_decorator() -> None: - @torch.inference_mode() - def main(x: int, device: str) -> Tuple[int, str]: - return x, device - - assert tyro.cli(main, args="--x 3 --device cuda".split(" ")) == (3, "cuda") - - -def test_torch_device_2() -> None: - assert tyro.cli(torch.device, args=["cpu"]) == torch.device("cpu") - - def test_just_int() -> None: assert tyro.cli(int, args=["123"]) == 123 diff --git a/tests/test_flax_min_py38.py b/tests/test_flax_min_py38_exclude_py313.py similarity index 100% rename from tests/test_flax_min_py38.py rename to tests/test_flax_min_py38_exclude_py313.py diff --git a/tests/test_helptext.py b/tests/test_helptext.py index 2301285c2..768da2f6f 100644 --- a/tests/test_helptext.py +++ b/tests/test_helptext.py @@ -3,11 +3,9 @@ import json import os import pathlib -from collections.abc import Callable from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar, Union, cast from helptext_utils import get_helptext_with_checks -from torch import nn from typing_extensions import Annotated, Literal, NotRequired, TypedDict import tyro @@ -662,26 +660,6 @@ class Something( assert "But this text should!" in helptext -def test_unparsable() -> None: - class Struct: - a: int = 5 - b: str = "7" - - def main(x: Any = Struct()): - pass - - helptext = get_helptext_with_checks(main) - assert "--x {fixed}" not in helptext - - def main2(x: Callable = nn.ReLU): - pass - - helptext = get_helptext_with_checks(main2) - assert "--x {fixed}" in helptext - assert "(fixed to:" in helptext - assert "torch" in helptext - - def test_pathlike() -> None: def main(x: os.PathLike) -> None: pass diff --git a/tests/test_nested.py b/tests/test_nested.py index 98ec5cb6e..9bc1748bd 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -1231,3 +1231,31 @@ class Args: assert tyro.cli(Args, args=[]) == Args(A("hello")) assert "default: inner:alt" in get_helptext_with_checks(Args) + + +def test_annotated_narrow_0() -> None: + @dataclasses.dataclass + class A: ... + + @dataclasses.dataclass + class B(A): + x: int + + def main(x: Annotated[A, tyro.conf.OmitArgPrefixes] = B(x=3)) -> Any: + return x + + assert tyro.cli(main, args=[]) == B(x=3) + assert tyro.cli(main, args="--x 5".split(" ")) == B(x=5) + + +def test_annotated_narrow_1() -> None: + @dataclasses.dataclass + class A: ... + + @dataclasses.dataclass + class B(A): + x: int + + from tyro._resolver import narrow_subtypes + + assert narrow_subtypes(Annotated[A, False], B(3)) == Annotated[B, False] # type: ignore diff --git a/tests/test_py311_generated/test_base_configs_nested_generated.py b/tests/test_py311_generated/test_base_configs_nested_exclude_py313_generated.py similarity index 100% rename from tests/test_py311_generated/test_base_configs_nested_generated.py rename to tests/test_py311_generated/test_base_configs_nested_exclude_py313_generated.py diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py index 7f45093be..909b3a038 100644 --- a/tests/test_py311_generated/test_dcargs_generated.py +++ b/tests/test_py311_generated/test_dcargs_generated.py @@ -16,13 +16,13 @@ List, Literal, Optional, + Text, Tuple, TypeAlias, TypeVar, ) import pytest -import torch import tyro @@ -575,6 +575,15 @@ def main(x: AnyStr) -> AnyStr: assert tyro.cli(main, args=["--x", "hello„"]) == "hello„" +def test_text() -> None: + # `Text` is an alias for `str` in Python 3. + def main(x: Text) -> Text: + return x + + assert tyro.cli(main, args=["--x", "hello"]) == "hello" + assert tyro.cli(main, args=["--x", "hello„"]) == "hello„" + + def test_fixed() -> None: def main(x: Callable[[int], int] = lambda x: x * 2) -> Callable[[int], int]: return x @@ -600,25 +609,6 @@ def test_missing_singleton() -> None: assert tyro.MISSING is copy.deepcopy(tyro.MISSING) -def test_torch_device() -> None: - def main(device: torch.device) -> torch.device: - return device - - assert tyro.cli(main, args=["--device", "cpu"]) == torch.device("cpu") - - -def test_supports_inference_mode_decorator() -> None: - @torch.inference_mode() - def main(x: int, device: str) -> Tuple[int, str]: - return x, device - - assert tyro.cli(main, args="--x 3 --device cuda".split(" ")) == (3, "cuda") - - -def test_torch_device_2() -> None: - assert tyro.cli(torch.device, args=["cpu"]) == torch.device("cpu") - - def test_just_int() -> None: assert tyro.cli(int, args=["123"]) == 123 diff --git a/tests/test_py311_generated/test_flax_min_py38_generated.py b/tests/test_py311_generated/test_flax_min_py38_exclude_py313_generated.py similarity index 100% rename from tests/test_py311_generated/test_flax_min_py38_generated.py rename to tests/test_py311_generated/test_flax_min_py38_exclude_py313_generated.py diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py index 3d0f8fb80..92eac7b57 100644 --- a/tests/test_py311_generated/test_helptext_generated.py +++ b/tests/test_py311_generated/test_helptext_generated.py @@ -3,7 +3,6 @@ import json import os import pathlib -from collections.abc import Callable from typing import ( Annotated, Any, @@ -20,7 +19,6 @@ ) from helptext_utils import get_helptext_with_checks -from torch import nn import tyro @@ -663,26 +661,6 @@ class Something( assert "But this text should!" in helptext -def test_unparsable() -> None: - class Struct: - a: int = 5 - b: str = "7" - - def main(x: Any = Struct()): - pass - - helptext = get_helptext_with_checks(main) - assert "--x {fixed}" not in helptext - - def main2(x: Callable = nn.ReLU): - pass - - helptext = get_helptext_with_checks(main2) - assert "--x {fixed}" in helptext - assert "(fixed to:" in helptext - assert "torch" in helptext - - def test_pathlike() -> None: def main(x: os.PathLike) -> None: pass diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py index 5a611b42c..973245446 100644 --- a/tests/test_py311_generated/test_nested_generated.py +++ b/tests/test_py311_generated/test_nested_generated.py @@ -1241,3 +1241,31 @@ class Args: assert tyro.cli(Args, args=[]) == Args(A("hello")) assert "default: inner:alt" in get_helptext_with_checks(Args) + + +def test_annotated_narrow_0() -> None: + @dataclasses.dataclass + class A: ... + + @dataclasses.dataclass + class B(A): + x: int + + def main(x: Annotated[A, tyro.conf.OmitArgPrefixes] = B(x=3)) -> Any: + return x + + assert tyro.cli(main, args=[]) == B(x=3) + assert tyro.cli(main, args="--x 5".split(" ")) == B(x=5) + + +def test_annotated_narrow_1() -> None: + @dataclasses.dataclass + class A: ... + + @dataclasses.dataclass + class B(A): + x: int + + from tyro._resolver import narrow_subtypes + + assert narrow_subtypes(Annotated[A, False], B(3)) == Annotated[B, False] # type: ignore diff --git a/tests/test_py311_generated/test_torch_exclude_py313_generated.py b/tests/test_py311_generated/test_torch_exclude_py313_generated.py new file mode 100644 index 000000000..8284a1634 --- /dev/null +++ b/tests/test_py311_generated/test_torch_exclude_py313_generated.py @@ -0,0 +1,46 @@ +from typing import Any, Callable, Tuple + +import torch +from helptext_utils import get_helptext_with_checks +from torch import nn + +import tyro + + +def test_torch_device() -> None: + def main(device: torch.device) -> torch.device: + return device + + assert tyro.cli(main, args=["--device", "cpu"]) == torch.device("cpu") + + +def test_supports_inference_mode_decorator() -> None: + @torch.inference_mode() + def main(x: int, device: str) -> Tuple[int, str]: + return x, device + + assert tyro.cli(main, args="--x 3 --device cuda".split(" ")) == (3, "cuda") + + +def test_torch_device_2() -> None: + assert tyro.cli(torch.device, args=["cpu"]) == torch.device("cpu") + + +def test_unparsable() -> None: + class Struct: + a: int = 5 + b: str = "7" + + def main(x: Any = Struct()): + pass + + helptext = get_helptext_with_checks(main) + assert "--x {fixed}" not in helptext + + def main2(x: Callable = nn.ReLU): + pass + + helptext = get_helptext_with_checks(main2) + assert "--x {fixed}" in helptext + assert "(fixed to:" in helptext + assert "torch" in helptext diff --git a/tests/test_torch_exclude_py313.py b/tests/test_torch_exclude_py313.py new file mode 100644 index 000000000..8284a1634 --- /dev/null +++ b/tests/test_torch_exclude_py313.py @@ -0,0 +1,46 @@ +from typing import Any, Callable, Tuple + +import torch +from helptext_utils import get_helptext_with_checks +from torch import nn + +import tyro + + +def test_torch_device() -> None: + def main(device: torch.device) -> torch.device: + return device + + assert tyro.cli(main, args=["--device", "cpu"]) == torch.device("cpu") + + +def test_supports_inference_mode_decorator() -> None: + @torch.inference_mode() + def main(x: int, device: str) -> Tuple[int, str]: + return x, device + + assert tyro.cli(main, args="--x 3 --device cuda".split(" ")) == (3, "cuda") + + +def test_torch_device_2() -> None: + assert tyro.cli(torch.device, args=["cpu"]) == torch.device("cpu") + + +def test_unparsable() -> None: + class Struct: + a: int = 5 + b: str = "7" + + def main(x: Any = Struct()): + pass + + helptext = get_helptext_with_checks(main) + assert "--x {fixed}" not in helptext + + def main2(x: Callable = nn.ReLU): + pass + + helptext = get_helptext_with_checks(main2) + assert "--x {fixed}" in helptext + assert "(fixed to:" in helptext + assert "torch" in helptext From 5eba34c7f7c7f5bebc14cc031651b87ca3d54c9a Mon Sep 17 00:00:00 2001 From: Brent Yi Date: Sat, 16 Nov 2024 12:09:45 -0500 Subject: [PATCH 479/491] More complete custom constructor docs, refinements (#201) * Constructors + examples refinements * formatting/regenerate docs * Move test * ruff * Python 3.7 * ruff --- README.md | 13 + docs/source/examples/custom_constructors.rst | 263 ++++++++++++++++-- docs/source/index.md | 34 +-- ...upported_types.rst => whats_supported.rst} | 5 +- .../01_simple_constructors.py | 43 +++ ...notation.py => 02_primitive_annotation.py} | 20 +- ...e_registry.py => 03_primitive_registry.py} | 13 +- .../04_struct_registry.py | 88 ++++++ examples/06_custom_constructors/README.rst | 13 +- src/tyro/_argparse_formatter.py | 4 +- src/tyro/_arguments.py | 21 +- src/tyro/_calling.py | 17 +- src/tyro/_fields.py | 16 +- src/tyro/_parsers.py | 11 +- src/tyro/_resolver.py | 4 +- src/tyro/_singleton.py | 69 +++-- src/tyro/_subcommand_matching.py | 6 +- src/tyro/constructors/__init__.py | 2 + src/tyro/constructors/_struct_spec.py | 55 ++-- src/tyro/extras/_serialization.py | 8 +- tests/test_custom_constructors.py | 30 +- .../test_conf_generated.py | 11 +- .../test_custom_constructors_generated.py | 31 ++- 23 files changed, 619 insertions(+), 158 deletions(-) rename docs/source/{supported_types.rst => whats_supported.rst} (92%) create mode 100644 examples/06_custom_constructors/01_simple_constructors.py rename examples/06_custom_constructors/{01_primitive_annotation.py => 02_primitive_annotation.py} (61%) rename examples/06_custom_constructors/{02_primitive_registry.py => 03_primitive_registry.py} (66%) create mode 100644 examples/06_custom_constructors/04_struct_registry.py diff --git a/README.md b/README.md index fd03bf0e3..0d09a5ae0 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,19 @@ Other features include helptext generation, nested structures, subcommands, and shell completion. For examples and the API reference, see our [documentation](https://brentyi.github.io/tyro). +### Why `tyro`? + +1. **Define things once.** Standard Python type annotations, docstrings, and default values are parsed to automatically generate command-line interfaces with informative helptext. + +2. **Static types.** Unlike tools dependent on dictionaries, YAML, or dynamic + namespaces, arguments populated by `tyro` benefit from IDE and language + server-supported operations — tab completion, rename, jump-to-def, + docstrings on hover — as well as static checking tools like `pyright` and + `mypy`. + +3. **Modularity.** `tyro` supports hierarchical configuration structures, which + make it easy to decentralize definitions, defaults, and documentation. + ### In the wild `tyro` is designed to be lightweight enough for throwaway scripts, while diff --git a/docs/source/examples/custom_constructors.rst b/docs/source/examples/custom_constructors.rst index 070cba69f..cd9eb459c 100644 --- a/docs/source/examples/custom_constructors.rst +++ b/docs/source/examples/custom_constructors.rst @@ -6,32 +6,116 @@ Custom constructors =================== -In these examples, we show how custom types can be parsed by and registered with :func:`tyro.cli`. +:func:`tyro.cli` is designed for comprehensive support of standard Python type +constructs. In some cases, however, it can be useful to extend the set of types +supported by :mod:`tyro`. +We provide two complementary approaches for doing so: + +- :mod:`tyro.conf` provides a simple API for specifying custom constructor + functions. +- :mod:`tyro.constructors` provides a more flexible API for defining behavior + for different types. There are two categories of types: *primitive* types are + instantiated from a single commandline argument, while *struct* types are + broken down into multiple arguments. + + +.. _example-01_simple_constructors: + +Simple Constructors +------------------- + +For simple custom constructors, we can pass a constructor function into +:func:`tyro.conf.arg` or :func:`tyro.conf.subcommand`. Arguments for will be +generated by parsing the signature of the constructor function. + +In this example, we use this pattern to define custom behavior for +instantiating a NumPy array. + + +.. code-block:: python + :linenos: + + # 01_simple_constructors.py + from typing import Literal + + import numpy as np + from typing_extensions import Annotated + + import tyro + + def construct_array( + values: tuple[float, ...], dtype: Literal["float32", "float64"] = "float64" + ) -> np.ndarray: + """A custom constructor for 1D NumPy arrays.""" + return np.array( + values, + dtype={"float32": np.float32, "float64": np.float64}[dtype], + ) + + def main( + # We can specify a custom constructor for an argument in `tyro.conf.arg()`. + array: Annotated[np.ndarray, tyro.conf.arg(constructor=construct_array)], + ) -> None: + print(f"{array=}") + + if __name__ == "__main__": + tyro.cli(main) + + + + +.. raw:: html + + 
+    $ python ./01_simple_constructors.py --help
+    usage: 01_simple_constructors.py [-h] --array.values [FLOAT
+                                     [FLOAT ...]] [--array.dtype 
+    {float32,float64}]
+    
+    ╭─ options ─────────────────────────────────────────╮
+    │ -h, --help        show this help message and exit │
+    ╰───────────────────────────────────────────────────╯
+    ╭─ array options ───────────────────────────────────╮
+    │ A custom constructor for 1D NumPy arrays.         │
+    │ ─────────────────────────────────────────         │
+    │ --array.values [FLOAT [FLOAT ...]]                │
+    │                   (required)                      │
+    │ --array.dtype {float32,float64}                   │
+    │                   (default: float64)              │
+    ╰───────────────────────────────────────────────────╯
+
-.. _example-01_primitive_annotation: -Custom Primitive ----------------- -.. note:: +.. raw:: html - This is an advanced feature, which should not be needed for the vast - majority of use cases. If :mod:`tyro` is missing support for a built-in - Python type, please open an issue on `GitHub `_. + 
+    $ python ./01_simple_constructors.py --array.values 1 2 3
+    array=array([1., 2., 3.])
+
-For additional flexibility, :mod:`tyro.constructors` exposes tyro's API for -defining behavior for different types. There are two categories of types: -primitive types can be instantiated from a single commandline argument, while -struct types are broken down into multiple. -In this example, we attach a custom constructor via a runtime annotation. + +.. raw:: html + + 
+    $ python ./01_simple_constructors.py --array.values 1 2 3 4 5 --array.dtype float32
+    array=array([1., 2., 3., 4., 5.], dtype=float32)
+
+.. _example-02_primitive_annotation: + +Custom Primitive +---------------- + +In this example, we use :mod:`tyro.constructors` to attach a primitive +constructor via a runtime annotation. .. code-block:: python :linenos: - # 01_primitive_annotation.py + # 02_primitive_annotation.py import json from typing_extensions import Annotated @@ -75,8 +159,8 @@ In this example, we attach a custom constructor via a runtime annotation. .. raw:: html 
-    $ python ./01_primitive_annotation.py --help
-    usage: 01_primitive_annotation.py [-h] --dict1 JSON [--dict2 JSON]
+    $ python ./02_primitive_annotation.py --help
+    usage: 02_primitive_annotation.py [-h] --dict1 JSON [--dict2 JSON]
     
     ╭─ options ───────────────────────────────────────────╮
     │ -h, --help          show this help message and exit │
@@ -90,7 +174,7 @@ In this example, we attach a custom constructor via a runtime annotation.
 .. raw:: html
 
     
-    $ python ./01_primitive_annotation.py --dict1 '{"hello": "world"}'
+    $ python ./02_primitive_annotation.py --dict1 '{"hello": "world"}'
     dict1={'hello': 'world'}
     dict2={'default': None}
     

@@ -100,30 +184,32 @@ In this example, we attach a custom constructor via a runtime annotation.
 .. raw:: html
 
     
-    $ python ./01_primitive_annotation.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
+    $ python ./02_primitive_annotation.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
     dict1={'hello': 'world'}
     dict2={'hello': 'world'}
     

-.. _example-02_primitive_registry:
+.. _example-03_primitive_registry:
 
 Custom Primitive (Registry)
 ---------------------------
 
-In this example, we use :class:`tyro.constructors.PrimitiveConstructorSpec` to
+In this example, we use a :class:`tyro.constructors.ConstructorRegistry` to
 define a rule that applies to all types that match ``dict[str, Any]``.
 
 
 .. code-block:: python
     :linenos:
 
-    # 02_primitive_registry.py
+    # 03_primitive_registry.py
     import json
     from typing import Any
 
     import tyro
 
+    # Create a custom registry, which stores constructor rules.
     custom_registry = tyro.constructors.ConstructorRegistry()
 
+    # Define a rule that applies to all types that match `dict[str, Any]`.
     @custom_registry.primitive_rule
     def _(
         type_info: tyro.constructors.PrimitiveTypeInfo,
@@ -145,11 +231,134 @@ define a rule that applies to all types that match ``dict[str, Any]``.
         dict1: dict[str, Any],
         dict2: dict[str, Any] = {"default": None},
     ) -> None:
+        """A function with two arguments, which can be populated from the CLI via JSON."""
         print(f"{dict1=}")
         print(f"{dict2=}")
 
     if __name__ == "__main__":
-        # The custom registry is used as a context.
+        # To activate a custom registry, we should use it as a context manager.
+        with custom_registry:
+            tyro.cli(main)
+
+
+
+
+.. raw:: html
+
+    
+    $ python ./03_primitive_registry.py --help
+    usage: 03_primitive_registry.py [-h] --dict1 JSON [--dict2 JSON]
+    
+    A function with two arguments, which can be populated from the CLI via JSON.
+    
+    ╭─ options ───────────────────────────────────────────╮
+    │ -h, --help          show this help message and exit │
+    │ --dict1 JSON        (required)                      │
+    │ --dict2 JSON        (default: '{"default": null}')  │
+    ╰─────────────────────────────────────────────────────╯
+    

+
+
+
+.. raw:: html
+
+    
+    $ python ./03_primitive_registry.py --dict1 '{"hello": "world"}'
+    dict1={'hello': 'world'}
+    dict2={'default': None}
+    

+
+
+
+.. raw:: html
+
+    
+    $ python ./03_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
+    dict1={'hello': 'world'}
+    dict2={'hello': 'world'}
+    

+.. _example-04_struct_registry:
+
+Custom Structs (Registry)
+-------------------------
+
+In this example, we use a :class:`tyro.constructors.ConstructorRegistry` to
+add support for a custom type.
+
+.. warning::
+
+    This will be complicated!
+
+
+.. code-block:: python
+    :linenos:
+
+    # 04_struct_registry.py
+    import tyro
+
+    # A custom type that we'll add support for to tyro.
+    class Bounds:
+        def __init__(self, lower: int, upper: int):
+            self.bounds = (lower, upper)
+
+    # Create a custom registry, which stores constructor rules.
+    custom_registry = tyro.constructors.ConstructorRegistry()
+
+    # Define a rule that applies to all types that match `Bounds`.
+    @custom_registry.struct_rule
+    def _(
+        type_info: tyro.constructors.StructTypeInfo,
+    ) -> tyro.constructors.StructConstructorSpec | None:
+        # We return `None` if the rule does not apply.
+        if type_info.type != Bounds:
+            return None
+
+        # We can extract the default value of the field from `type_info`.
+        if isinstance(type_info.default, Bounds):
+            # If the default value is a `Bounds` instance, we don't need to generate a constructor.
+            default = (type_info.default.bounds[0], type_info.default.bounds[1])
+            is_default_overridden = True
+        else:
+            # Otherwise, the default value is missing. We'll mark the child defaults as missing as well.
+            assert type_info.default in (
+                tyro.constructors.MISSING,
+                tyro.constructors.MISSING_NONPROP,
+            )
+            default = (tyro.MISSING, tyro.MISSING)
+            is_default_overridden = False
+
+        # If the rule applies, we return the constructor spec.
+        return tyro.constructors.StructConstructorSpec(
+            # The instantiate function will be called with the fields as keyword arguments.
+            instantiate=Bounds,
+            fields=(
+                tyro.constructors.StructFieldSpec(
+                    name="lower",
+                    type=int,
+                    default=default[0],
+                    is_default_overridden=is_default_overridden,
+                    helptext="Lower bound." "",
+                ),
+                tyro.constructors.StructFieldSpec(
+                    name="upper",
+                    type=int,
+                    default=default[1],
+                    is_default_overridden=is_default_overridden,
+                    helptext="Upper bound." "",
+                ),
+            ),
+        )
+
+    def main(
+        bounds: Bounds,
+        bounds_with_default: Bounds = Bounds(0, 100),
+    ) -> None:
+        """A function with two `Bounds` instances as input."""
+        print(f"{bounds=}")
+        print(f"{bounds_with_default=}")
+
+    if __name__ == "__main__":
+        # To activate a custom registry, we should use it as a context manager.
         with custom_registry:
             tyro.cli(main)
 
@@ -159,8 +368,10 @@ define a rule that applies to all types that match ``dict[str, Any]``.
 .. raw:: html
 
     
-    $ python ./02_primitive_registry.py --help
-    usage: 02_primitive_registry.py [-h] --dict1 JSON [--dict2 JSON]
+    $ python ./03_primitive_registry.py --help
+    usage: 03_primitive_registry.py [-h] --dict1 JSON [--dict2 JSON]
+    
+    A function with two arguments, which can be populated from the CLI via JSON.
     
     ╭─ options ───────────────────────────────────────────╮
     │ -h, --help          show this help message and exit │
@@ -174,7 +385,7 @@ define a rule that applies to all types that match ``dict[str, Any]``.
 .. raw:: html
 
     
-    $ python ./02_primitive_registry.py --dict1 '{"hello": "world"}'
+    $ python ./03_primitive_registry.py --dict1 '{"hello": "world"}'
     dict1={'hello': 'world'}
     dict2={'default': None}
     

@@ -184,7 +395,7 @@ define a rule that applies to all types that match ``dict[str, Any]``.
 .. raw:: html
 
     
-    $ python ./02_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
+    $ python ./03_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
     dict1={'hello': 'world'}
     dict2={'hello': 'world'}
     

\ No newline at end of file
diff --git a/docs/source/index.md b/docs/source/index.md
index e95497ef1..03f06c51c 100644
--- a/docs/source/index.md
+++ b/docs/source/index.md
@@ -54,30 +54,18 @@ shell completion.
 
 #### Why `tyro`?
 
-1. **Types.**
+1. **Define things once.** Standard Python type annotations, docstrings, and
+   default values are parsed to automatically generate command-line interfaces
+   with informative helptext.
 
-   Unlike tools dependent on dictionaries, YAML, or dynamic namespaces,
-   arguments populated by `tyro` benefit from IDE and language server-supported
-   operations — think tab completion, rename, jump-to-def, docstrings on hover —
-   as well as static checking tools like `pyright` and `mypy`.
+2. **Static types.** Unlike tools dependent on dictionaries, YAML, or dynamic
+   namespaces, arguments populated by `tyro` benefit from IDE and language
+   server-supported operations — tab completion, rename, jump-to-def,
+   docstrings on hover — as well as static checking tools like `pyright` and
+   `mypy`.
 
-2. **Define things once.**
-
-   Standard Python type annotations, docstrings, and default values are parsed
-   to automatically generate command-line interfaces with informative helptext.
-
-   `tyro` works seamlessly with tools you already use: examples are included for
-   [dataclasses](https://docs.python.org/3/library/dataclasses.html),
-   [attrs](https://www.attrs.org/),
-   [pydantic](https://pydantic-docs.helpmanual.io/),
-   [flax.linen](https://flax.readthedocs.io/en/latest/api_reference/flax.linen.html),
-   and more.
-
-3. **Modularity.**
-
-   `tyro` supports hierarchical configuration structures, which make it easy to
-   distribute definitions, defaults, and documentation of configurable fields
-   across modules or source files.
+3. **Modularity.** `tyro` supports hierarchical configuration structures, which
+   make it easy to decentralize definitions, defaults, and documentation.
 
 
 
@@ -89,7 +77,7 @@ shell completion.
 
    installation
    your_first_cli
-   supported_types
+   whats_supported
 
 .. toctree::
    :caption: Examples
diff --git a/docs/source/supported_types.rst b/docs/source/whats_supported.rst
similarity index 92%
rename from docs/source/supported_types.rst
rename to docs/source/whats_supported.rst
index 772d1e95f..fa05cb2b2 100644
--- a/docs/source/supported_types.rst
+++ b/docs/source/whats_supported.rst
@@ -1,7 +1,7 @@
 What's supported
 ================
 
-For minimum-boilerplate CLIs, `tyro` aims to maximize support of
+For minimum-boilerplate CLIs, :mod:`tyro` aims to maximize support of
 Python's standard :mod:`typing` features.
 
 As a partial list, inputs can be annotated with:
@@ -13,9 +13,8 @@ As a partial list, inputs can be annotated with:
 - :py:data:`typing.Literal` and :class:`enum.Enum`.
 - Type aliases, for example using Python 3.12's `PEP 695 `_ `type` statement.
 - Generics, such as those annotated with :py:class:`typing.TypeVar` or with the type parameter syntax introduced by Python 3.12's `PEP 695 `_.
-- etc
+- Compositions of the above types, like ``tuple[int | str, ...] | None``.
 
-Compositions of the above types, like ``tuple[int | str, ...] | None``, are also supported.
 
 Types can also be placed and nested in various structures, such as:
 
diff --git a/examples/06_custom_constructors/01_simple_constructors.py b/examples/06_custom_constructors/01_simple_constructors.py
new file mode 100644
index 000000000..9c144e258
--- /dev/null
+++ b/examples/06_custom_constructors/01_simple_constructors.py
@@ -0,0 +1,43 @@
+"""Simple Constructors
+
+For simple custom constructors, we can pass a constructor function into
+:func:`tyro.conf.arg` or :func:`tyro.conf.subcommand`. Arguments for will be
+generated by parsing the signature of the constructor function.
+
+In this example, we use this pattern to define custom behavior for
+instantiating a NumPy array.
+
+Usage:
+
+    python ./01_simple_constructors.py --help
+    python ./01_simple_constructors.py --array.values 1 2 3
+    python ./01_simple_constructors.py --array.values 1 2 3 4 5 --array.dtype float32
+"""
+
+from typing import Literal
+
+import numpy as np
+from typing_extensions import Annotated
+
+import tyro
+
+
+def construct_array(
+    values: tuple[float, ...], dtype: Literal["float32", "float64"] = "float64"
+) -> np.ndarray:
+    """A custom constructor for 1D NumPy arrays."""
+    return np.array(
+        values,
+        dtype={"float32": np.float32, "float64": np.float64}[dtype],
+    )
+
+
+def main(
+    # We can specify a custom constructor for an argument in `tyro.conf.arg()`.
+    array: Annotated[np.ndarray, tyro.conf.arg(constructor=construct_array)],
+) -> None:
+    print(f"{array=}")
+
+
+if __name__ == "__main__":
+    tyro.cli(main)
diff --git a/examples/06_custom_constructors/01_primitive_annotation.py b/examples/06_custom_constructors/02_primitive_annotation.py
similarity index 61%
rename from examples/06_custom_constructors/01_primitive_annotation.py
rename to examples/06_custom_constructors/02_primitive_annotation.py
index e22fbca42..6495e68eb 100644
--- a/examples/06_custom_constructors/01_primitive_annotation.py
+++ b/examples/06_custom_constructors/02_primitive_annotation.py
@@ -1,23 +1,13 @@
 """Custom Primitive
 
-.. note::
-
-    This is an advanced feature, which should not be needed for the vast
-    majority of use cases. If :mod:`tyro` is missing support for a built-in
-    Python type, please open an issue on `GitHub `_.
-
-For additional flexibility, :mod:`tyro.constructors` exposes tyro's API for
-defining behavior for different types. There are two categories of types:
-primitive types can be instantiated from a single commandline argument, while
-struct types are broken down into multiple.
-
-In this example, we attach a custom constructor via a runtime annotation.
+In this example, we use :mod:`tyro.constructors` to attach a primitive
+constructor via a runtime annotation.
 
 Usage:
 
-    python ./01_primitive_annotation.py --help
-    python ./01_primitive_annotation.py --dict1 '{"hello": "world"}'
-    python ./01_primitive_annotation.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
+    python ./02_primitive_annotation.py --help
+    python ./02_primitive_annotation.py --dict1 '{"hello": "world"}'
+    python ./02_primitive_annotation.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
 """
 
 import json
diff --git a/examples/06_custom_constructors/02_primitive_registry.py b/examples/06_custom_constructors/03_primitive_registry.py
similarity index 66%
rename from examples/06_custom_constructors/02_primitive_registry.py
rename to examples/06_custom_constructors/03_primitive_registry.py
index 43589c235..ca309ec3a 100644
--- a/examples/06_custom_constructors/02_primitive_registry.py
+++ b/examples/06_custom_constructors/03_primitive_registry.py
@@ -1,13 +1,13 @@
 """Custom Primitive (Registry)
 
-In this example, we use :class:`tyro.constructors.PrimitiveConstructorSpec` to
+In this example, we use a :class:`tyro.constructors.ConstructorRegistry` to
 define a rule that applies to all types that match ``dict[str, Any]``.
 
 Usage:
 
-    python ./02_primitive_registry.py --help
-    python ./02_primitive_registry.py --dict1 '{"hello": "world"}'
-    python ./02_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
+    python ./03_primitive_registry.py --help
+    python ./03_primitive_registry.py --dict1 '{"hello": "world"}'
+    python ./03_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
 """
 
 import json
@@ -15,9 +15,11 @@
 
 import tyro
 
+# Create a custom registry, which stores constructor rules.
 custom_registry = tyro.constructors.ConstructorRegistry()
 
 
+# Define a rule that applies to all types that match `dict[str, Any]`.
 @custom_registry.primitive_rule
 def _(
     type_info: tyro.constructors.PrimitiveTypeInfo,
@@ -40,11 +42,12 @@ def main(
     dict1: dict[str, Any],
     dict2: dict[str, Any] = {"default": None},
 ) -> None:
+    """A function with two arguments, which can be populated from the CLI via JSON."""
     print(f"{dict1=}")
     print(f"{dict2=}")
 
 
 if __name__ == "__main__":
-    # The custom registry is used as a context.
+    # To activate a custom registry, we should use it as a context manager.
     with custom_registry:
         tyro.cli(main)
diff --git a/examples/06_custom_constructors/04_struct_registry.py b/examples/06_custom_constructors/04_struct_registry.py
new file mode 100644
index 000000000..e836a6278
--- /dev/null
+++ b/examples/06_custom_constructors/04_struct_registry.py
@@ -0,0 +1,88 @@
+"""Custom Structs (Registry)
+
+In this example, we use a :class:`tyro.constructors.ConstructorRegistry` to
+add support for a custom type.
+
+.. warning::
+
+    This will be complicated!
+
+Usage:
+
+    python ./03_primitive_registry.py --help
+    python ./03_primitive_registry.py --dict1 '{"hello": "world"}'
+    python ./03_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
+"""
+
+import tyro
+
+
+# A custom type that we'll add support for to tyro.
+class Bounds:
+    def __init__(self, lower: int, upper: int):
+        self.bounds = (lower, upper)
+
+
+# Create a custom registry, which stores constructor rules.
+custom_registry = tyro.constructors.ConstructorRegistry()
+
+
+# Define a rule that applies to all types that match `Bounds`.
+@custom_registry.struct_rule
+def _(
+    type_info: tyro.constructors.StructTypeInfo,
+) -> tyro.constructors.StructConstructorSpec | None:
+    # We return `None` if the rule does not apply.
+    if type_info.type != Bounds:
+        return None
+
+    # We can extract the default value of the field from `type_info`.
+    if isinstance(type_info.default, Bounds):
+        # If the default value is a `Bounds` instance, we don't need to generate a constructor.
+        default = (type_info.default.bounds[0], type_info.default.bounds[1])
+        is_default_overridden = True
+    else:
+        # Otherwise, the default value is missing. We'll mark the child defaults as missing as well.
+        assert type_info.default in (
+            tyro.constructors.MISSING,
+            tyro.constructors.MISSING_NONPROP,
+        )
+        default = (tyro.MISSING, tyro.MISSING)
+        is_default_overridden = False
+
+    # If the rule applies, we return the constructor spec.
+    return tyro.constructors.StructConstructorSpec(
+        # The instantiate function will be called with the fields as keyword arguments.
+        instantiate=Bounds,
+        fields=(
+            tyro.constructors.StructFieldSpec(
+                name="lower",
+                type=int,
+                default=default[0],
+                is_default_overridden=is_default_overridden,
+                helptext="Lower bound." "",
+            ),
+            tyro.constructors.StructFieldSpec(
+                name="upper",
+                type=int,
+                default=default[1],
+                is_default_overridden=is_default_overridden,
+                helptext="Upper bound." "",
+            ),
+        ),
+    )
+
+
+def main(
+    bounds: Bounds,
+    bounds_with_default: Bounds = Bounds(0, 100),
+) -> None:
+    """A function with two `Bounds` instances as input."""
+    print(f"{bounds=}")
+    print(f"{bounds_with_default=}")
+
+
+if __name__ == "__main__":
+    # To activate a custom registry, we should use it as a context manager.
+    with custom_registry:
+        tyro.cli(main)
diff --git a/examples/06_custom_constructors/README.rst b/examples/06_custom_constructors/README.rst
index f954f2cea..23698ee2b 100644
--- a/examples/06_custom_constructors/README.rst
+++ b/examples/06_custom_constructors/README.rst
@@ -1,4 +1,15 @@
 Custom constructors
 ===================
 
-In these examples, we show how custom types can be parsed by and registered with :func:`tyro.cli`.
+:func:`tyro.cli` is designed for comprehensive support of standard Python type
+constructs. In some cases, however, it can be useful to extend the set of types
+supported by :mod:`tyro`.
+
+We provide two complementary approaches for doing so:
+
+- :mod:`tyro.conf` provides a simple API for specifying custom constructor
+  functions.
+- :mod:`tyro.constructors` provides a more flexible API for defining behavior
+  for different types. There are two categories of types: *primitive* types are
+  instantiated from a single commandline argument, while *struct* types are
+  broken down into multiple arguments.
diff --git a/src/tyro/_argparse_formatter.py b/src/tyro/_argparse_formatter.py
index fda0df251..24baeb90f 100644
--- a/src/tyro/_argparse_formatter.py
+++ b/src/tyro/_argparse_formatter.py
@@ -285,9 +285,9 @@ def _check_value(self, action, value):
         This solves a choices error raised by argparse in a very specific edge case:
         literals in containers as positional arguments.
         """
-        from ._fields import MISSING_SINGLETONS
+        from ._fields import MISSING_AND_MISSING_NONPROP
 
-        if value in MISSING_SINGLETONS:
+        if value in MISSING_AND_MISSING_NONPROP:
             return
         return super()._check_value(action, value)
 
diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py
index 34198ff84..56e0e2b8e 100644
--- a/src/tyro/_arguments.py
+++ b/src/tyro/_arguments.py
@@ -237,7 +237,7 @@ def _rule_handle_boolean_flags(
         return
 
     if (
-        arg.field.default in _singleton.MISSING_SINGLETONS
+        arg.field.default in _singleton.MISSING_AND_MISSING_NONPROP
         or arg.field.is_positional()
         or _markers.FlagConversionOff in arg.field.markers
         or _markers.Fixed in arg.field.markers
@@ -276,7 +276,7 @@ def _rule_apply_primitive_specs(
         lowered.instance_from_str = None
         lowered.metavar = "{fixed}"
         lowered.required = False
-        lowered.default = _singleton.MISSING_PROP
+        lowered.default = _singleton.MISSING
         return
     if lowered.instance_from_str is not None:
         return
@@ -289,7 +289,7 @@ def _rule_apply_primitive_specs(
             )
         )
     except UnsupportedTypeAnnotationError as e:
-        if arg.field.default in _singleton.MISSING_SINGLETONS:
+        if arg.field.default in _singleton.MISSING_AND_MISSING_NONPROP:
             field_name = _strings.make_field_name(
                 [arg.extern_prefix, arg.field.intern_name]
             )
@@ -309,19 +309,19 @@ def _rule_apply_primitive_specs(
             # available.
             lowered.metavar = "{fixed}"
             lowered.required = False
-            lowered.default = _singleton.MISSING_PROP
+            lowered.default = _singleton.MISSING
             return
 
     # Mark lowered as required if a default is missing.
     if (
-        arg.field.default in _singleton.MISSING_SINGLETONS
+        arg.field.default in _singleton.MISSING_AND_MISSING_NONPROP
         and _markers._OPTIONAL_GROUP not in arg.field.markers
     ):
         lowered.default = None
         lowered.required = True
     elif (
         arg.field.default is not _singleton.EXCLUDE_FROM_CALL
-        and arg.field.default not in _singleton.MISSING_SINGLETONS
+        and arg.field.default not in _singleton.MISSING_AND_MISSING_NONPROP
     ):
         # Set default.
         lowered.default = spec.str_from_instance(arg.field.default)
@@ -342,7 +342,7 @@ def append_instantiator(x: list[list[str]]) -> Any:
             # Instantiate initial output.
             out = (
                 arg.field.default
-                if arg.field.default not in _singleton.MISSING_SINGLETONS
+                if arg.field.default not in _singleton.MISSING_AND_MISSING_NONPROP
                 else None
             )
             if out is None:
@@ -450,7 +450,7 @@ def _rule_generate_helptext(
         default = lowered.default
         if lowered.is_fixed() or lowered.action == "append":
             # Cases where we'll be missing the lowered default. Use field default instead.
-            assert default in _singleton.MISSING_SINGLETONS or default is None
+            assert default in _singleton.MISSING_AND_MISSING_NONPROP or default is None
             default = arg.field.default
 
         # Get the default value label.
@@ -488,7 +488,8 @@ def _rule_generate_helptext(
             # Repeatable argument.
             behavior_hint = "(repeatable)"
         elif lowered.action == "append" and (
-            default in _singleton.MISSING_SINGLETONS or len(cast(tuple, default)) == 0
+            default in _singleton.MISSING_AND_MISSING_NONPROP
+            or len(cast(tuple, default)) == 0
         ):
             behavior_hint = "(repeatable)"
         elif lowered.action == "append" and len(cast(tuple, default)) > 0:
@@ -499,7 +500,7 @@ def _rule_generate_helptext(
             behavior_hint = "(unset by default)"
         elif (
             _markers._OPTIONAL_GROUP in arg.field.markers
-            and default in _singleton.MISSING_SINGLETONS
+            and default in _singleton.MISSING_AND_MISSING_NONPROP
         ):
             # Argument in an optional group, but with no default. This is typically used
             # when general (non-argument, non-dataclass) object arguments are given a
diff --git a/src/tyro/_calling.py b/src/tyro/_calling.py
index 53c091a16..138263975 100644
--- a/src/tyro/_calling.py
+++ b/src/tyro/_calling.py
@@ -81,7 +81,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any:
             if not arg.lowered.is_fixed():
                 value = get_value_from_arg(name_maybe_prefixed)
 
-                if value in _fields.MISSING_SINGLETONS:
+                if value in _fields.MISSING_AND_MISSING_NONPROP:
                     value = arg.field.default
 
                     # Consider a function with a positional sequence argument:
@@ -92,7 +92,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any:
                     # as empty input for x. But the argparse default will be a MISSING
                     # value, and the field default will be inspect.Parameter.empty.
                     if (
-                        value in _fields.MISSING_SINGLETONS
+                        value in _fields.MISSING_AND_MISSING_NONPROP
                         and field.is_positional_call()
                         and arg.lowered.nargs in ("?", "*")
                     ):
@@ -113,10 +113,10 @@ def get_value_from_arg(prefixed_field_name: str) -> Any:
                             arg,
                         )
             else:
-                assert arg.field.default not in _fields.MISSING_SINGLETONS
+                assert arg.field.default not in _fields.MISSING_AND_MISSING_NONPROP
                 value = arg.field.default
                 parsed_value = value_from_prefixed_field_name.get(prefixed_field_name)
-                if parsed_value not in _fields.MISSING_SINGLETONS:
+                if parsed_value not in _fields.MISSING_AND_MISSING_NONPROP:
                     raise InstantiationError(
                         f"{arg.lowered.name_or_flag} was passed in, but"
                         " is a fixed argument that cannot be parsed",
@@ -146,7 +146,10 @@ def get_value_from_arg(prefixed_field_name: str) -> Any:
             if subparser_dest in value_from_prefixed_field_name:
                 subparser_name = get_value_from_arg(subparser_dest)
             else:
-                assert subparser_def.default_instance not in _fields.MISSING_SINGLETONS
+                assert (
+                    subparser_def.default_instance
+                    not in _fields.MISSING_AND_MISSING_NONPROP
+                )
                 subparser_name = None
 
             if subparser_name is None:
@@ -196,7 +199,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any:
 
     # Logic for _markers._OPTIONAL_GROUP.
     is_missing_list = [
-        v in _fields.MISSING_SINGLETONS
+        any(v is m for m in _fields.MISSING_AND_MISSING_NONPROP)
         for v in itertools.chain(positional_args, kwargs.values())
     ]
     if any(is_missing_list):
@@ -208,7 +211,7 @@ def get_value_from_arg(prefixed_field_name: str) -> Any:
         if len(kwargs) > 0:
             missing_args: List[str] = []
             for k, v in kwargs.items():
-                if v not in _fields.MISSING_SINGLETONS:
+                if v not in _fields.MISSING_AND_MISSING_NONPROP:
                     break
 
                 # Argument is missing.
diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py
index 03e7d1a93..e4701ae21 100644
--- a/src/tyro/_fields.py
+++ b/src/tyro/_fields.py
@@ -15,7 +15,11 @@
 from typing_extensions import Annotated, get_args, get_origin
 
 from . import _docstrings, _resolver, _strings, _unsafe_cache
-from ._singleton import DEFAULT_SENTINEL_SINGLETONS, MISSING_SINGLETONS
+from ._singleton import (
+    DEFAULT_SENTINEL_SINGLETONS,
+    MISSING_AND_MISSING_NONPROP,
+    MISSING_NONPROP,
+)
 from ._typing import TypeForm
 from .conf import _confstruct, _markers
 from .constructors._primitive_spec import (
@@ -57,7 +61,7 @@ class FieldDefinition:
     def __post_init__(self):
         if (
             _markers.Fixed in self.markers or _markers.Suppress in self.markers
-        ) and self.default in MISSING_SINGLETONS:
+        ) and self.default in MISSING_AND_MISSING_NONPROP:
             raise UnsupportedTypeAnnotationError(
                 f"Field {self.intern_name} is missing a default value!"
             )
@@ -95,7 +99,7 @@ def make(
         typ = _resolver.TypeParamResolver.concretize_type_params(typ)
 
         # Narrow types.
-        if typ is Any and default not in MISSING_SINGLETONS:
+        if typ is Any and default not in MISSING_AND_MISSING_NONPROP:
             typ = type(default)
         else:
             # TypeVar constraints are already applied in
@@ -205,7 +209,7 @@ def is_positional(self) -> bool:
             or (
                 # Make required arguments positional.
                 _markers.PositionalRequiredArgs in self.markers
-                and self.default in MISSING_SINGLETONS
+                and self.default in MISSING_AND_MISSING_NONPROP
             )
         )
 
@@ -397,9 +401,9 @@ def _field_list_from_function(
                     name=param.name,
                     # param.annotation doesn't resolve forward references.
                     typ=typ
-                    if default_instance in MISSING_SINGLETONS
+                    if default_instance in MISSING_AND_MISSING_NONPROP
                     else Annotated[(typ, _markers._OPTIONAL_GROUP)],  # type: ignore
-                    default=default,
+                    default=default if default is not param.empty else MISSING_NONPROP,
                     is_default_from_default_instance=False,
                     helptext=helptext,
                 )
diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py
index 41a9d84b7..b380f9b1f 100644
--- a/src/tyro/_parsers.py
+++ b/src/tyro/_parsers.py
@@ -478,7 +478,10 @@ def from_field(
             subcommand_type_from_name[subcommand_name] = cast(type, option)
 
         # If a field default is provided, try to find a matching subcommand name.
-        if field.default is None or field.default in _singleton.MISSING_SINGLETONS:
+        if (
+            field.default is None
+            or field.default in _singleton.MISSING_AND_MISSING_NONPROP
+        ):
             default_name = None
         else:
             default_name = _subcommand_matching.match_subcommand(
@@ -524,7 +527,7 @@ def from_field(
             # If names match, borrow subcommand default from field default.
             if default_name == subcommand_name and (
                 field.is_default_from_default_instance
-                or subcommand_config.default in _singleton.MISSING_SINGLETONS
+                or subcommand_config.default in _singleton.MISSING_AND_MISSING_NONPROP
             ):
                 subcommand_config = dataclasses.replace(
                     subcommand_config, default=field.default
@@ -567,7 +570,7 @@ def from_field(
             parser_from_name[subcommand_name] = subparser
 
         # Required if a default is missing.
-        required = field.default in _fields.MISSING_SINGLETONS
+        required = field.default in _fields.MISSING_AND_MISSING_NONPROP
 
         # Required if a default is passed in, but the default value has missing
         # parameters.
@@ -589,7 +592,7 @@ def from_field(
         description_parts = []
         if field.helptext is not None:
             description_parts.append(field.helptext)
-        if not required and field.default not in _fields.MISSING_SINGLETONS:
+        if not required and field.default not in _fields.MISSING_AND_MISSING_NONPROP:
             description_parts.append(f" (default: {default_name})")
         description = (
             # We use `None` instead of an empty string to prevent a line break from
diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py
index f045a0cfc..41a715448 100644
--- a/src/tyro/_resolver.py
+++ b/src/tyro/_resolver.py
@@ -38,7 +38,7 @@
 )
 
 from . import _unsafe_cache, conf
-from ._singleton import MISSING_SINGLETONS
+from ._singleton import MISSING_AND_MISSING_NONPROP
 from ._typing import TypeForm
 
 UnionType = getattr(types, "UnionType", Union)
@@ -469,7 +469,7 @@ def narrow_union_type(typ: TypeOrCallable, default_instance: Any) -> TypeOrCalla
     options_unwrapped = [unwrap_origin_strip_extras(o) for o in options]
 
     try:
-        if default_instance not in MISSING_SINGLETONS and not any(
+        if default_instance not in MISSING_AND_MISSING_NONPROP and not any(
             isinstance_with_fuzzy_numeric_tower(default_instance, o) is not False
             for o in options_unwrapped
         ):
diff --git a/src/tyro/_singleton.py b/src/tyro/_singleton.py
index 80d367d9a..d2462c762 100644
--- a/src/tyro/_singleton.py
+++ b/src/tyro/_singleton.py
@@ -1,5 +1,3 @@
-import dataclasses
-import inspect
 from typing import Any
 
 
@@ -37,8 +35,46 @@ class NotRequiredButWeDontKnowTheValueType(Singleton):
 # We have two types of missing sentinels: a propagating missing value, which when set as
 # a default will set all child values of nested structures as missing as well, and a
 # nonpropagating missing sentinel, which does not override child defaults.
-MISSING_PROP = PropagatingMissingType()
-MISSING_NONPROP = NonpropagatingMissingType()
+MISSING: Any = PropagatingMissingType()
+"""Sentinel value to mark default values as missing. Can be used to mark fields
+passed in via `default=` for `tyro.cli()` as required.
+
+When used, the 'missing' semantics propagate to children. For example, if we write:
+
+.. code-block:: python
+
+    def main(inner: Dataclass = tyro.MISSING) -> None:
+        ...
+
+    tyro.cli(main)
+
+then all fields belonging to ``Dataclass`` will be marked as missing, even if a
+default exists in the dataclass definition.
+"""
+MISSING_NONPROP: Any = NonpropagatingMissingType()
+"""Non-propagating version of :data:`tyro.MISSING`.
+
+When used, the 'missing' semantics do not propagate to children. For example:
+
+.. code-block:: python
+
+    def main(inner: Dataclass = tyro.constructors.MISSING_NONPROP) -> None:
+        ...
+
+    tyro.cli(main)
+
+is equivalent to:
+
+.. code-block:: python
+
+    def main(inner: Dataclass) -> None:
+        ...
+
+    tyro.cli(main)
+
+where default values for fields belonging to ``Dataclass`` will be taken from
+the dataclass definition.
+"""
 
 # When total=False in a TypedDict, we exclude fields from the constructor by default.
 NOT_REQUIRED_BUT_WE_DONT_KNOW_THE_VALUE = NotRequiredButWeDontKnowTheValueType()
@@ -46,27 +82,14 @@ class NotRequiredButWeDontKnowTheValueType(Singleton):
 
 EXCLUDE_FROM_CALL = ExcludeFromCallType()
 
-# Our "public" missing API will always be the propagating missing sentinel.
-MISSING: Any = MISSING_PROP
-"""Sentinel value to mark fields as missing. Can be used to mark fields passed
-in via `default=` for `tyro.cli()` as required."""
 
-
-MISSING_SINGLETONS = [
-    dataclasses.MISSING,
-    MISSING_PROP,
+MISSING_AND_MISSING_NONPROP = (
+    MISSING,
     MISSING_NONPROP,
-    inspect.Parameter.empty,
-]
-try:
-    # Undocumented feature: support omegaconf dataclasses out of the box.
-    import omegaconf
-
-    MISSING_SINGLETONS.append(omegaconf.MISSING)
-except ImportError:
-    pass
+)
+"""Singletons that are considered missing values when generating CLI interfaces."""
 
-DEFAULT_SENTINEL_SINGLETONS = MISSING_SINGLETONS + [
+DEFAULT_SENTINEL_SINGLETONS = MISSING_AND_MISSING_NONPROP + (
     NOT_REQUIRED_BUT_WE_DONT_KNOW_THE_VALUE,
     EXCLUDE_FROM_CALL,
-]
+)
diff --git a/src/tyro/_subcommand_matching.py b/src/tyro/_subcommand_matching.py
index ed7bed7e0..e4168bc64 100644
--- a/src/tyro/_subcommand_matching.py
+++ b/src/tyro/_subcommand_matching.py
@@ -26,14 +26,14 @@ def match_subcommand(
     # Get default subcommand name: by default hash.
     default_hash = object.__hash__(default)
     for subcommand_name, conf in subcommand_config_from_name.items():
-        if conf.default in _singleton.MISSING_SINGLETONS:
+        if conf.default in _singleton.MISSING_AND_MISSING_NONPROP:
             continue
         if default_hash == object.__hash__(conf.default):
             return subcommand_name
 
     # Get default subcommand name: by default value.
     for subcommand_name, conf in subcommand_config_from_name.items():
-        if conf.default in _singleton.MISSING_SINGLETONS:
+        if conf.default in _singleton.MISSING_AND_MISSING_NONPROP:
             continue
         equal = default == conf.default
         if isinstance(equal, bool) and equal:
@@ -42,7 +42,7 @@ def match_subcommand(
     # Get default subcommand name: by concrete type tree.
     typetree = _TypeTree.make(type(default), default)
     for subcommand_name, conf in subcommand_config_from_name.items():
-        if conf.default in _singleton.MISSING_SINGLETONS:
+        if conf.default in _singleton.MISSING_AND_MISSING_NONPROP:
             continue
         if typetree == _TypeTree.make(type(conf.default), conf.default):
             return subcommand_name
diff --git a/src/tyro/constructors/__init__.py b/src/tyro/constructors/__init__.py
index 5c41e2b0d..712101740 100644
--- a/src/tyro/constructors/__init__.py
+++ b/src/tyro/constructors/__init__.py
@@ -7,6 +7,8 @@
 
 """
 
+from .._singleton import MISSING as MISSING
+from .._singleton import MISSING_NONPROP as MISSING_NONPROP
 from ._primitive_spec import PrimitiveConstructorSpec as PrimitiveConstructorSpec
 from ._primitive_spec import PrimitiveTypeInfo as PrimitiveTypeInfo
 from ._primitive_spec import (
diff --git a/src/tyro/constructors/_struct_spec.py b/src/tyro/constructors/_struct_spec.py
index d826aef29..97be03bd8 100644
--- a/src/tyro/constructors/_struct_spec.py
+++ b/src/tyro/constructors/_struct_spec.py
@@ -20,9 +20,9 @@
 from .. import _docstrings, _resolver
 from .._singleton import (
     EXCLUDE_FROM_CALL,
+    MISSING,
+    MISSING_AND_MISSING_NONPROP,
     MISSING_NONPROP,
-    MISSING_PROP,
-    MISSING_SINGLETONS,
 )
 from .._typing import TypeForm
 from ..conf import _confstruct, _markers
@@ -81,11 +81,17 @@ class StructConstructorSpec:
 
 @dataclasses.dataclass(frozen=True)
 class StructTypeInfo:
-    """Information used to generate constructors for primitive types."""
+    """Information used to generate constructors for struct types."""
 
     type: TypeForm
+    """The type of the (potential) struct."""
     markers: tuple[Any, ...]
+    """Markers from :mod:`tyro.conf` that are associated with this field."""
     default: Any
+    """The default value of the struct, or a member of
+    :data:`tyro.constructors.MISSING_SINGLETONS` if not present. In a function
+    signature, this is `X` in `def main(x=X): ...`. This can be useful for
+    populating the default values of the struct."""
     _typevar_context: _resolver.TypeParamAssignmentContext
 
     @staticmethod
@@ -178,7 +184,7 @@ def typeddict_rule(info: StructTypeInfo) -> StructConstructorSpec | None:
         # Handle TypedDicts.
         field_list = []
         valid_default_instance = (
-            info.default not in MISSING_SINGLETONS
+            info.default not in MISSING_AND_MISSING_NONPROP
             and info.default is not EXCLUDE_FROM_CALL
         )
         assert not valid_default_instance or isinstance(info.default, dict)
@@ -195,7 +201,7 @@ def typeddict_rule(info: StructTypeInfo) -> StructConstructorSpec | None:
                 is_default_from_default_instance = True
             elif typ_origin is Required and total is False:
                 # Support total=False.
-                default = MISSING_PROP
+                default = MISSING
             elif total is False:
                 # Support total=False.
                 default = EXCLUDE_FROM_CALL
@@ -209,7 +215,7 @@ def typeddict_rule(info: StructTypeInfo) -> StructConstructorSpec | None:
                 # Support typing.NotRequired[].
                 default = EXCLUDE_FROM_CALL
             else:
-                default = MISSING_PROP
+                default = MISSING
 
             # Nested types need to be populated / can't be excluded from the call.
             if default is EXCLUDE_FROM_CALL and is_struct_type(typ, MISSING_NONPROP):
@@ -264,7 +270,7 @@ def attrs_rule(info: StructTypeInfo) -> StructConstructorSpec | None:
             name = attr_field.name
             default = attr_field.default
             is_default_from_default_instance = False
-            if info.default not in MISSING_SINGLETONS:
+            if info.default not in MISSING_AND_MISSING_NONPROP:
                 if hasattr(info.default, name):
                     default = getattr(info.default, name)
                     is_default_from_default_instance = True
@@ -309,7 +315,7 @@ def dict_rule(info: StructTypeInfo) -> StructConstructorSpec | None:
         ):
             return None
 
-        if info.default in MISSING_SINGLETONS or len(info.default) == 0:
+        if info.default in MISSING_AND_MISSING_NONPROP or len(info.default) == 0:
             return None
 
         field_list = []
@@ -344,11 +350,13 @@ def namedtuple_rule(info: StructTypeInfo) -> StructConstructorSpec | None:
             default = field_defaults.get(name, MISSING_NONPROP)
             is_default_from_default_instance = False
 
-            if info.default not in MISSING_SINGLETONS and hasattr(info.default, name):
+            if info.default not in MISSING_AND_MISSING_NONPROP and hasattr(
+                info.default, name
+            ):
                 default = getattr(info.default, name)
                 is_default_from_default_instance = True
-            elif info.default is MISSING_PROP:
-                default = MISSING_PROP
+            elif info.default is MISSING:
+                default = MISSING
 
             field_list.append(
                 StructFieldSpec(
@@ -407,13 +415,16 @@ def tuple_rule(info: StructTypeInfo) -> StructConstructorSpec | None:
 
         # Infer more specific type when tuple annotation isn't subscripted.
         if len(children) == 0:
-            if info.default in MISSING_SINGLETONS:
+            if info.default in MISSING_AND_MISSING_NONPROP:
                 return None
             else:
                 assert isinstance(info.default, tuple)
                 children = tuple(type(x) for x in info.default)
 
-        if info.default in MISSING_SINGLETONS or info.default is EXCLUDE_FROM_CALL:
+        if (
+            info.default in MISSING_AND_MISSING_NONPROP
+            or info.default is EXCLUDE_FROM_CALL
+        ):
             default_instance = (info.default,) * len(children)
         else:
             default_instance = info.default
@@ -567,12 +578,12 @@ def _get_dataclass_field_default(
     """Helper for getting the default instance for a dataclass field."""
     # If the dataclass's parent is explicitly marked MISSING, mark this field as missing
     # as well.
-    if parent_default_instance is MISSING_PROP:
-        return MISSING_PROP, False
+    if parent_default_instance is MISSING:
+        return MISSING, False
 
     # Try grabbing default from parent instance.
     if (
-        parent_default_instance not in MISSING_SINGLETONS
+        parent_default_instance not in MISSING_AND_MISSING_NONPROP
         and parent_default_instance is not None
     ):
         # Populate default from some parent, eg `default=` in `tyro.cli()`.
@@ -587,7 +598,10 @@ def _get_dataclass_field_default(
             )
 
     # Try grabbing default from dataclass field.
-    if field.default not in MISSING_SINGLETONS:
+    if (
+        field.default not in MISSING_AND_MISSING_NONPROP
+        and field.default is not dataclasses.MISSING
+    ):
         default = field.default
         # dataclasses.is_dataclass() will also return true for dataclass
         # _types_, not just instances.
@@ -609,8 +623,7 @@ def _get_dataclass_field_default(
     ):
         return field.default_factory(), False
 
-    # Otherwise, no default. This is different from MISSING, because MISSING propagates
-    # to children. We could revisit this design to make it clearer.
+    # Otherwise, no default.
     return MISSING_NONPROP, False
 
 
@@ -628,7 +641,7 @@ def _get_pydantic_v1_field_default(
 
     # Try grabbing default from parent instance.
     if (
-        parent_default_instance not in MISSING_SINGLETONS
+        parent_default_instance not in MISSING_AND_MISSING_NONPROP
         and parent_default_instance is not None
     ):
         # Populate default from some parent, eg `default=` in `tyro.cli()`.
@@ -658,7 +671,7 @@ def _get_pydantic_v2_field_default(
 
     # Try grabbing default from parent instance.
     if (
-        parent_default_instance not in MISSING_SINGLETONS
+        parent_default_instance not in MISSING_AND_MISSING_NONPROP
         and parent_default_instance is not None
     ):
         # Populate default from some parent, eg `default=` in `tyro.cli()`.
diff --git a/src/tyro/extras/_serialization.py b/src/tyro/extras/_serialization.py
index 1dab5787c..4e07a2929 100644
--- a/src/tyro/extras/_serialization.py
+++ b/src/tyro/extras/_serialization.py
@@ -14,7 +14,7 @@
 from typing_extensions import get_args, get_origin
 
 from .. import _resolver
-from ..constructors._struct_spec import MISSING_PROP
+from ..constructors._struct_spec import MISSING
 
 ENUM_YAML_TAG_PREFIX = "!enum:"
 DATACLASS_YAML_TAG_PREFIX = "!dataclass:"
@@ -119,7 +119,7 @@ def make_enum_constructor(typ: Type[enum.Enum]):
 
     DataclassLoader.add_constructor(
         tag=MISSING_YAML_TAG_PREFIX,
-        constructor=lambda *_unused: MISSING_PROP,  # type: ignore
+        constructor=lambda *_unused: MISSING,  # type: ignore
     )
 
     return DataclassLoader
@@ -130,7 +130,7 @@ def _make_dumper(instance: Any) -> Type[yaml.Dumper]:
 
     class DataclassDumper(yaml.Dumper):
         def ignore_aliases(self, data):
-            return super().ignore_aliases(data) or data is MISSING_PROP
+            return super().ignore_aliases(data) or data is MISSING
 
     contained_types = list(_get_contained_special_types_from_type(type(instance)))
     contained_type_names = list(map(lambda cls: cls.__name__, contained_types))
@@ -164,7 +164,7 @@ def representer(dumper: DataclassDumper, data: Any) -> yaml.Node:
         DataclassDumper.add_representer(typ, make_representer(name))
 
     DataclassDumper.add_representer(
-        type(MISSING_PROP),
+        type(MISSING),
         lambda dumper, data: dumper.represent_scalar(
             tag=MISSING_YAML_TAG_PREFIX, value=""
         ),
diff --git a/tests/test_custom_constructors.py b/tests/test_custom_constructors.py
index 4c2a1cbe7..1ca7a5e6e 100644
--- a/tests/test_custom_constructors.py
+++ b/tests/test_custom_constructors.py
@@ -3,7 +3,8 @@
 import json
 from typing import Any, Dict, Union
 
-from typing_extensions import Annotated, get_args
+import numpy as np
+from typing_extensions import Annotated, Literal, get_args
 
 import tyro
 
@@ -63,3 +64,30 @@ def main(
 
     assert tyro.cli(main, args=["--x", "3"]) == 3
     assert tyro.cli(main, args=["--x", '{"a": 1}']) == {"a": 1}
+
+
+def _construct_array(
+    values: tuple[float, ...], dtype: Literal["float32", "float64"] = "float64"
+) -> np.ndarray:
+    return np.array(
+        values,
+        dtype={"float32": np.float32, "float64": np.float64}[dtype],
+    )
+
+
+def test_custom_constructor_numpy() -> None:
+    def main(
+        array: Annotated[np.ndarray, tyro.conf.arg(constructor=_construct_array)],
+    ) -> np.ndarray:
+        return array
+
+    assert np.allclose(
+        tyro.cli(main, args="--array.values 1 2 3 4 5".split(" ")),
+        np.array([1, 2, 3, 4, 5], dtype=np.float64),
+    )
+    assert (
+        tyro.cli(
+            main, args="--array.values 1 2 3 4 5 --array.dtype float32".split(" ")
+        ).dtype
+        == np.float32
+    )
diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py
index 2572a7c64..87e4c2317 100644
--- a/tests/test_py311_generated/test_conf_generated.py
+++ b/tests/test_py311_generated/test_conf_generated.py
@@ -5,7 +5,16 @@
 import json as json_
 import shlex
 import sys
-from typing import Annotated, Any, Dict, Generic, List, Tuple, Type, TypeVar
+from typing import (
+    Annotated,
+    Any,
+    Dict,
+    Generic,
+    List,
+    Tuple,
+    Type,
+    TypeVar,
+)
 
 import pytest
 from helptext_utils import get_helptext_with_checks
diff --git a/tests/test_py311_generated/test_custom_constructors_generated.py b/tests/test_py311_generated/test_custom_constructors_generated.py
index 6e0cd35eb..f2a39cc44 100644
--- a/tests/test_py311_generated/test_custom_constructors_generated.py
+++ b/tests/test_py311_generated/test_custom_constructors_generated.py
@@ -1,7 +1,9 @@
 from __future__ import annotations
 
 import json
-from typing import Annotated, Any, Dict, get_args
+from typing import Annotated, Any, Dict, Literal, get_args
+
+import numpy as np
 
 import tyro
 
@@ -61,3 +63,30 @@ def main(
 
     assert tyro.cli(main, args=["--x", "3"]) == 3
     assert tyro.cli(main, args=["--x", '{"a": 1}']) == {"a": 1}
+
+
+def _construct_array(
+    values: tuple[float, ...], dtype: Literal["float32", "float64"] = "float64"
+) -> np.ndarray:
+    return np.array(
+        values,
+        dtype={"float32": np.float32, "float64": np.float64}[dtype],
+    )
+
+
+def test_custom_constructor_numpy() -> None:
+    def main(
+        array: Annotated[np.ndarray, tyro.conf.arg(constructor=_construct_array)],
+    ) -> np.ndarray:
+        return array
+
+    assert np.allclose(
+        tyro.cli(main, args="--array.values 1 2 3 4 5".split(" ")),
+        np.array([1, 2, 3, 4, 5], dtype=np.float64),
+    )
+    assert (
+        tyro.cli(
+            main, args="--array.values 1 2 3 4 5 --array.dtype float32".split(" ")
+        ).dtype
+        == np.float32
+    )

From 3a603a20ac8f0a06cef3c42aa79c0a1c3bf9bf84 Mon Sep 17 00:00:00 2001
From: brentyi 
Date: Sat, 16 Nov 2024 12:18:56 -0500
Subject: [PATCH 480/491] Example usage fix

---
 docs/source/examples/custom_constructors.rst  | 43 ++++++++++---------
 .../04_struct_registry.py                     |  9 ++--
 2 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/docs/source/examples/custom_constructors.rst b/docs/source/examples/custom_constructors.rst
index cd9eb459c..38f8ce63d 100644
--- a/docs/source/examples/custom_constructors.rst
+++ b/docs/source/examples/custom_constructors.rst
@@ -301,6 +301,9 @@ add support for a custom type.
         def __init__(self, lower: int, upper: int):
             self.bounds = (lower, upper)
 
+        def __repr__(self) -> str:
+            return f"(lower={self.bounds[0]}, upper={self.bounds[1]})"
+
     # Create a custom registry, which stores constructor rules.
     custom_registry = tyro.constructors.ConstructorRegistry()
 
@@ -368,26 +371,24 @@ add support for a custom type.
 .. raw:: html
 
     
-    $ python ./03_primitive_registry.py --help
-    usage: 03_primitive_registry.py [-h] --dict1 JSON [--dict2 JSON]
+    $ python ./04_struct_registry.py --help
+    usage: 04_struct_registry.py [-h] [OPTIONS]
     
-    A function with two arguments, which can be populated from the CLI via JSON.
+    A function with two `Bounds` instances as input.
     
-    ╭─ options ───────────────────────────────────────────╮
-    │ -h, --help          show this help message and exit │
-    │ --dict1 JSON        (required)                      │
-    │ --dict2 JSON        (default: '{"default": null}')  │
-    ╰─────────────────────────────────────────────────────╯
-    

-
-
-
-.. raw:: html
-
-    
-    $ python ./03_primitive_registry.py --dict1 '{"hello": "world"}'
-    dict1={'hello': 'world'}
-    dict2={'default': None}
+    ╭─ options ───────────────────────────────────────────────╮
+    │ -h, --help              show this help message and exit │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ bounds options ────────────────────────────────────────╮
+    │ --bounds.lower INT      Lower bound. (required)         │
+    │ --bounds.upper INT      Upper bound. (required)         │
+    ╰─────────────────────────────────────────────────────────╯
+    ╭─ bounds-with-default options ───────────────────────────╮
+    │ --bounds-with-default.lower INT                         │
+    │                         Lower bound. (default: 0)       │
+    │ --bounds-with-default.upper INT                         │
+    │                         Upper bound. (default: 100)     │
+    ╰─────────────────────────────────────────────────────────╯
     

 
 
@@ -395,7 +396,7 @@ add support for a custom type.
 .. raw:: html
 
     
-    $ python ./03_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
-    dict1={'hello': 'world'}
-    dict2={'hello': 'world'}
+    $ python ./04_struct_registry.py --bounds.lower 5 --bounds.upper 10
+    bounds=(lower=5, upper=10)
+    bounds_with_default=(lower=0, upper=100)
     

\ No newline at end of file
diff --git a/examples/06_custom_constructors/04_struct_registry.py b/examples/06_custom_constructors/04_struct_registry.py
index e836a6278..da09e992b 100644
--- a/examples/06_custom_constructors/04_struct_registry.py
+++ b/examples/06_custom_constructors/04_struct_registry.py
@@ -8,10 +8,8 @@
     This will be complicated!
 
 Usage:
-
-    python ./03_primitive_registry.py --help
-    python ./03_primitive_registry.py --dict1 '{"hello": "world"}'
-    python ./03_primitive_registry.py --dict1 '{"hello": "world"}' --dict2 '{"hello": "world"}'
+    python ./04_struct_registry.py --help
+    python ./04_struct_registry.py --bounds.lower 5 --bounds.upper 10
 """
 
 import tyro
@@ -22,6 +20,9 @@ class Bounds:
     def __init__(self, lower: int, upper: int):
         self.bounds = (lower, upper)
 
+    def __repr__(self) -> str:
+        return f"(lower={self.bounds[0]}, upper={self.bounds[1]})"
+
 
 # Create a custom registry, which stores constructor rules.
 custom_registry = tyro.constructors.ConstructorRegistry()

From 91dfac237f7cd962b57fd41ce4e0db0b68423630 Mon Sep 17 00:00:00 2001
From: brentyi 
Date: Sun, 17 Nov 2024 23:46:53 -0500
Subject: [PATCH 481/491] `0.9.0`

---
 docs/source/examples/custom_constructors.rst          | 11 ++++++++---
 docs/source/examples/hierarchical_structures.rst      |  4 ++--
 docs/source/whats_supported.rst                       |  2 +-
 .../03_nesting_containers.py                          |  4 ++--
 examples/06_custom_constructors/README.rst            | 11 ++++++++---
 pyproject.toml                                        |  2 +-
 src/tyro/__init__.py                                  |  2 +-
 7 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/docs/source/examples/custom_constructors.rst b/docs/source/examples/custom_constructors.rst
index 38f8ce63d..dd8605fdb 100644
--- a/docs/source/examples/custom_constructors.rst
+++ b/docs/source/examples/custom_constructors.rst
@@ -6,9 +6,9 @@
 Custom constructors
 ===================
 
-:func:`tyro.cli` is designed for comprehensive support of standard Python type
-constructs. In some cases, however, it can be useful to extend the set of types
-supported by :mod:`tyro`.
+:func:`tyro.cli` aims for comprehensive support of standard Python type
+constructs. It can still, however, be useful to extend the set of suported
+types.
 
 We provide two complementary approaches for doing so:
 
@@ -19,6 +19,11 @@ We provide two complementary approaches for doing so:
   instantiated from a single commandline argument, while *struct* types are
   broken down into multiple arguments.
 
+.. warning::
+
+    Custom constructors are useful, but can be verbose and require care. We
+    recommend using them sparingly.
+
 
 .. _example-01_simple_constructors:
 
diff --git a/docs/source/examples/hierarchical_structures.rst b/docs/source/examples/hierarchical_structures.rst
index 9c920f0eb..67021bdea 100644
--- a/docs/source/examples/hierarchical_structures.rst
+++ b/docs/source/examples/hierarchical_structures.rst
@@ -214,11 +214,11 @@ Structures can be nested inside of standard containers.
         color_tuple: tuple[RGB, RGB]
         color_dict: dict[str, RGB] = dataclasses.field(
             # We can't use mutable values as defaults directly.
-            default_factory={
+            default_factory=lambda: {
                 "red": RGB(255, 0, 0),
                 "green": RGB(0, 255, 0),
                 "blue": RGB(0, 0, 255),
-            }.copy
+            }
         )
 
     if __name__ == "__main__":
diff --git a/docs/source/whats_supported.rst b/docs/source/whats_supported.rst
index fa05cb2b2..f579c40a5 100644
--- a/docs/source/whats_supported.rst
+++ b/docs/source/whats_supported.rst
@@ -9,7 +9,7 @@ As a partial list, inputs can be annotated with:
 - Basic types like :class:`int`, :class:`str`, :class:`float`, :class:`bool`, :class:`pathlib.Path`, :data:`None`.
 - :class:`datetime.date`, :class:`datetime.datetime`, and :class:`datetime.time`.
 - Container types like :class:`list`, :class:`dict`, :class:`tuple`, and :class:`set`.
-- Union types, like `X | Y`, :py:data:`typing.Union`, and :py:data:`typing.Optional`.
+- Union types, like ``X | Y``, :py:data:`typing.Union`, and :py:data:`typing.Optional`.
 - :py:data:`typing.Literal` and :class:`enum.Enum`.
 - Type aliases, for example using Python 3.12's `PEP 695 `_ `type` statement.
 - Generics, such as those annotated with :py:class:`typing.TypeVar` or with the type parameter syntax introduced by Python 3.12's `PEP 695 `_.
diff --git a/examples/02_hierarchical_structures/03_nesting_containers.py b/examples/02_hierarchical_structures/03_nesting_containers.py
index 9f3ecd08d..1206be99d 100644
--- a/examples/02_hierarchical_structures/03_nesting_containers.py
+++ b/examples/02_hierarchical_structures/03_nesting_containers.py
@@ -31,11 +31,11 @@ class Args:
     color_tuple: tuple[RGB, RGB]
     color_dict: dict[str, RGB] = dataclasses.field(
         # We can't use mutable values as defaults directly.
-        default_factory={
+        default_factory=lambda: {
             "red": RGB(255, 0, 0),
             "green": RGB(0, 255, 0),
             "blue": RGB(0, 0, 255),
-        }.copy
+        }
     )
 
 
diff --git a/examples/06_custom_constructors/README.rst b/examples/06_custom_constructors/README.rst
index 23698ee2b..277ea3b7c 100644
--- a/examples/06_custom_constructors/README.rst
+++ b/examples/06_custom_constructors/README.rst
@@ -1,9 +1,9 @@
 Custom constructors
 ===================
 
-:func:`tyro.cli` is designed for comprehensive support of standard Python type
-constructs. In some cases, however, it can be useful to extend the set of types
-supported by :mod:`tyro`.
+:func:`tyro.cli` aims for comprehensive support of standard Python type
+constructs. It can still, however, be useful to extend the set of suported
+types.
 
 We provide two complementary approaches for doing so:
 
@@ -13,3 +13,8 @@ We provide two complementary approaches for doing so:
   for different types. There are two categories of types: *primitive* types are
   instantiated from a single commandline argument, while *struct* types are
   broken down into multiple arguments.
+
+.. warning::
+
+    Custom constructors are useful, but can be verbose and require care. We
+    recommend using them sparingly.
diff --git a/pyproject.toml b/pyproject.toml
index fffa4ffe4..54bf2177b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,7 +10,7 @@ name = "tyro"
 authors = [
     {name = "brentyi", email = "brentyi@berkeley.edu"},
 ]
-version = "0.8.14"  # TODO: currently needs to be synchronized manually with __init__.py.
+version = "0.9.0"  # TODO: currently needs to be synchronized manually with __init__.py.
 description = "Strongly typed, zero-effort CLI interfaces"
 readme = "README.md"
 license = { text = "MIT" }
diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py
index 7be6652a5..d21704d5c 100644
--- a/src/tyro/__init__.py
+++ b/src/tyro/__init__.py
@@ -15,4 +15,4 @@
 
 
 # TODO: this should be synchronized automatically with the pyproject.toml.
-__version__ = "0.8.14"
+__version__ = "0.9.0"

From 604992e38facb82d6b8767deea0c919a5ebbb98e Mon Sep 17 00:00:00 2001
From: brentyi 
Date: Mon, 18 Nov 2024 09:18:42 -0500
Subject: [PATCH 482/491] `0.9.1`, fix build

---
 pyproject.toml       | 9 ++++++---
 src/tyro/__init__.py | 7 +++----
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 54bf2177b..8110e3171 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -2,7 +2,10 @@
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
-[tool.hatch.build]
+[tool.hatch.version]
+path = "src/tyro/__init__.py"
+
+[tool.hatch.build.targets.wheel]
 packages = ["src/tyro"]
 
 [project]
@@ -10,8 +13,8 @@ name = "tyro"
 authors = [
     {name = "brentyi", email = "brentyi@berkeley.edu"},
 ]
-version = "0.9.0"  # TODO: currently needs to be synchronized manually with __init__.py.
-description = "Strongly typed, zero-effort CLI interfaces"
+description = "CLI interfaces & config objects, from types"
+dynamic = ["version"]
 readme = "README.md"
 license = { text = "MIT" }
 requires-python = ">=3.7"
diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py
index d21704d5c..7c80250e5 100644
--- a/src/tyro/__init__.py
+++ b/src/tyro/__init__.py
@@ -1,5 +1,8 @@
 from typing import TYPE_CHECKING
 
+__version__ = "0.9.1"
+
+
 from . import conf as conf
 from . import constructors as constructors
 from . import extras as extras
@@ -12,7 +15,3 @@
     from .constructors._primitive_spec import (
         UnsupportedTypeAnnotationError as UnsupportedTypeAnnotationError,
     )
-
-
-# TODO: this should be synchronized automatically with the pyproject.toml.
-__version__ = "0.9.0"

From 9c5dd26a37a0295d1bfe2e4ebe196da0e0d28c6c Mon Sep 17 00:00:00 2001
From: Brent Yi 
Date: Fri, 22 Nov 2024 01:51:37 -0800
Subject: [PATCH 483/491] Apply `use_underscores` to decorator-based subcommand
 names (#208)

* Apply `use_underscores` to subcommand names for `tyro.extras.SubcommandApp`

* pydantic NewType fix
---
 src/tyro/_cli.py                              |  4 +--
 src/tyro/_strings.py                          |  4 +--
 src/tyro/extras/_subcommand_app.py            | 19 +++++++-----
 tests/test_decorator_subcommands.py           | 31 +++++++++++++------
 .../test_conf_generated.py                    | 11 +------
 .../test_decorator_subcommands_generated.py   | 31 +++++++++++++------
 .../test_pydantic_with_newtype_generated.py   |  8 +++--
 tests/test_pydantic_with_newtype.py           |  8 +++--
 8 files changed, 71 insertions(+), 45 deletions(-)

diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py
index a25fb17b3..81971c874 100644
--- a/src/tyro/_cli.py
+++ b/src/tyro/_cli.py
@@ -323,9 +323,9 @@ def _cli_impl(
 
         if "=" in arg:
             arg, _, val = arg.partition("=")
-            fixed = "--" + _strings.replace_delimeter_in_part(arg[2:]) + "=" + val
+            fixed = "--" + _strings.swap_delimeters(arg[2:]) + "=" + val
         else:
-            fixed = "--" + _strings.replace_delimeter_in_part(arg[2:])
+            fixed = "--" + _strings.swap_delimeters(arg[2:])
         if (
             return_unknown_args
             and fixed in modified_args
diff --git a/src/tyro/_strings.py b/src/tyro/_strings.py
index 2e32c7962..082ed01bd 100644
--- a/src/tyro/_strings.py
+++ b/src/tyro/_strings.py
@@ -30,7 +30,7 @@ def get_delimeter() -> Literal["-", "_"]:
     return DELIMETER
 
 
-def replace_delimeter_in_part(p: str) -> str:
+def swap_delimeters(p: str) -> str:
     """Replace hyphens with underscores (or vice versa) except when at the start."""
     if get_delimeter() == "-":
         stripped = p.lstrip("_")
@@ -49,7 +49,7 @@ def make_field_name(parts: Sequence[str]) -> str:
     """
     out = ".".join(parts)
     return ".".join(
-        replace_delimeter_in_part(part)
+        swap_delimeters(part)
         for part in out.split(".")
         if len(part) > 0 and part != dummy_field_name
     )
diff --git a/src/tyro/extras/_subcommand_app.py b/src/tyro/extras/_subcommand_app.py
index 7ce21494b..560dc27b8 100644
--- a/src/tyro/extras/_subcommand_app.py
+++ b/src/tyro/extras/_subcommand_app.py
@@ -3,6 +3,7 @@
 from typing import Any, Callable, Dict, Optional, Sequence, TypeVar, overload
 
 import tyro
+from tyro._strings import delimeter_context, swap_delimeters
 
 CallableT = TypeVar("CallableT", bound=Callable)
 
@@ -127,15 +128,19 @@ def cli(
 
         # Sort subcommands by name.
         if sort_subcommands:
-            sorted_subcommands = dict(
-                sorted(self._subcommands.items(), key=lambda x: x[0])
-            )
+            subcommands = dict(sorted(self._subcommands.items(), key=lambda x: x[0]))
         else:
-            sorted_subcommands = self._subcommands
+            subcommands = self._subcommands.copy()
+
+        # Replace delimeter in subcommand names.
+        with delimeter_context("_" if use_underscores else "-"):
+            orig_subcommand_names = list(subcommands.keys())
+            for orig_name in orig_subcommand_names:
+                subcommands[swap_delimeters(orig_name)] = subcommands.pop(orig_name)
 
-        if len(sorted_subcommands) == 1:
+        if len(subcommands) == 1:
             return tyro.cli(
-                next(iter(sorted_subcommands.values())),
+                next(iter(subcommands.values())),
                 prog=prog,
                 description=description,
                 args=args,
@@ -145,7 +150,7 @@ def cli(
             )
         else:
             return tyro.extras.subcommand_cli_from_dict(
-                sorted_subcommands,
+                subcommands,
                 prog=prog,
                 description=description,
                 args=args,
diff --git a/tests/test_decorator_subcommands.py b/tests/test_decorator_subcommands.py
index 0ed47227c..d9e8f9de7 100644
--- a/tests/test_decorator_subcommands.py
+++ b/tests/test_decorator_subcommands.py
@@ -8,7 +8,7 @@
 
 @app_just_one.command
 @app.command
-def greet(name: str, loud: bool = False) -> None:
+def greet_person(name: str, loud: bool = False) -> None:
     """Greet someone."""
     greeting = f"Hello, {name}!"
     if loud:
@@ -28,7 +28,7 @@ def test_app_just_one_cli(capsys):
         app_just_one.cli(args=["--help"])
     captured = capsys.readouterr()
     assert "usage: " in captured.out
-    assert "greet" not in captured.out
+    assert "greet-person" not in captured.out
     assert "addition" not in captured.out
     assert "--name" in captured.out
 
@@ -39,23 +39,23 @@ def test_app_cli(capsys):
         app.cli(args=["--help"])
     captured = capsys.readouterr()
     assert "usage: " in captured.out
-    assert "greet" in captured.out
+    assert "greet-person" in captured.out
     assert "addition" in captured.out
 
-    # Test: `python my_script.py greet --help`
+    # Test: `python my_script.py greet-person --help`
     with pytest.raises(SystemExit):
-        app.cli(args=["greet", "--help"], sort_subcommands=False)
+        app.cli(args=["greet-person", "--help"], sort_subcommands=False)
     captured = capsys.readouterr()
     assert "usage: " in captured.out
     assert "Greet someone." in captured.out
 
-    # Test: `python my_script.py greet --name Alice`
-    app.cli(args=["greet", "--name", "Alice"], sort_subcommands=True)
+    # Test: `python my_script.py greet-person --name Alice`
+    app.cli(args=["greet-person", "--name", "Alice"], sort_subcommands=True)
     captured = capsys.readouterr()
     assert captured.out.strip() == "Hello, Alice!"
 
-    # Test: `python my_script.py greet --name Bob --loud`
-    app.cli(args=["greet", "--name", "Bob", "--loud"])
+    # Test: `python my_script.py greet-person --name Bob --loud`
+    app.cli(args=["greet-person", "--name", "Bob", "--loud"])
     captured = capsys.readouterr()
     assert captured.out.strip() == "HELLO, BOB!"
 
@@ -70,3 +70,16 @@ def test_app_cli(capsys):
     app.cli(args=["addition", "--a", "5", "--b", "3"])
     captured = capsys.readouterr()
     assert captured.out.strip() == "5 + 3 = 8"
+
+
+def test_use_underscores(capsys) -> None:
+    with pytest.raises(SystemExit):
+        app.cli(args=["--help"], use_underscores=True)
+    captured = capsys.readouterr()
+    assert "greet-person" not in captured.out
+    assert "greet_person" in captured.out
+
+    # Test: `python my_script.py greet-person --name Bob --loud`
+    app.cli(args=["greet_person", "--name", "Bob", "--loud"], use_underscores=True)
+    captured = capsys.readouterr()
+    assert captured.out.strip() == "HELLO, BOB!"
diff --git a/tests/test_py311_generated/test_conf_generated.py b/tests/test_py311_generated/test_conf_generated.py
index 87e4c2317..2572a7c64 100644
--- a/tests/test_py311_generated/test_conf_generated.py
+++ b/tests/test_py311_generated/test_conf_generated.py
@@ -5,16 +5,7 @@
 import json as json_
 import shlex
 import sys
-from typing import (
-    Annotated,
-    Any,
-    Dict,
-    Generic,
-    List,
-    Tuple,
-    Type,
-    TypeVar,
-)
+from typing import Annotated, Any, Dict, Generic, List, Tuple, Type, TypeVar
 
 import pytest
 from helptext_utils import get_helptext_with_checks
diff --git a/tests/test_py311_generated/test_decorator_subcommands_generated.py b/tests/test_py311_generated/test_decorator_subcommands_generated.py
index 0ed47227c..d9e8f9de7 100644
--- a/tests/test_py311_generated/test_decorator_subcommands_generated.py
+++ b/tests/test_py311_generated/test_decorator_subcommands_generated.py
@@ -8,7 +8,7 @@
 
 @app_just_one.command
 @app.command
-def greet(name: str, loud: bool = False) -> None:
+def greet_person(name: str, loud: bool = False) -> None:
     """Greet someone."""
     greeting = f"Hello, {name}!"
     if loud:
@@ -28,7 +28,7 @@ def test_app_just_one_cli(capsys):
         app_just_one.cli(args=["--help"])
     captured = capsys.readouterr()
     assert "usage: " in captured.out
-    assert "greet" not in captured.out
+    assert "greet-person" not in captured.out
     assert "addition" not in captured.out
     assert "--name" in captured.out
 
@@ -39,23 +39,23 @@ def test_app_cli(capsys):
         app.cli(args=["--help"])
     captured = capsys.readouterr()
     assert "usage: " in captured.out
-    assert "greet" in captured.out
+    assert "greet-person" in captured.out
     assert "addition" in captured.out
 
-    # Test: `python my_script.py greet --help`
+    # Test: `python my_script.py greet-person --help`
     with pytest.raises(SystemExit):
-        app.cli(args=["greet", "--help"], sort_subcommands=False)
+        app.cli(args=["greet-person", "--help"], sort_subcommands=False)
     captured = capsys.readouterr()
     assert "usage: " in captured.out
     assert "Greet someone." in captured.out
 
-    # Test: `python my_script.py greet --name Alice`
-    app.cli(args=["greet", "--name", "Alice"], sort_subcommands=True)
+    # Test: `python my_script.py greet-person --name Alice`
+    app.cli(args=["greet-person", "--name", "Alice"], sort_subcommands=True)
     captured = capsys.readouterr()
     assert captured.out.strip() == "Hello, Alice!"
 
-    # Test: `python my_script.py greet --name Bob --loud`
-    app.cli(args=["greet", "--name", "Bob", "--loud"])
+    # Test: `python my_script.py greet-person --name Bob --loud`
+    app.cli(args=["greet-person", "--name", "Bob", "--loud"])
     captured = capsys.readouterr()
     assert captured.out.strip() == "HELLO, BOB!"
 
@@ -70,3 +70,16 @@ def test_app_cli(capsys):
     app.cli(args=["addition", "--a", "5", "--b", "3"])
     captured = capsys.readouterr()
     assert captured.out.strip() == "5 + 3 = 8"
+
+
+def test_use_underscores(capsys) -> None:
+    with pytest.raises(SystemExit):
+        app.cli(args=["--help"], use_underscores=True)
+    captured = capsys.readouterr()
+    assert "greet-person" not in captured.out
+    assert "greet_person" in captured.out
+
+    # Test: `python my_script.py greet-person --name Bob --loud`
+    app.cli(args=["greet_person", "--name", "Bob", "--loud"], use_underscores=True)
+    captured = capsys.readouterr()
+    assert captured.out.strip() == "HELLO, BOB!"
diff --git a/tests/test_py311_generated/test_pydantic_with_newtype_generated.py b/tests/test_py311_generated/test_pydantic_with_newtype_generated.py
index 44ecb7300..fa7a5ebb7 100644
--- a/tests/test_py311_generated/test_pydantic_with_newtype_generated.py
+++ b/tests/test_py311_generated/test_pydantic_with_newtype_generated.py
@@ -9,11 +9,13 @@
 
 
 class Measurements(BaseModel):
-    single: Microliter = pydantic.Field(10)
+    single: Microliter = pydantic.Field(Microliter(10))
     renamed_single: Annotated[Microliter, tyro.conf.arg(name="other_single")] = (
-        pydantic.Field(10)
+        pydantic.Field(Microliter(10))
+    )
+    pair: Tuple[Microliter, Microliter] = pydantic.Field(
+        (Microliter(20), Microliter(30))
     )
-    pair: Tuple[Microliter, Microliter] = pydantic.Field((20, 30))
 
 
 IncorrectMeasurements = NewType("IncorrectMeasurements", Measurements)
diff --git a/tests/test_pydantic_with_newtype.py b/tests/test_pydantic_with_newtype.py
index 2b4d71d30..1abfc0dd2 100644
--- a/tests/test_pydantic_with_newtype.py
+++ b/tests/test_pydantic_with_newtype.py
@@ -10,11 +10,13 @@
 
 
 class Measurements(BaseModel):
-    single: Microliter = pydantic.Field(10)
+    single: Microliter = pydantic.Field(Microliter(10))
     renamed_single: Annotated[Microliter, tyro.conf.arg(name="other_single")] = (
-        pydantic.Field(10)
+        pydantic.Field(Microliter(10))
+    )
+    pair: Tuple[Microliter, Microliter] = pydantic.Field(
+        (Microliter(20), Microliter(30))
     )
-    pair: Tuple[Microliter, Microliter] = pydantic.Field((20, 30))
 
 
 IncorrectMeasurements = NewType("IncorrectMeasurements", Measurements)

From ae0ab2cb87eaee51b1d368f3a6f05e3005ed0972 Mon Sep 17 00:00:00 2001
From: Brent Yi 
Date: Fri, 22 Nov 2024 22:11:45 -0800
Subject: [PATCH 484/491] Less restricted type expansion, more robust
 subcommand matching, improved error messages (#204)

* Improve mismatched type for StrEnum

* Overhaul subcommand matching

* Additional subcommand matching tests

* Min 3.11 for StrEnum tests

* fix test ignore

* Test nits

* typeguard compatibility

* Add minimum typeguard version

* fix typeguard errors

* typeguard refactor, more subcommand matching tests

* Add 3.13 to CI (fixes #206)

* Add pydantic version exclude, waiting for patch
Related: https://github.com/pydantic/pydantic/issues/10912, https://github.com/pydantic/pydantic/pull/10909

* Improve type warnings + errors

* Generic subcommand example, docs updates

* Remove unused

* `build` => `pytest`, closes #206

* mypy
---
 .github/workflows/{build.yml => pytest.yml}   |   6 +-
 docs/source/examples/basics.rst               |   4 +-
 docs/source/examples/custom_constructors.rst  |   9 +-
 docs/source/examples/generics.rst             | 116 +++++++++++++++-
 .../examples/hierarchical_structures.rst      |  10 +-
 docs/source/examples/pytorch_jax.rst          |  18 +--
 docs/source/index.md                          |   5 +-
 examples/01_basics/05_choices.py              |   2 +-
 examples/01_basics/07_unions.py               |   2 +-
 .../02_nesting_in_func.py                     |   2 +-
 .../04_dictionaries.py                        |   6 +-
 .../02_hierarchical_structures/05_tuples.py   |   2 +-
 examples/05_generics/02_generics.py           |   2 +-
 .../05_generics/03_generic_subcommands.py     |  40 ++++++
 .../01_simple_constructors.py                 |   5 +-
 .../04_struct_registry.py                     |   4 +-
 .../07_pytorch_jax/01_pytorch_parallelism.py  |  18 +--
 pyproject.toml                                |   3 +-
 src/tyro/_arguments.py                        |   4 +-
 src/tyro/_cli.py                              |   4 +-
 src/tyro/_fields.py                           |  39 ++----
 src/tyro/_parsers.py                          |  28 ++++
 src/tyro/_resolver.py                         |  13 ++
 src/tyro/_subcommand_matching.py              | 127 ++++++------------
 src/tyro/constructors/_primitive_spec.py      |   8 +-
 src/tyro/constructors/_registry.py            |  45 +++++++
 src/tyro/constructors/_struct_spec.py         |  41 ++----
 tests/conftest.py                             |   4 +-
 tests/test_errors.py                          |  19 ++-
 tests/test_nested.py                          |  75 +++++++++++
 tests/test_new_style_annotations_min_py312.py | 104 +++++++++++---
 .../test_errors_generated.py                  |  19 ++-
 .../test_nested_generated.py                  |  75 +++++++++++
 ...w_style_annotations_min_py312_generated.py | 104 +++++++++++---
 .../test_str_enum_min_py311_generated.py      |  87 ++++++++++++
 tests/test_str_enum_min_py311.py              |  87 ++++++++++++
 36 files changed, 875 insertions(+), 262 deletions(-)
 rename .github/workflows/{build.yml => pytest.yml} (95%)
 create mode 100644 examples/05_generics/03_generic_subcommands.py
 create mode 100644 tests/test_py311_generated/test_str_enum_min_py311_generated.py
 create mode 100644 tests/test_str_enum_min_py311.py

diff --git a/.github/workflows/build.yml b/.github/workflows/pytest.yml
similarity index 95%
rename from .github/workflows/build.yml
rename to .github/workflows/pytest.yml
index 94c1a7d4d..969e7a43d 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/pytest.yml
@@ -1,4 +1,4 @@
-name: build
+name: pytest
 
 on:
   push:
@@ -7,11 +7,11 @@ on:
     branches: [main]
 
 jobs:
-  build:
+  pytest:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 
     steps:
       - uses: actions/checkout@v2
diff --git a/docs/source/examples/basics.rst b/docs/source/examples/basics.rst
index e701e6b7d..55961c894 100644
--- a/docs/source/examples/basics.rst
+++ b/docs/source/examples/basics.rst
@@ -304,7 +304,7 @@ To turn off conversion, see :class:`tyro.conf.FlagConversionOff`.
 Choices
 -------
 
-:code:`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices.
+:py:data:`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices.
 
 
 .. code-block:: python
@@ -443,7 +443,7 @@ choices.
 Unions
 ------
 
-:code:`X | Y` or :code:`typing.Union[X, Y]` can be used to expand inputs to
+:code:`X | Y` or :py:data:`typing.Union` can be used to expand inputs to
 multiple types.
 
 
diff --git a/docs/source/examples/custom_constructors.rst b/docs/source/examples/custom_constructors.rst
index dd8605fdb..9caf52a92 100644
--- a/docs/source/examples/custom_constructors.rst
+++ b/docs/source/examples/custom_constructors.rst
@@ -31,11 +31,10 @@ Simple Constructors
 -------------------
 
 For simple custom constructors, we can pass a constructor function into
-:func:`tyro.conf.arg` or :func:`tyro.conf.subcommand`. Arguments for will be
+:func:`tyro.conf.arg` or :func:`tyro.conf.subcommand`. Arguments will be
 generated by parsing the signature of the constructor function.
 
-In this example, we use this pattern to define custom behavior for
-instantiating a NumPy array.
+In this example, we define custom behavior for instantiating a NumPy array.
 
 
 .. code-block:: python
@@ -288,7 +287,9 @@ Custom Structs (Registry)
 -------------------------
 
 In this example, we use a :class:`tyro.constructors.ConstructorRegistry` to
-add support for a custom type.
+add support for a custom struct type. Each struct type is broken down into
+multiple fields, which themselves can be either primitive types or nested
+structs.
 
 .. warning::
 
diff --git a/docs/source/examples/generics.rst b/docs/source/examples/generics.rst
index 22163858a..aabd9836e 100644
--- a/docs/source/examples/generics.rst
+++ b/docs/source/examples/generics.rst
@@ -84,8 +84,8 @@ This example uses syntax introduced in Python 3.12 (`PEP 695 
 .. _example-02_generics:
 
-Generics (Legacy)
------------------
+Generics (Python <3.12)
+-----------------------
 
 The legacy :py:class:`typing.Generic` and :py:class:`typing.TypeVar` syntax for
 generic types is also supported.
@@ -154,4 +154,116 @@ generic types is also supported.
     │ --shape.c.z FLOAT       (required)                      │
     │ --shape.c.frame-id STR  (required)                      │
     ╰─────────────────────────────────────────────────────────╯
+    

+.. _example-03_generic_subcommands:
+
+Generic Subcommands
+-------------------
+
+
+
+
+.. code-block:: python
+    :linenos:
+
+    # 03_generic_subcommands.py
+    import dataclasses
+    from pathlib import Path
+
+    import tyro
+
+    @dataclasses.dataclass
+    class Sgd:
+        lr: float = 1e-4
+
+    @dataclasses.dataclass
+    class Adam:
+        lr: float = 3e-4
+        betas: tuple[float, float] = (0.9, 0.999)
+
+    @dataclasses.dataclass(frozen=True)
+    class Experiment[OptimizerT: (Adam, Sgd)]:
+        path: Path
+        opt: OptimizerT
+
+    if __name__ == "__main__":
+        args = tyro.cli(Experiment[Adam] | Experiment[Sgd])
+        print(args)
+
+
+
+
+.. raw:: html
+
+    
+    $ python ./03_generic_subcommands.py --help
+    usage: 03_generic_subcommands.py [-h] {experiment-adam,experiment-sgd}
+    
+    ╭─ options ─────────────────────────────────────────╮
+    │ -h, --help        show this help message and exit │
+    ╰───────────────────────────────────────────────────╯
+    ╭─ subcommands ─────────────────────────────────────╮
+    │ {experiment-adam,experiment-sgd}                  │
+    │     experiment-adam                               │
+    │     experiment-sgd                                │
+    ╰───────────────────────────────────────────────────╯
+    

+
+
+
+.. raw:: html
+
+    
+    $ python ./03_generic_subcommands.py experiment-adam --help
+    usage: 03_generic_subcommands.py experiment-adam [-h] --path PATH
+                                                     [--opt.lr FLOAT]
+                                                     [--opt.betas FLOAT FLOAT]
+    
+    ╭─ options ─────────────────────────────────────────────╮
+    │ -h, --help            show this help message and exit │
+    │ --path PATH           (required)                      │
+    ╰───────────────────────────────────────────────────────╯
+    ╭─ opt options ─────────────────────────────────────────╮
+    │ --opt.lr FLOAT        (default: 0.0003)               │
+    │ --opt.betas FLOAT FLOAT                               │
+    │                       (default: 0.9 0.999)            │
+    ╰───────────────────────────────────────────────────────╯
+    

+
+
+
+.. raw:: html
+
+    
+    $ python ./03_generic_subcommands.py experiment-sgd --help
+    usage: 03_generic_subcommands.py experiment-sgd [-h] --path PATH
+                                                    [--opt.lr FLOAT]
+    
+    ╭─ options ─────────────────────────────────────────────╮
+    │ -h, --help            show this help message and exit │
+    │ --path PATH           (required)                      │
+    ╰───────────────────────────────────────────────────────╯
+    ╭─ opt options ─────────────────────────────────────────╮
+    │ --opt.lr FLOAT        (default: 0.0001)               │
+    ╰───────────────────────────────────────────────────────╯
+    

+
+
+
+.. raw:: html
+
+    
+    $ python ./03_generic_subcommands.py experiment-adam --path /tmp --lr 1e-3
+    ╭─ Unrecognized options ──────────────────────────────────────────╮
+    │ Unrecognized options: --lr                                      │
+    │ ─────────────────────────────────────────────────────────────── │
+    │ Perhaps you meant:                                              │
+    │     --opt.lr FLOAT                                              │
+    │         (default: 0.0003)                                       │
+    │             in 03_generic_subcommands.py experiment-adam --help │
+    │         (default: 0.0001)                                       │
+    │             in 03_generic_subcommands.py experiment-sgd --help  │
+    │ ─────────────────────────────────────────────────────────────── │
+    │ For full helptext, run 03_generic_subcommands.py --help         │
+    ╰─────────────────────────────────────────────────────────────────╯
     

\ No newline at end of file
diff --git a/docs/source/examples/hierarchical_structures.rst b/docs/source/examples/hierarchical_structures.rst
index 67021bdea..4fbf82f23 100644
--- a/docs/source/examples/hierarchical_structures.rst
+++ b/docs/source/examples/hierarchical_structures.rst
@@ -91,7 +91,7 @@ objects. This helps with modularity and grouping in larger projects.
 Structures as Function Arguments
 --------------------------------
 
-Structures can also be used as input to functions.
+Structures can be used as input to functions.
 
 
 .. code-block:: python
@@ -275,10 +275,10 @@ Dictionaries and TypedDict
 --------------------------
 
 Dictionary inputs can be specified using either a standard ``dict[K, V]``
-annotation, or a :code:`TypedDict` subclass.
+annotation, or a :py:class:`TypedDict` subclass.
 
-For configuring :code:`TypedDict`, we also support :code:`total={True/False}`,
-:code:`typing.Required`, and :code:`typing.NotRequired`. See the `Python docs `_ for all :code:`TypedDict` features.
+For configuring :py:class:`TypedDict`, we also support :code:`total={True/False}`,
+:py:data:`typing.Required`, and :py:data:`typing.NotRequired`. See the `Python docs `_ for all :py:class:`TypedDict` features.
 
 
 .. code-block:: python
@@ -384,7 +384,7 @@ For configuring :code:`TypedDict`, we also support :code:`total={True/False}`,
 Tuples and NamedTuple
 ---------------------
 
-Example using :func:`tyro.cli()` to instantiate tuple types. :code:`tuple`,
+Example using :func:`tyro.cli()` to instantiate tuple types. :py:class:`tuple`,
 :py:data:`typing.Tuple`, and :py:class:`typing.NamedTuple` are all supported.
 
 
diff --git a/docs/source/examples/pytorch_jax.rst b/docs/source/examples/pytorch_jax.rst
index fe9735c61..23b1622cc 100644
--- a/docs/source/examples/pytorch_jax.rst
+++ b/docs/source/examples/pytorch_jax.rst
@@ -18,7 +18,7 @@ The :code:`console_outputs=` argument can be set to :code:`False` to suppress he
 error message printing.
 
 This is useful in PyTorch for distributed training scripts, where you only want
-to print the helptext from the main process:
+to print helptext from the main process:
 
 
 .. code-block:: python
@@ -37,23 +37,13 @@ to print the helptext from the main process:
     :linenos:
 
     # 01_pytorch_parallelism.py
-    import dataclasses
-
     import tyro
 
-    @dataclasses.dataclass
-    class Args:
-        """Description.
-        This should show up in the helptext!"""
-
-        field1: int
-        """A field."""
-
-        field2: int = 3
-        """A numeric field, with a default value."""
+    def train(foo: int, bar: str) -> None:
+        """Description. This should show up in the helptext!"""
 
     if __name__ == "__main__":
-        args = tyro.cli(Args, console_outputs=False)
+        args = tyro.cli(train, console_outputs=False)
         print(args)
 
 
diff --git a/docs/source/index.md b/docs/source/index.md
index 03f06c51c..672e89107 100644
--- a/docs/source/index.md
+++ b/docs/source/index.md
@@ -1,6 +1,6 @@
 # tyro
 
-|build| |nbsp| |ruff| |nbsp| |mypy| |nbsp| |pyright| |nbsp| |coverage| |nbsp| |versions|
+|ruff| |nbsp| |mypy| |nbsp| |pyright| |nbsp| |coverage| |nbsp| |versions|
 
 :func:`tyro.cli()` is a tool for generating CLI interfaces in Python.
 
@@ -114,9 +114,6 @@ shell completion.
 
 
 
-.. |build| image:: https://github.com/brentyi/tyro/actions/workflows/build.yml/badge.svg
-   :alt: Build status icon
-   :target: https://github.com/brentyi/tyro
 .. |mypy| image:: https://github.com/brentyi/tyro/actions/workflows/mypy.yml/badge.svg
    :alt: Mypy status icon
    :target: https://github.com/brentyi/tyro
diff --git a/examples/01_basics/05_choices.py b/examples/01_basics/05_choices.py
index 602eaef9a..97679158b 100644
--- a/examples/01_basics/05_choices.py
+++ b/examples/01_basics/05_choices.py
@@ -1,6 +1,6 @@
 """Choices
 
-:code:`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices.
+:py:data:`typing.Literal[]` can be used to restrict inputs to a fixed set of literal choices.
 
 Usage:
 
diff --git a/examples/01_basics/07_unions.py b/examples/01_basics/07_unions.py
index 3289d0e05..7f06ff1fe 100644
--- a/examples/01_basics/07_unions.py
+++ b/examples/01_basics/07_unions.py
@@ -1,6 +1,6 @@
 """Unions
 
-:code:`X | Y` or :code:`typing.Union[X, Y]` can be used to expand inputs to
+:code:`X | Y` or :py:data:`typing.Union` can be used to expand inputs to
 multiple types.
 
 Usage:
diff --git a/examples/02_hierarchical_structures/02_nesting_in_func.py b/examples/02_hierarchical_structures/02_nesting_in_func.py
index 893994463..d636b1a29 100644
--- a/examples/02_hierarchical_structures/02_nesting_in_func.py
+++ b/examples/02_hierarchical_structures/02_nesting_in_func.py
@@ -1,6 +1,6 @@
 """Structures as Function Arguments
 
-Structures can also be used as input to functions.
+Structures can be used as input to functions.
 
 Usage:
 
diff --git a/examples/02_hierarchical_structures/04_dictionaries.py b/examples/02_hierarchical_structures/04_dictionaries.py
index b1fa51086..70c61d84c 100644
--- a/examples/02_hierarchical_structures/04_dictionaries.py
+++ b/examples/02_hierarchical_structures/04_dictionaries.py
@@ -1,10 +1,10 @@
 """Dictionaries and TypedDict
 
 Dictionary inputs can be specified using either a standard ``dict[K, V]``
-annotation, or a :code:`TypedDict` subclass.
+annotation, or a :py:class:`TypedDict` subclass.
 
-For configuring :code:`TypedDict`, we also support :code:`total={True/False}`,
-:code:`typing.Required`, and :code:`typing.NotRequired`. See the `Python docs `_ for all :code:`TypedDict` features.
+For configuring :py:class:`TypedDict`, we also support :code:`total={True/False}`,
+:py:data:`typing.Required`, and :py:data:`typing.NotRequired`. See the `Python docs `_ for all :py:class:`TypedDict` features.
 
 Usage:
 
diff --git a/examples/02_hierarchical_structures/05_tuples.py b/examples/02_hierarchical_structures/05_tuples.py
index d9e839813..4ad213f8f 100644
--- a/examples/02_hierarchical_structures/05_tuples.py
+++ b/examples/02_hierarchical_structures/05_tuples.py
@@ -1,6 +1,6 @@
 """Tuples and NamedTuple
 
-Example using :func:`tyro.cli()` to instantiate tuple types. :code:`tuple`,
+Example using :func:`tyro.cli()` to instantiate tuple types. :py:class:`tuple`,
 :py:data:`typing.Tuple`, and :py:class:`typing.NamedTuple` are all supported.
 
 Usage:
diff --git a/examples/05_generics/02_generics.py b/examples/05_generics/02_generics.py
index 698d15e8c..a881d1158 100644
--- a/examples/05_generics/02_generics.py
+++ b/examples/05_generics/02_generics.py
@@ -1,4 +1,4 @@
-"""Generics (Legacy)
+"""Generics (Python <3.12)
 
 The legacy :py:class:`typing.Generic` and :py:class:`typing.TypeVar` syntax for
 generic types is also supported.
diff --git a/examples/05_generics/03_generic_subcommands.py b/examples/05_generics/03_generic_subcommands.py
new file mode 100644
index 000000000..663bd3c92
--- /dev/null
+++ b/examples/05_generics/03_generic_subcommands.py
@@ -0,0 +1,40 @@
+# mypy: ignore-errors
+#
+# Passing a Union type directly to tyro.cli() doesn't type-check correctly in
+# mypy. This will be fixed by `typing.TypeForm`: https://peps.python.org/pep-0747/
+"""Generic Subcommands
+
+Usage:
+
+    python ./03_generic_subcommands.py --help
+    python ./03_generic_subcommands.py experiment-adam --help
+    python ./03_generic_subcommands.py experiment-sgd --help
+    python ./03_generic_subcommands.py experiment-adam --path /tmp --lr 1e-3
+"""
+
+import dataclasses
+from pathlib import Path
+
+import tyro
+
+
+@dataclasses.dataclass
+class Sgd:
+    lr: float = 1e-4
+
+
+@dataclasses.dataclass
+class Adam:
+    lr: float = 3e-4
+    betas: tuple[float, float] = (0.9, 0.999)
+
+
+@dataclasses.dataclass(frozen=True)
+class Experiment[OptimizerT: (Adam, Sgd)]:
+    path: Path
+    opt: OptimizerT
+
+
+if __name__ == "__main__":
+    args = tyro.cli(Experiment[Adam] | Experiment[Sgd])
+    print(args)
diff --git a/examples/06_custom_constructors/01_simple_constructors.py b/examples/06_custom_constructors/01_simple_constructors.py
index 9c144e258..ebce9cd3f 100644
--- a/examples/06_custom_constructors/01_simple_constructors.py
+++ b/examples/06_custom_constructors/01_simple_constructors.py
@@ -1,11 +1,10 @@
 """Simple Constructors
 
 For simple custom constructors, we can pass a constructor function into
-:func:`tyro.conf.arg` or :func:`tyro.conf.subcommand`. Arguments for will be
+:func:`tyro.conf.arg` or :func:`tyro.conf.subcommand`. Arguments will be
 generated by parsing the signature of the constructor function.
 
-In this example, we use this pattern to define custom behavior for
-instantiating a NumPy array.
+In this example, we define custom behavior for instantiating a NumPy array.
 
 Usage:
 
diff --git a/examples/06_custom_constructors/04_struct_registry.py b/examples/06_custom_constructors/04_struct_registry.py
index da09e992b..b5b13f5c9 100644
--- a/examples/06_custom_constructors/04_struct_registry.py
+++ b/examples/06_custom_constructors/04_struct_registry.py
@@ -1,7 +1,9 @@
 """Custom Structs (Registry)
 
 In this example, we use a :class:`tyro.constructors.ConstructorRegistry` to
-add support for a custom type.
+add support for a custom struct type. Each struct type is broken down into
+multiple fields, which themselves can be either primitive types or nested
+structs.
 
 .. warning::
 
diff --git a/examples/07_pytorch_jax/01_pytorch_parallelism.py b/examples/07_pytorch_jax/01_pytorch_parallelism.py
index 97ef6a74a..92510e0e0 100644
--- a/examples/07_pytorch_jax/01_pytorch_parallelism.py
+++ b/examples/07_pytorch_jax/01_pytorch_parallelism.py
@@ -4,7 +4,7 @@
 error message printing.
 
 This is useful in PyTorch for distributed training scripts, where you only want
-to print the helptext from the main process:
+to print helptext from the main process:
 
 
 .. code-block:: python
@@ -24,23 +24,13 @@
     python ./01_pytorch_parallelism.py --help
 """
 
-import dataclasses
-
 import tyro
 
 
-@dataclasses.dataclass
-class Args:
-    """Description.
-    This should show up in the helptext!"""
-
-    field1: int
-    """A field."""
-
-    field2: int = 3
-    """A numeric field, with a default value."""
+def train(foo: int, bar: str) -> None:
+    """Description. This should show up in the helptext!"""
 
 
 if __name__ == "__main__":
-    args = tyro.cli(Args, console_outputs=False)
+    args = tyro.cli(train, console_outputs=False)
     print(args)
diff --git a/pyproject.toml b/pyproject.toml
index 8110e3171..21c04cded 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -31,6 +31,7 @@ classifiers = [
     "Operating System :: OS Independent"
 ]
 dependencies = [
+    "typeguard>=4.0.0",
     "docstring-parser>=0.16",
     "typing-extensions>=4.9.0; python_version>='3.8'",
     "typing-extensions>=4.7.0; python_version<'3.8'",
@@ -57,7 +58,7 @@ dev = [
     # As of 7/27/2023, flax install fails for Python 3.7 without pinning to an
     # old version. But doing so breaks other Python versions.
     "flax>=0.6.9;python_version>='3.8' and python_version<='3.12'",
-    "pydantic>=2.5.2",
+    "pydantic>=2.5.2,!=2.10.0",
     "coverage[toml]>=6.5.0",
     "eval_type_backport>=0.1.3",
 ]
diff --git a/src/tyro/_arguments.py b/src/tyro/_arguments.py
index 56e0e2b8e..10506f063 100644
--- a/src/tyro/_arguments.py
+++ b/src/tyro/_arguments.py
@@ -291,11 +291,11 @@ def _rule_apply_primitive_specs(
     except UnsupportedTypeAnnotationError as e:
         if arg.field.default in _singleton.MISSING_AND_MISSING_NONPROP:
             field_name = _strings.make_field_name(
-                [arg.extern_prefix, arg.field.intern_name]
+                [arg.extern_prefix, arg.field.extern_name]
             )
             if field_name != "":
                 raise UnsupportedTypeAnnotationError(
-                    f"Unsupported type annotation for the field {field_name}; "
+                    f"Unsupported type annotation for field with name `{field_name}`, which is annotated as `{arg.field.type}`. "
                     f"{e.args[0]} "
                     "To suppress this error, assign the field either a default value or a different type."
                 ) from e
diff --git a/src/tyro/_cli.py b/src/tyro/_cli.py
index 81971c874..864ab9056 100644
--- a/src/tyro/_cli.py
+++ b/src/tyro/_cli.py
@@ -9,7 +9,7 @@
 from typing import Callable, Sequence, TypeVar, cast, overload
 
 import shtab
-from typing_extensions import Literal
+from typing_extensions import Annotated, Literal
 
 from . import _argparse as argparse
 from . import (
@@ -163,7 +163,7 @@ def cli(
     _unsafe_cache.clear_cache()
 
     if config is not None:
-        f = conf.configure(*config)(f)
+        f = Annotated[(f, *config)]  # type: ignore
 
     with _strings.delimeter_context("_" if use_underscores else "-"):
         output = _cli_impl(
diff --git a/src/tyro/_fields.py b/src/tyro/_fields.py
index e4701ae21..38792135c 100644
--- a/src/tyro/_fields.py
+++ b/src/tyro/_fields.py
@@ -7,8 +7,6 @@
 import dataclasses
 import functools
 import inspect
-import numbers
-import warnings
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
 
 import docstring_parser
@@ -16,7 +14,6 @@
 
 from . import _docstrings, _resolver, _strings, _unsafe_cache
 from ._singleton import (
-    DEFAULT_SENTINEL_SINGLETONS,
     MISSING_AND_MISSING_NONPROP,
     MISSING_NONPROP,
 )
@@ -72,8 +69,10 @@ def marker_context(markers: Tuple[_markers.Marker, ...]):
         """Context for setting markers on fields. All fields created within the
         context will have the specified markers."""
         global_context_markers.append(markers)
-        yield
-        global_context_markers.pop()
+        try:
+            yield
+        finally:
+            global_context_markers.pop()
 
     @staticmethod
     def from_field_spec(field_spec: StructFieldSpec) -> FieldDefinition:
@@ -158,32 +157,6 @@ def make(
         if argconf.constructor_factory is not None:
             out = out.with_new_type_stripped(argconf.constructor_factory())
 
-        # Check that the default value matches the final resolved type.
-        # There's some similar Union-specific logic for this in narrow_union_type(). We
-        # may be able to consolidate this.
-        if (
-            # Be relatively conservative: isinstance() can be checked on non-type
-            # types (like unions in Python >=3.10), but we'll only consider single types
-            # for now.
-            type(out.type_stripped) is type
-            and not isinstance(default, out.type_stripped)  # type: ignore
-            # If a custom constructor is set, static_type may not be
-            # matched to the annotated type.
-            and argconf.constructor_factory is None
-            and default not in DEFAULT_SENTINEL_SINGLETONS
-            # The numeric tower in Python is wacky. This logic is non-critical, so
-            # we'll just skip it (+the complexity) for numbers.
-            and not isinstance(default, numbers.Number)
-        ):
-            # If the default value doesn't match the resolved type, we expand the
-            # type. This is inspired by https://github.com/brentyi/tyro/issues/88.
-            warnings.warn(
-                f"The field {name} is annotated with type {typ}, "
-                f"but the default value {default} has type {type(default)}. "
-                f"We'll try to handle this gracefully, but it may cause unexpected behavior."
-            )
-            out = out.with_new_type_stripped(Union[out.type_stripped, type(default)])  # type: ignore
-
         return out
 
     def with_new_type_stripped(
@@ -303,6 +276,10 @@ def _field_list_from_function(
     f: Callable, default_instance: Any
 ) -> UnsupportedStructTypeMessage | tuple[Callable, list[FieldDefinition]]:
     """Generate field lists from non-class callables."""
+
+    if f is Any:
+        return UnsupportedStructTypeMessage("`Any` is not a valid struct type!")
+
     try:
         params = list(inspect.signature(f).parameters.values())
     except ValueError:
diff --git a/src/tyro/_parsers.py b/src/tyro/_parsers.py
index b380f9b1f..436a3c71d 100644
--- a/src/tyro/_parsers.py
+++ b/src/tyro/_parsers.py
@@ -3,6 +3,8 @@
 from __future__ import annotations
 
 import dataclasses
+import numbers
+import warnings
 from typing import (
     Any,
     Callable,
@@ -321,6 +323,32 @@ def handle_field(
 
     registry = ConstructorRegistry._get_active_registry()
 
+    # Check that the default value matches the final resolved type.
+    # There's some similar Union-specific logic for this in narrow_union_type(). We
+    # may be able to consolidate this.
+    if (
+        not _resolver.is_instance(field.type_stripped, field.default)
+        # If a custom constructor is set, static_type may not be
+        # matched to the annotated type.
+        and field.argconf.constructor_factory is None
+        and field.default not in _singleton.DEFAULT_SENTINEL_SINGLETONS
+        # The numeric tower in Python is wacky. This logic is non-critical, so
+        # we'll just skip it (+the complexity) for numbers.
+        and not isinstance(field.default, numbers.Number)
+    ):
+        # If the default value doesn't match the resolved type, we expand the
+        # type. This is inspired by https://github.com/brentyi/tyro/issues/88.
+        field_name = _strings.make_field_name([extern_prefix, field.extern_name])
+        message = (
+            f"The field `{field_name}` is annotated with type `{field.type}`, "
+            f"but the default value `{field.default}` has type `{type(field.default)}`. "
+            f"We'll try to handle this gracefully, but it may cause unexpected behavior."
+        )
+        warnings.warn(message)
+        field = field.with_new_type_stripped(
+            Union[field.type_stripped, type(field.default)]  # type: ignore
+        )
+
     # Force primitive if (1) the field is annotated with a primitive constructor spec, or (2) if
     force_primitive = len(
         _resolver.unwrap_annotated(field.type, PrimitiveConstructorSpec)[1]
diff --git a/src/tyro/_resolver.py b/src/tyro/_resolver.py
index 41a715448..845bc69f7 100644
--- a/src/tyro/_resolver.py
+++ b/src/tyro/_resolver.py
@@ -25,6 +25,7 @@
     overload,
 )
 
+import typeguard
 from typing_extensions import (
     Annotated,
     Final,
@@ -176,6 +177,9 @@ def narrow_subtypes(
 
     typ = resolve_newtype_and_aliases(typ)
 
+    if default_instance in MISSING_AND_MISSING_NONPROP:
+        return typ
+
     try:
         potential_subclass = type(default_instance)
 
@@ -650,3 +654,12 @@ def get_type_hints_with_backported_syntax(
             except ImportError:
                 pass
         raise e
+
+
+def is_instance(typ: Any, value: Any) -> bool:
+    """Typeguard-based alternative for `isinstance()`."""
+    try:
+        typeguard.check_type(value, typ)
+        return True
+    except (typeguard.TypeCheckError, TypeError):
+        return False
diff --git a/src/tyro/_subcommand_matching.py b/src/tyro/_subcommand_matching.py
index e4168bc64..db8d93bb3 100644
--- a/src/tyro/_subcommand_matching.py
+++ b/src/tyro/_subcommand_matching.py
@@ -1,13 +1,14 @@
 from __future__ import annotations
 
-import dataclasses
-from typing import Any, Callable, Dict, Optional, Tuple, Union
+from typing import Any, Dict, Optional
 
-from typing_extensions import get_args, get_origin
+from tyro.constructors._registry import check_default_instances_context
+from tyro.constructors._struct_spec import (
+    InvalidDefaultInstanceError,
+    UnsupportedStructTypeMessage,
+)
 
-from tyro.constructors._struct_spec import UnsupportedStructTypeMessage
-
-from . import _fields, _resolver, _singleton, _typing
+from . import _fields, _singleton
 from .conf import _confstruct
 
 
@@ -39,26 +40,20 @@ def match_subcommand(
         if isinstance(equal, bool) and equal:
             return subcommand_name
 
-    # Get default subcommand name: by concrete type tree.
-    typetree = _TypeTree.make(type(default), default)
-    for subcommand_name, conf in subcommand_config_from_name.items():
-        if conf.default in _singleton.MISSING_AND_MISSING_NONPROP:
-            continue
-        if typetree == _TypeTree.make(type(conf.default), conf.default):
-            return subcommand_name
+    # Get first subcommand that doesn't throw an error in strict mode.
+    for subcommand_name, subcommand_type in subcommand_type_from_name.items():
+        # We could also use typeguard here, but for now (November 19, 2024)
+        # our own implementation has better support for nested generics.
 
-    # Get default subcommand name: by annotated type tree.
-    typetree_from_name = {
-        subcommand_name: _TypeTree.make(subcommand_type, _singleton.MISSING_NONPROP)
-        for subcommand_name, subcommand_type in subcommand_type_from_name.items()
-    }
-    for subcommand_name in subcommand_type_from_name.keys():
-        # Equality check for type tree.
-        if typetree == typetree_from_name[subcommand_name]:
-            return subcommand_name
-    for subcommand_name in subcommand_type_from_name.keys():
-        # Leaky subtype check.
-        if typetree.is_subtype_of(typetree_from_name[subcommand_name]):
+        # try:
+        #     import typeguard
+        #
+        #     typeguard.check_type(default, subcommand_type)
+        #     return subcommand_name
+        # except typeguard.TypeCheckError:
+        #     continue
+
+        if _recursive_struct_match(subcommand_type, default, root=True):
             return subcommand_name
 
     # Failed. This should never happen, we'll raise an error outside of this function if
@@ -66,64 +61,26 @@ def match_subcommand(
     return None  # pragma: no cover
 
 
-@dataclasses.dataclass(frozen=True)
-class _TypeTree:
-    typ: Any
-    children: Dict[str, _TypeTree]
-
-    @staticmethod
-    def make(
-        typ: Union[Callable, _typing.TypeForm],
-        default_instance: Any,
-    ) -> _TypeTree:
-        """From an object instance, return a data structure representing the types in the object."""
-
-        typ_unwrap = _resolver.resolve_generic_types(typ)[0]
-
-        field_list_out = _fields.field_list_from_type_or_callable(
-            typ, default_instance=default_instance, support_single_arg_types=False
-        )
-        if isinstance(field_list_out, UnsupportedStructTypeMessage):
-            return _TypeTree(typ_unwrap, {})
-
-        typ, field_list = field_list_out
-        return _TypeTree(
-            typ_unwrap,
-            {
-                field.intern_name: _TypeTree.make(field.type_stripped, field.default)
-                for field in field_list
-            },
-        )
-
-    def is_subtype_of(self, supertype: _TypeTree) -> bool:
-        """Best-effort subcommand matching."""
-
-        # Generalize to unions.
-        def _get_type_options(typ: _typing.TypeForm) -> Tuple[_typing.TypeForm, ...]:
-            return get_args(typ) if get_origin(typ) is Union else (typ,)
-
-        self_types = _get_type_options(self.typ)
-        super_types = _get_type_options(supertype.typ)
-
-        # Check against supertypes. TODO: the heuristics below could be more
-        # principled. We should revisit.
-        for self_type in self_types:
-            self_type = _resolver.unwrap_annotated(self_type)
-            ok = False
-            for super_type in super_types:
-                super_type = _resolver.unwrap_annotated(super_type)
-                try:
-                    if issubclass(self_type, super_type):
-                        ok = True
-                except TypeError:
-                    pass
-            if not ok:
-                return False
-
-        for child_name, child in self.children.items():
-            if child_name not in supertype.children or not child.is_subtype_of(
-                supertype.children[child_name]
-            ):
-                return False
-
+def _recursive_struct_match(subcommand_type: Any, default: Any, root: bool) -> bool:
+    """Returns `True` if the given type and default instance are compatible
+    with each other."""
+    # Can we generate a field list from this type?
+    try:
+        with check_default_instances_context():
+            field_list = _fields.field_list_from_type_or_callable(
+                subcommand_type, default, support_single_arg_types=root
+            )
+    except InvalidDefaultInstanceError:
+        # Found a struct type that matches, but the default instance isn't
+        # compatible.
+        return False
+
+    # Base case: found a leaf.
+    if isinstance(field_list, UnsupportedStructTypeMessage):
         return True
+
+    for field in field_list[1]:
+        if not _recursive_struct_match(field.type, field.default, root=False):
+            return False
+
+    return True
diff --git a/src/tyro/constructors/_primitive_spec.py b/src/tyro/constructors/_primitive_spec.py
index 5cf4567b0..d3418c665 100644
--- a/src/tyro/constructors/_primitive_spec.py
+++ b/src/tyro/constructors/_primitive_spec.py
@@ -249,11 +249,9 @@ def enum_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None:
             ),
             is_instance=lambda x: isinstance(x, cast_type),
             str_from_instance=lambda instance: [
-                (
-                    str(instance.value)
-                    if _markers.EnumChoicesFromValues in type_info.markers
-                    else instance.name
-                )
+                str(instance.value)
+                if _markers.EnumChoicesFromValues in type_info.markers
+                else instance.name
             ],
             choices=choices,
         )
diff --git a/src/tyro/constructors/_registry.py b/src/tyro/constructors/_registry.py
index 781d406aa..d62623a03 100644
--- a/src/tyro/constructors/_registry.py
+++ b/src/tyro/constructors/_registry.py
@@ -1,9 +1,13 @@
 from __future__ import annotations
 
+from contextlib import contextmanager
 from typing import Any, Callable, ClassVar, Union
 
 from typing_extensions import Literal
 
+from tyro._singleton import DEFAULT_SENTINEL_SINGLETONS
+
+from .. import _resolver
 from ._primitive_spec import (
     PrimitiveConstructorSpec,
     PrimitiveTypeInfo,
@@ -11,6 +15,7 @@
     apply_default_primitive_rules,
 )
 from ._struct_spec import (
+    InvalidDefaultInstanceError,
     StructConstructorSpec,
     StructTypeInfo,
     apply_default_struct_rules,
@@ -21,6 +26,37 @@
 PrimitiveSpecRule = Callable[[PrimitiveTypeInfo], Union[PrimitiveConstructorSpec, None]]
 StructSpecRule = Callable[[StructTypeInfo], Union[StructConstructorSpec, None]]
 
+_check_default_instances_flag: bool = False
+
+
+def check_default_instances() -> bool:
+    """Check whether we should be strict about checking that default types and
+    instances match.
+
+    This is usually `False`; tyro attempts to be somewhat lenient when
+    inconsistent types are encounted. Strictness, however, is useful for
+    matching annotated subcommands to default values.
+    """
+    return _check_default_instances_flag
+
+
+@contextmanager
+def check_default_instances_context():
+    """Context for whether we should be strict about checking that default
+    types and instances match.
+
+    This is usually `False`; tyro attempts to be somewhat lenient when
+    inconsistent types are encounted. Strictness, however, is useful for
+    matching annotated subcommands to default values.
+    """
+    global _check_default_instances_flag
+    old_value = _check_default_instances_flag
+    _check_default_instances_flag = True
+    try:
+        yield
+    finally:
+        _check_default_instances_flag = old_value
+
 
 class ConstructorRegistry:
     """Registry for rules that define how types are constructed from
@@ -121,6 +157,15 @@ def get_struct_spec(
         """Get a constructor specification for a given type. Returns `None` if
         unsuccessful."""
 
+        if (
+            check_default_instances()
+            and type_info.default not in DEFAULT_SENTINEL_SINGLETONS
+            and not _resolver.is_instance(type_info.type, type_info.default)
+        ):
+            raise InvalidDefaultInstanceError(
+                f"Invalid default instance for type {type_info.type}: {type_info.default}"
+            )
+
         for spec_factory in self._struct_rules[::-1]:
             maybe_spec = spec_factory(type_info)
             if maybe_spec is not None:
diff --git a/src/tyro/constructors/_struct_spec.py b/src/tyro/constructors/_struct_spec.py
index 97be03bd8..2ca5d27f6 100644
--- a/src/tyro/constructors/_struct_spec.py
+++ b/src/tyro/constructors/_struct_spec.py
@@ -38,6 +38,13 @@ class UnsupportedStructTypeMessage:
     message: str
 
 
+class InvalidDefaultInstanceError(Exception):
+    """Raised when a default instance is not applicable to an annoated struct type."""
+
+    def __init__(self, message: str):
+        super().__init__(message)
+
+
 @dataclasses.dataclass(frozen=True)
 class StructFieldSpec:
     """Behavior specification for a single field in our callable."""
@@ -271,16 +278,9 @@ def attrs_rule(info: StructTypeInfo) -> StructConstructorSpec | None:
             default = attr_field.default
             is_default_from_default_instance = False
             if info.default not in MISSING_AND_MISSING_NONPROP:
-                if hasattr(info.default, name):
-                    default = getattr(info.default, name)
-                    is_default_from_default_instance = True
-                else:
-                    warnings.warn(
-                        f"Could not find field {name} in default instance"
-                        f" {info.default}, which has"
-                        f" type {type(info.default)},",
-                        stacklevel=2,
-                    )
+                assert hasattr(info.default, name)
+                default = getattr(info.default, name)
+                is_default_from_default_instance = True
             elif default is attr.NOTHING:
                 default = MISSING_NONPROP
             elif isinstance(default, attr.Factory):  # type: ignore
@@ -589,13 +589,6 @@ def _get_dataclass_field_default(
         # Populate default from some parent, eg `default=` in `tyro.cli()`.
         if hasattr(parent_default_instance, field.name):
             return getattr(parent_default_instance, field.name), True
-        else:
-            warnings.warn(
-                f"Could not find field {field.name} in default instance"
-                f" {parent_default_instance}, which has"
-                f" type {type(parent_default_instance)},",
-                stacklevel=2,
-            )
 
     # Try grabbing default from dataclass field.
     if (
@@ -647,13 +640,6 @@ def _get_pydantic_v1_field_default(
         # Populate default from some parent, eg `default=` in `tyro.cli()`.
         if hasattr(parent_default_instance, name):
             return getattr(parent_default_instance, name), True
-        else:
-            warnings.warn(
-                f"Could not find field {name} in default instance"
-                f" {parent_default_instance}, which has"
-                f" type {type(parent_default_instance)},",
-                stacklevel=2,
-            )
 
     if not field.required:
         return field.get_default(), False
@@ -677,13 +663,6 @@ def _get_pydantic_v2_field_default(
         # Populate default from some parent, eg `default=` in `tyro.cli()`.
         if hasattr(parent_default_instance, name):
             return getattr(parent_default_instance, name), True
-        else:
-            warnings.warn(
-                f"Could not find field {name} in default instance"
-                f" {parent_default_instance}, which has"
-                f" type {type(parent_default_instance)},",
-                stacklevel=2,
-            )
 
     if not field.is_required():
         return field.get_default(call_default_factory=True), False
diff --git a/tests/conftest.py b/tests/conftest.py
index b4605fec8..326283e18 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -11,7 +11,9 @@
 
 if not sys.version_info >= (3, 10):
     collect_ignore_glob.append("*min_py310*.py")
-    collect_ignore_glob.append("*_min_py310_generated.py")
+
+if not sys.version_info >= (3, 11):
+    collect_ignore_glob.append("*min_py311*.py")
 
 if not sys.version_info >= (3, 12):
     collect_ignore_glob.append("*min_py312*.py")
diff --git a/tests/test_errors.py b/tests/test_errors.py
index 09ea7b532..2e33970a6 100644
--- a/tests/test_errors.py
+++ b/tests/test_errors.py
@@ -34,7 +34,7 @@ def main(x: Tuple[List[str], List[str]]) -> None:
 
     with pytest.raises(UnsupportedTypeAnnotationError) as e:
         tyro.cli(main, args=["--help"])
-    assert "Unsupported type annotation for the field x" in e.value.args[0]
+    assert "Unsupported type annotation for field with name `x`" in e.value.args[0]
 
 
 def test_ambiguous_collection_3() -> None:
@@ -43,14 +43,14 @@ def main(x: List[Union[Tuple[int, int], Tuple[int, int, int]]]) -> None:
 
     with pytest.raises(UnsupportedTypeAnnotationError) as e:
         tyro.cli(main, args=["--help"])
-    assert "Unsupported type annotation for the field x" in e.value.args[0]
+    assert "Unsupported type annotation for field with name `x`" in e.value.args[0]
 
 
 def test_ambiguous_collection_4() -> None:
     X = List[Union[Tuple[int, int], Tuple[int, int, int]]]
     with pytest.raises(UnsupportedTypeAnnotationError) as e:
         tyro.cli(X, args=["--help"])
-    assert "Unsupported type annotation for the field" not in e.value.args[0]
+    assert "Unsupported type annotation for field with name `x`" not in e.value.args[0]
 
 
 def test_ambiguous_collection_5() -> None:
@@ -71,6 +71,19 @@ class A:
         tyro.cli(A, args=["--x", "0", "1"])
 
 
+def test_ambiguous_collection_7() -> None:
+    @dataclasses.dataclass
+    class A:
+        x: Dict[List[int], str]
+
+    def main(x: A) -> None:
+        pass
+
+    with pytest.raises(UnsupportedTypeAnnotationError) as e:
+        tyro.cli(main, args=["--help"])
+    assert "Unsupported type annotation for field with name `x.x`" in e.value.args[0]
+
+
 # Must be global.
 @dataclasses.dataclass
 class _CycleDataclass:
diff --git a/tests/test_nested.py b/tests/test_nested.py
index 9bc1748bd..f510156df 100644
--- a/tests/test_nested.py
+++ b/tests/test_nested.py
@@ -1259,3 +1259,78 @@ class B(A):
     from tyro._resolver import narrow_subtypes
 
     assert narrow_subtypes(Annotated[A, False], B(3)) == Annotated[B, False]  # type: ignore
+
+
+def test_union_with_dict() -> None:
+    @dataclasses.dataclass(frozen=True)
+    class Config:
+        name: str
+        age: int
+
+    def main(config: Union[Config, dict] = {"name": "hello", "age": 25}) -> Any:
+        return config
+
+    assert tyro.cli(main, args=[]) == {"name": "hello", "age": 25}
+    assert tyro.cli(main, args="config:dict".split(" ")) == {"name": "hello", "age": 25}
+    assert tyro.cli(main, args="config:dict --config.age 27".split(" ")) == {
+        "name": "hello",
+        "age": 27,
+    }
+    assert tyro.cli(
+        main, args="config:config --config.name world --config.age 27".split(" ")
+    ) == Config(name="world", age=27)
+
+
+def test_union_with_tuple() -> None:
+    @dataclasses.dataclass(frozen=True)
+    class Config:
+        name: str
+        age: int
+
+    def main(config: Union[Config, tuple] = ("hello", 5)) -> Any:
+        return config
+
+    assert tyro.cli(main, args=[]) == ("hello", 5)
+    assert tyro.cli(main, args="config:tuple".split(" ")) == ("hello", 5)
+    assert tyro.cli(main, args="config:tuple hello 27".split(" ")) == ("hello", 27)
+    assert tyro.cli(
+        main, args="config:config --config.name world --config.age 27".split(" ")
+    ) == Config(name="world", age=27)
+
+
+def test_union_with_tuple_subscripted() -> None:
+    @dataclasses.dataclass(frozen=True)
+    class Config:
+        name: str
+        age: int
+
+    def main(config: Union[Config, Tuple[str, int]] = ("hello", 5)) -> Any:
+        return config
+
+    assert tyro.cli(main, args=[]) == ("hello", 5)
+    assert tyro.cli(main, args="config:tuple-str-int".split(" ")) == ("hello", 5)
+    assert tyro.cli(main, args="config:tuple-str-int hello 27".split(" ")) == (
+        "hello",
+        27,
+    )
+    assert tyro.cli(
+        main, args="config:config --config.name world --config.age 27".split(" ")
+    ) == Config(name="world", age=27)
+
+
+def test_union_with_tuple_autoexpand() -> None:
+    @dataclasses.dataclass(frozen=True)
+    class Config:
+        name: str
+        age: int
+
+    # tyro should automatically expand this `Config` type to `Config | tuple`.
+    def main(config: Config = ("hello", 5)) -> Any:  # type: ignore
+        return config
+
+    assert tyro.cli(main, args=[]) == ("hello", 5)
+    assert tyro.cli(main, args="config:tuple".split(" ")) == ("hello", 5)
+    assert tyro.cli(main, args="config:tuple hello 27".split(" ")) == ("hello", 27)
+    assert tyro.cli(
+        main, args="config:config --config.name world --config.age 27".split(" ")
+    ) == Config(name="world", age=27)
diff --git a/tests/test_new_style_annotations_min_py312.py b/tests/test_new_style_annotations_min_py312.py
index 0e2f4e8c7..2d4e4b50e 100644
--- a/tests/test_new_style_annotations_min_py312.py
+++ b/tests/test_new_style_annotations_min_py312.py
@@ -5,13 +5,14 @@
 from typing import Annotated, Any, Literal, NewType
 
 import pytest
+from helptext_utils import get_helptext_with_checks
 
 import tyro
 from tyro.conf._markers import OmitArgPrefixes
 from tyro.constructors import UnsupportedTypeAnnotationError
 
 
-def test_simple_generic():
+def test_simple_generic() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: T
@@ -31,7 +32,7 @@ class Inner[T]:
     b: T
 
 
-def test_generic_with_type_statement_0():
+def test_generic_with_type_statement_0() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: T
@@ -40,7 +41,7 @@ class Container[T]:
     assert tyro.cli(Container[X], args="--a 1 --b 2".split(" ")) == Container(1, 2)
 
 
-def test_generic_with_type_statement_1():
+def test_generic_with_type_statement_1() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: tuple[X, ...]
@@ -49,7 +50,7 @@ class Container[T]:
     assert tyro.cli(Container[Y], args="--a 1 --b 2".split(" ")) == Container((1,), [2])
 
 
-def test_generic_with_type_statement_2():
+def test_generic_with_type_statement_2() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: Z
@@ -62,7 +63,7 @@ class Container[T]:
 type AnnotatedBasic = Annotated[int, tyro.conf.arg(name="basic")]
 
 
-def test_annotated_alias():
+def test_annotated_alias() -> None:
     @dataclass(frozen=True)
     class Container:
         a: AnnotatedBasic
@@ -204,7 +205,7 @@ def main(arg: list[Int7], /) -> Any:
     assert tyro.cli(main, args=["1", "2"]) == [1, 2]
 
 
-def test_generic_config():
+def test_generic_config() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: Inner[T]
@@ -216,7 +217,7 @@ class Container[T]:
     ) == Container(Inner(True, False))
 
 
-def test_generic_config_subcommand():
+def test_generic_config_subcommand() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: T
@@ -243,7 +244,7 @@ class Container[T]:
     ) == Container(Container(False))
 
 
-def test_generic_config_subcommand2():
+def test_generic_config_subcommand2() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: tyro.conf.OmitSubcommandPrefixes[T]
@@ -254,7 +255,7 @@ class Container[T]:
     ) == Container(Container(True))
 
 
-def test_generic_config_subcommand3():
+def test_generic_config_subcommand3() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: T
@@ -267,7 +268,7 @@ class Container[T]:
     ) == Container(Container(True))
 
 
-def test_generic_config_subcommand4():
+def test_generic_config_subcommand4() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: T
@@ -277,7 +278,6 @@ class Container[T]:
         args="container-literal-1-2 --a 2".split(" "),
         config=(tyro.conf.OmitSubcommandPrefixes,),
     ) == Container(Container("2"))
-
     assert tyro.cli(
         Container[Container[bool] | Container[Literal["1", "2"]]],
         args=[],
@@ -285,11 +285,77 @@ class Container[T]:
         config=(tyro.conf.OmitSubcommandPrefixes,),
     ) == Container(Container(True))
 
-    # This case is currently too hard for tyro's subcommand matcher.
-    with pytest.raises(AssertionError):
-        tyro.cli(
-            Container[Container[bool] | Container[Literal["1", "2"]]],
-            args=[],
-            default=Container(Container(a="1")),
-            config=(tyro.conf.OmitSubcommandPrefixes,),
-        )
+
+def test_generic_config_subcommand_matching_nested() -> None:
+    @dataclass(frozen=True)
+    class Container[T]:
+        a: T
+
+    assert "default: a:container-bool" in get_helptext_with_checks(
+        Container[Container[bool] | Container[str]],
+        default=Container(Container(a=False)),
+    )
+    assert "default: a:container-str" in get_helptext_with_checks(
+        Container[Container[bool] | Container[str]],
+        default=Container(Container(a="False")),
+    )
+    assert "default: a:container-literal-1-2" in get_helptext_with_checks(
+        Container[Container[Literal[1, 2]] | Container[str]],
+        default=Container(Container(a=1)),
+    )
+    assert "default: a:container-str" in get_helptext_with_checks(
+        Container[Container[Literal[1, 2]] | Container[str]],
+        default=Container(Container(a="1")),
+    )
+
+    assert tyro.cli(
+        Container[Container[Literal["1", "2"]] | Container[bool]],
+        args=[],
+        default=Container(Container(a="1")),
+        config=(tyro.conf.OmitSubcommandPrefixes,),
+    ) == Container(Container("1"))
+
+
+def test_generic_config_subcommand_matching_dict() -> None:
+    @dataclass(frozen=True)
+    class Container[T]:
+        a: T
+
+    assert "default: container-dict-str-str" in get_helptext_with_checks(
+        Container[dict[str, int]] | Container[dict[str, str]],  # type: ignore
+        default=Container({"a": "text"}),
+    )
+    assert "default: container-dict-str-int" in get_helptext_with_checks(
+        Container[dict[str, int]] | Container[dict[str, str]],  # type: ignore
+        default=Container({"a": 5}),
+    )
+
+
+def test_generic_config_subcommand_matching_tuple() -> None:
+    @dataclass(frozen=True)
+    class Container[T]:
+        a: T
+
+    assert "default: container-tuple-str-str" in get_helptext_with_checks(
+        Container[tuple[str, int]] | Container[tuple[str, str]],  # type: ignore
+        default=Container(("a", "text")),
+    )
+    assert "default: container-tuple-str-int" in get_helptext_with_checks(
+        Container[tuple[str, int]] | Container[tuple[str, str]],  # type: ignore
+        default=Container(("a", 5)),
+    )
+
+
+def test_generic_config_subcommand_matching_tuple_variable() -> None:
+    @dataclass(frozen=True)
+    class Container[T]:
+        a: T
+
+    assert "default: container-tuple-str-ellipsis" in get_helptext_with_checks(
+        Container[tuple[str, ...]] | Container[tuple[int, ...]],  # type: ignore
+        default=Container(("a", "text")),
+    )
+    assert "default: container-tuple-int-ellipsis" in get_helptext_with_checks(
+        Container[tuple[str, ...]] | Container[tuple[int, ...]],  # type: ignore
+        default=Container((1, 2, 3)),
+    )
diff --git a/tests/test_py311_generated/test_errors_generated.py b/tests/test_py311_generated/test_errors_generated.py
index 7cbe5a546..779b6c48b 100644
--- a/tests/test_py311_generated/test_errors_generated.py
+++ b/tests/test_py311_generated/test_errors_generated.py
@@ -33,7 +33,7 @@ def main(x: Tuple[List[str], List[str]]) -> None:
 
     with pytest.raises(UnsupportedTypeAnnotationError) as e:
         tyro.cli(main, args=["--help"])
-    assert "Unsupported type annotation for the field x" in e.value.args[0]
+    assert "Unsupported type annotation for field with name `x`" in e.value.args[0]
 
 
 def test_ambiguous_collection_3() -> None:
@@ -42,14 +42,14 @@ def main(x: List[Tuple[int, int] | Tuple[int, int, int]]) -> None:
 
     with pytest.raises(UnsupportedTypeAnnotationError) as e:
         tyro.cli(main, args=["--help"])
-    assert "Unsupported type annotation for the field x" in e.value.args[0]
+    assert "Unsupported type annotation for field with name `x`" in e.value.args[0]
 
 
 def test_ambiguous_collection_4() -> None:
     X = List[Tuple[int, int] | Tuple[int, int, int]]
     with pytest.raises(UnsupportedTypeAnnotationError) as e:
         tyro.cli(X, args=["--help"])
-    assert "Unsupported type annotation for the field" not in e.value.args[0]
+    assert "Unsupported type annotation for field with name `x`" not in e.value.args[0]
 
 
 def test_ambiguous_collection_5() -> None:
@@ -70,6 +70,19 @@ class A:
         tyro.cli(A, args=["--x", "0", "1"])
 
 
+def test_ambiguous_collection_7() -> None:
+    @dataclasses.dataclass
+    class A:
+        x: Dict[List[int], str]
+
+    def main(x: A) -> None:
+        pass
+
+    with pytest.raises(UnsupportedTypeAnnotationError) as e:
+        tyro.cli(main, args=["--help"])
+    assert "Unsupported type annotation for field with name `x.x`" in e.value.args[0]
+
+
 # Must be global.
 @dataclasses.dataclass
 class _CycleDataclass:
diff --git a/tests/test_py311_generated/test_nested_generated.py b/tests/test_py311_generated/test_nested_generated.py
index 973245446..c47f383d2 100644
--- a/tests/test_py311_generated/test_nested_generated.py
+++ b/tests/test_py311_generated/test_nested_generated.py
@@ -1269,3 +1269,78 @@ class B(A):
     from tyro._resolver import narrow_subtypes
 
     assert narrow_subtypes(Annotated[A, False], B(3)) == Annotated[B, False]  # type: ignore
+
+
+def test_union_with_dict() -> None:
+    @dataclasses.dataclass(frozen=True)
+    class Config:
+        name: str
+        age: int
+
+    def main(config: Config | dict = {"name": "hello", "age": 25}) -> Any:
+        return config
+
+    assert tyro.cli(main, args=[]) == {"name": "hello", "age": 25}
+    assert tyro.cli(main, args="config:dict".split(" ")) == {"name": "hello", "age": 25}
+    assert tyro.cli(main, args="config:dict --config.age 27".split(" ")) == {
+        "name": "hello",
+        "age": 27,
+    }
+    assert tyro.cli(
+        main, args="config:config --config.name world --config.age 27".split(" ")
+    ) == Config(name="world", age=27)
+
+
+def test_union_with_tuple() -> None:
+    @dataclasses.dataclass(frozen=True)
+    class Config:
+        name: str
+        age: int
+
+    def main(config: Config | tuple = ("hello", 5)) -> Any:
+        return config
+
+    assert tyro.cli(main, args=[]) == ("hello", 5)
+    assert tyro.cli(main, args="config:tuple".split(" ")) == ("hello", 5)
+    assert tyro.cli(main, args="config:tuple hello 27".split(" ")) == ("hello", 27)
+    assert tyro.cli(
+        main, args="config:config --config.name world --config.age 27".split(" ")
+    ) == Config(name="world", age=27)
+
+
+def test_union_with_tuple_subscripted() -> None:
+    @dataclasses.dataclass(frozen=True)
+    class Config:
+        name: str
+        age: int
+
+    def main(config: Config | Tuple[str, int] = ("hello", 5)) -> Any:
+        return config
+
+    assert tyro.cli(main, args=[]) == ("hello", 5)
+    assert tyro.cli(main, args="config:tuple-str-int".split(" ")) == ("hello", 5)
+    assert tyro.cli(main, args="config:tuple-str-int hello 27".split(" ")) == (
+        "hello",
+        27,
+    )
+    assert tyro.cli(
+        main, args="config:config --config.name world --config.age 27".split(" ")
+    ) == Config(name="world", age=27)
+
+
+def test_union_with_tuple_autoexpand() -> None:
+    @dataclasses.dataclass(frozen=True)
+    class Config:
+        name: str
+        age: int
+
+    # tyro should automatically expand this `Config` type to `Config | tuple`.
+    def main(config: Config = ("hello", 5)) -> Any:  # type: ignore
+        return config
+
+    assert tyro.cli(main, args=[]) == ("hello", 5)
+    assert tyro.cli(main, args="config:tuple".split(" ")) == ("hello", 5)
+    assert tyro.cli(main, args="config:tuple hello 27".split(" ")) == ("hello", 27)
+    assert tyro.cli(
+        main, args="config:config --config.name world --config.age 27".split(" ")
+    ) == Config(name="world", age=27)
diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py
index 0e2f4e8c7..2d4e4b50e 100644
--- a/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py
+++ b/tests/test_py311_generated/test_new_style_annotations_min_py312_generated.py
@@ -5,13 +5,14 @@
 from typing import Annotated, Any, Literal, NewType
 
 import pytest
+from helptext_utils import get_helptext_with_checks
 
 import tyro
 from tyro.conf._markers import OmitArgPrefixes
 from tyro.constructors import UnsupportedTypeAnnotationError
 
 
-def test_simple_generic():
+def test_simple_generic() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: T
@@ -31,7 +32,7 @@ class Inner[T]:
     b: T
 
 
-def test_generic_with_type_statement_0():
+def test_generic_with_type_statement_0() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: T
@@ -40,7 +41,7 @@ class Container[T]:
     assert tyro.cli(Container[X], args="--a 1 --b 2".split(" ")) == Container(1, 2)
 
 
-def test_generic_with_type_statement_1():
+def test_generic_with_type_statement_1() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: tuple[X, ...]
@@ -49,7 +50,7 @@ class Container[T]:
     assert tyro.cli(Container[Y], args="--a 1 --b 2".split(" ")) == Container((1,), [2])
 
 
-def test_generic_with_type_statement_2():
+def test_generic_with_type_statement_2() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: Z
@@ -62,7 +63,7 @@ class Container[T]:
 type AnnotatedBasic = Annotated[int, tyro.conf.arg(name="basic")]
 
 
-def test_annotated_alias():
+def test_annotated_alias() -> None:
     @dataclass(frozen=True)
     class Container:
         a: AnnotatedBasic
@@ -204,7 +205,7 @@ def main(arg: list[Int7], /) -> Any:
     assert tyro.cli(main, args=["1", "2"]) == [1, 2]
 
 
-def test_generic_config():
+def test_generic_config() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: Inner[T]
@@ -216,7 +217,7 @@ class Container[T]:
     ) == Container(Inner(True, False))
 
 
-def test_generic_config_subcommand():
+def test_generic_config_subcommand() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: T
@@ -243,7 +244,7 @@ class Container[T]:
     ) == Container(Container(False))
 
 
-def test_generic_config_subcommand2():
+def test_generic_config_subcommand2() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: tyro.conf.OmitSubcommandPrefixes[T]
@@ -254,7 +255,7 @@ class Container[T]:
     ) == Container(Container(True))
 
 
-def test_generic_config_subcommand3():
+def test_generic_config_subcommand3() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: T
@@ -267,7 +268,7 @@ class Container[T]:
     ) == Container(Container(True))
 
 
-def test_generic_config_subcommand4():
+def test_generic_config_subcommand4() -> None:
     @dataclass(frozen=True)
     class Container[T]:
         a: T
@@ -277,7 +278,6 @@ class Container[T]:
         args="container-literal-1-2 --a 2".split(" "),
         config=(tyro.conf.OmitSubcommandPrefixes,),
     ) == Container(Container("2"))
-
     assert tyro.cli(
         Container[Container[bool] | Container[Literal["1", "2"]]],
         args=[],
@@ -285,11 +285,77 @@ class Container[T]:
         config=(tyro.conf.OmitSubcommandPrefixes,),
     ) == Container(Container(True))
 
-    # This case is currently too hard for tyro's subcommand matcher.
-    with pytest.raises(AssertionError):
-        tyro.cli(
-            Container[Container[bool] | Container[Literal["1", "2"]]],
-            args=[],
-            default=Container(Container(a="1")),
-            config=(tyro.conf.OmitSubcommandPrefixes,),
-        )
+
+def test_generic_config_subcommand_matching_nested() -> None:
+    @dataclass(frozen=True)
+    class Container[T]:
+        a: T
+
+    assert "default: a:container-bool" in get_helptext_with_checks(
+        Container[Container[bool] | Container[str]],
+        default=Container(Container(a=False)),
+    )
+    assert "default: a:container-str" in get_helptext_with_checks(
+        Container[Container[bool] | Container[str]],
+        default=Container(Container(a="False")),
+    )
+    assert "default: a:container-literal-1-2" in get_helptext_with_checks(
+        Container[Container[Literal[1, 2]] | Container[str]],
+        default=Container(Container(a=1)),
+    )
+    assert "default: a:container-str" in get_helptext_with_checks(
+        Container[Container[Literal[1, 2]] | Container[str]],
+        default=Container(Container(a="1")),
+    )
+
+    assert tyro.cli(
+        Container[Container[Literal["1", "2"]] | Container[bool]],
+        args=[],
+        default=Container(Container(a="1")),
+        config=(tyro.conf.OmitSubcommandPrefixes,),
+    ) == Container(Container("1"))
+
+
+def test_generic_config_subcommand_matching_dict() -> None:
+    @dataclass(frozen=True)
+    class Container[T]:
+        a: T
+
+    assert "default: container-dict-str-str" in get_helptext_with_checks(
+        Container[dict[str, int]] | Container[dict[str, str]],  # type: ignore
+        default=Container({"a": "text"}),
+    )
+    assert "default: container-dict-str-int" in get_helptext_with_checks(
+        Container[dict[str, int]] | Container[dict[str, str]],  # type: ignore
+        default=Container({"a": 5}),
+    )
+
+
+def test_generic_config_subcommand_matching_tuple() -> None:
+    @dataclass(frozen=True)
+    class Container[T]:
+        a: T
+
+    assert "default: container-tuple-str-str" in get_helptext_with_checks(
+        Container[tuple[str, int]] | Container[tuple[str, str]],  # type: ignore
+        default=Container(("a", "text")),
+    )
+    assert "default: container-tuple-str-int" in get_helptext_with_checks(
+        Container[tuple[str, int]] | Container[tuple[str, str]],  # type: ignore
+        default=Container(("a", 5)),
+    )
+
+
+def test_generic_config_subcommand_matching_tuple_variable() -> None:
+    @dataclass(frozen=True)
+    class Container[T]:
+        a: T
+
+    assert "default: container-tuple-str-ellipsis" in get_helptext_with_checks(
+        Container[tuple[str, ...]] | Container[tuple[int, ...]],  # type: ignore
+        default=Container(("a", "text")),
+    )
+    assert "default: container-tuple-int-ellipsis" in get_helptext_with_checks(
+        Container[tuple[str, ...]] | Container[tuple[int, ...]],  # type: ignore
+        default=Container((1, 2, 3)),
+    )
diff --git a/tests/test_py311_generated/test_str_enum_min_py311_generated.py b/tests/test_py311_generated/test_str_enum_min_py311_generated.py
new file mode 100644
index 000000000..da138da33
--- /dev/null
+++ b/tests/test_py311_generated/test_str_enum_min_py311_generated.py
@@ -0,0 +1,87 @@
+import enum
+
+from pydantic import BaseModel, ConfigDict, Field
+
+import tyro
+
+
+class SomeEnum(enum.StrEnum):
+    A = enum.auto()
+    B = enum.auto()
+
+
+def test_str_enum():
+    def main(x: SomeEnum) -> SomeEnum:
+        return x
+
+    assert tyro.cli(main, args="--x A".split(" ")) == SomeEnum.A
+
+
+def test_str_enum_value_config():
+    def main(x: SomeEnum) -> SomeEnum:
+        return x
+
+    assert (
+        tyro.cli(
+            main, args="--x a".split(" "), config=(tyro.conf.EnumChoicesFromValues,)
+        )
+        == SomeEnum.A
+    )
+
+
+def test_str_enum_default():
+    def main(x: SomeEnum = SomeEnum.A) -> SomeEnum:
+        return x
+
+    assert tyro.cli(main, args=[]) == SomeEnum.A
+    assert tyro.cli(main, args="--x A".split(" ")) == SomeEnum.A
+
+
+def test_pydantic():
+    class Model(BaseModel):
+        x: SomeEnum = Field(default=SomeEnum.A)
+
+    assert tyro.cli(Model, args=[]).x == SomeEnum.A
+
+
+def test_pydantic_use_enum_values():
+    class Model(BaseModel):
+        model_config = ConfigDict(use_enum_values=True)
+        x: SomeEnum = Field(default=SomeEnum.A)
+
+    # Check default value of `x`.
+    assert SomeEnum.A == SomeEnum.A.value
+    x = tyro.cli(
+        Model,
+        args=[],
+        default=Model.model_validate({}),
+    ).x
+    assert x == SomeEnum.A == SomeEnum.A.value
+    assert isinstance(x, str)
+    assert not isinstance(x, SomeEnum)
+
+    # Check default value of `x` with `EnumChoicesFromValues`.
+    x = tyro.cli(
+        Model,
+        args=[],
+        default=Model.model_validate({}),
+        config=(tyro.conf.EnumChoicesFromValues,),
+    ).x
+    assert x == SomeEnum.A == SomeEnum.A.value
+    assert isinstance(x, str)
+    assert not isinstance(x, SomeEnum)
+
+    # Pass some values in.
+    x = tyro.cli(
+        Model,
+        args="--x A".split(" "),
+        default=Model.model_validate({}),
+    ).x
+    assert x == "a"
+    x = tyro.cli(
+        Model,
+        args="--x a".split(" "),
+        default=Model.model_validate({}),
+        config=(tyro.conf.EnumChoicesFromValues,),
+    ).x
+    assert x == SomeEnum.A
diff --git a/tests/test_str_enum_min_py311.py b/tests/test_str_enum_min_py311.py
new file mode 100644
index 000000000..da138da33
--- /dev/null
+++ b/tests/test_str_enum_min_py311.py
@@ -0,0 +1,87 @@
+import enum
+
+from pydantic import BaseModel, ConfigDict, Field
+
+import tyro
+
+
+class SomeEnum(enum.StrEnum):
+    A = enum.auto()
+    B = enum.auto()
+
+
+def test_str_enum():
+    def main(x: SomeEnum) -> SomeEnum:
+        return x
+
+    assert tyro.cli(main, args="--x A".split(" ")) == SomeEnum.A
+
+
+def test_str_enum_value_config():
+    def main(x: SomeEnum) -> SomeEnum:
+        return x
+
+    assert (
+        tyro.cli(
+            main, args="--x a".split(" "), config=(tyro.conf.EnumChoicesFromValues,)
+        )
+        == SomeEnum.A
+    )
+
+
+def test_str_enum_default():
+    def main(x: SomeEnum = SomeEnum.A) -> SomeEnum:
+        return x
+
+    assert tyro.cli(main, args=[]) == SomeEnum.A
+    assert tyro.cli(main, args="--x A".split(" ")) == SomeEnum.A
+
+
+def test_pydantic():
+    class Model(BaseModel):
+        x: SomeEnum = Field(default=SomeEnum.A)
+
+    assert tyro.cli(Model, args=[]).x == SomeEnum.A
+
+
+def test_pydantic_use_enum_values():
+    class Model(BaseModel):
+        model_config = ConfigDict(use_enum_values=True)
+        x: SomeEnum = Field(default=SomeEnum.A)
+
+    # Check default value of `x`.
+    assert SomeEnum.A == SomeEnum.A.value
+    x = tyro.cli(
+        Model,
+        args=[],
+        default=Model.model_validate({}),
+    ).x
+    assert x == SomeEnum.A == SomeEnum.A.value
+    assert isinstance(x, str)
+    assert not isinstance(x, SomeEnum)
+
+    # Check default value of `x` with `EnumChoicesFromValues`.
+    x = tyro.cli(
+        Model,
+        args=[],
+        default=Model.model_validate({}),
+        config=(tyro.conf.EnumChoicesFromValues,),
+    ).x
+    assert x == SomeEnum.A == SomeEnum.A.value
+    assert isinstance(x, str)
+    assert not isinstance(x, SomeEnum)
+
+    # Pass some values in.
+    x = tyro.cli(
+        Model,
+        args="--x A".split(" "),
+        default=Model.model_validate({}),
+    ).x
+    assert x == "a"
+    x = tyro.cli(
+        Model,
+        args="--x a".split(" "),
+        default=Model.model_validate({}),
+        config=(tyro.conf.EnumChoicesFromValues,),
+    ).x
+    assert x == SomeEnum.A

From cb79b2ad2c57d685e3afd03e0d22d23f1bedb4bd Mon Sep 17 00:00:00 2001
From: brentyi 
Date: Fri, 22 Nov 2024 22:21:11 -0800
Subject: [PATCH 485/491] Fix generic subcommand example doc

---
 docs/source/examples/generics.rst              | 18 ++++--------------
 examples/05_generics/03_generic_subcommands.py |  5 ++++-
 2 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/docs/source/examples/generics.rst b/docs/source/examples/generics.rst
index aabd9836e..4b194a152 100644
--- a/docs/source/examples/generics.rst
+++ b/docs/source/examples/generics.rst
@@ -160,7 +160,8 @@ generic types is also supported.
 Generic Subcommands
 -------------------
 
-
+Just like standard classes, generic classes within unions can be selected
+between using subcommands.
 
 
 .. code-block:: python
@@ -253,17 +254,6 @@ Generic Subcommands
 .. raw:: html
 
     
-    $ python ./03_generic_subcommands.py experiment-adam --path /tmp --lr 1e-3
-    ╭─ Unrecognized options ──────────────────────────────────────────╮
-    │ Unrecognized options: --lr                                      │
-    │ ─────────────────────────────────────────────────────────────── │
-    │ Perhaps you meant:                                              │
-    │     --opt.lr FLOAT                                              │
-    │         (default: 0.0003)                                       │
-    │             in 03_generic_subcommands.py experiment-adam --help │
-    │         (default: 0.0001)                                       │
-    │             in 03_generic_subcommands.py experiment-sgd --help  │
-    │ ─────────────────────────────────────────────────────────────── │
-    │ For full helptext, run 03_generic_subcommands.py --help         │
-    ╰─────────────────────────────────────────────────────────────────╯
+    $ python ./03_generic_subcommands.py experiment-adam --path /tmp --opt.lr 1e-3
+    Experiment(path=PosixPath('/tmp'), opt=Adam(lr=0.001, betas=(0.9, 0.999)))
     

\ No newline at end of file
diff --git a/examples/05_generics/03_generic_subcommands.py b/examples/05_generics/03_generic_subcommands.py
index 663bd3c92..2c69adf00 100644
--- a/examples/05_generics/03_generic_subcommands.py
+++ b/examples/05_generics/03_generic_subcommands.py
@@ -4,12 +4,15 @@
 # mypy. This will be fixed by `typing.TypeForm`: https://peps.python.org/pep-0747/
 """Generic Subcommands
 
+Just like standard classes, generic classes within unions can be selected
+between using subcommands.
+
 Usage:
 
     python ./03_generic_subcommands.py --help
     python ./03_generic_subcommands.py experiment-adam --help
     python ./03_generic_subcommands.py experiment-sgd --help
-    python ./03_generic_subcommands.py experiment-adam --path /tmp --lr 1e-3
+    python ./03_generic_subcommands.py experiment-adam --path /tmp --opt.lr 1e-3
 """
 
 import dataclasses

From c195f3a26c2a5f2aa58c28904ccedf756b4f155a Mon Sep 17 00:00:00 2001
From: Brent Yi 
Date: Fri, 22 Nov 2024 23:42:38 -0800
Subject: [PATCH 486/491] `0.9.2`

Signed-off-by: Brent Yi 
---
 src/tyro/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tyro/__init__.py b/src/tyro/__init__.py
index 7c80250e5..ab348c19a 100644
--- a/src/tyro/__init__.py
+++ b/src/tyro/__init__.py
@@ -1,6 +1,6 @@
 from typing import TYPE_CHECKING
 
-__version__ = "0.9.1"
+__version__ = "0.9.2"
 
 
 from . import conf as conf

From af0ae1eb2f5ae51ec0a86baccc85ccfd6b252e62 Mon Sep 17 00:00:00 2001
From: brentyi 
Date: Sat, 23 Nov 2024 13:57:21 -0800
Subject: [PATCH 487/491] Simplify examples

---
 docs/source/examples/basics.rst                           | 2 +-
 docs/source/examples/generics.rst                         | 8 ++++----
 docs/source/examples/overriding_configs.rst               | 2 +-
 docs/source/examples/subcommands.rst                      | 8 ++++----
 examples/01_basics/07_unions.py                           | 2 +-
 examples/03_subcommands/01_subcommands.py                 | 4 ++--
 examples/03_subcommands/02_subcommands_in_func.py         | 4 ++--
 .../04_overriding_configs/03_choosing_base_configs.py     | 2 +-
 examples/05_generics/02_generics.py                       | 6 +++---
 examples/05_generics/03_generic_subcommands.py            | 2 +-
 10 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/docs/source/examples/basics.rst b/docs/source/examples/basics.rst
index 55961c894..2dde04b35 100644
--- a/docs/source/examples/basics.rst
+++ b/docs/source/examples/basics.rst
@@ -463,7 +463,7 @@ multiple types.
         GREEN = enum.auto()
         BLUE = enum.auto()
 
-    @dataclasses.dataclass(frozen=True)
+    @dataclasses.dataclass
     class Args:
         # Unions can be used to specify multiple allowable types.
         union_over_types: int | str = 0
diff --git a/docs/source/examples/generics.rst b/docs/source/examples/generics.rst
index 4b194a152..22c0149ee 100644
--- a/docs/source/examples/generics.rst
+++ b/docs/source/examples/generics.rst
@@ -103,20 +103,20 @@ generic types is also supported.
     ScalarType = TypeVar("ScalarType", int, float)
     ShapeType = TypeVar("ShapeType")
 
-    @dataclasses.dataclass(frozen=True)
+    @dataclasses.dataclass
     class Point3(Generic[ScalarType]):
         x: ScalarType
         y: ScalarType
         z: ScalarType
         frame_id: str
 
-    @dataclasses.dataclass(frozen=True)
+    @dataclasses.dataclass
     class Triangle:
         a: Point3[float]
         b: Point3[float]
         c: Point3[float]
 
-    @dataclasses.dataclass(frozen=True)
+    @dataclasses.dataclass
     class Args(Generic[ShapeType]):
         shape: ShapeType
 
@@ -182,7 +182,7 @@ between using subcommands.
         lr: float = 3e-4
         betas: tuple[float, float] = (0.9, 0.999)
 
-    @dataclasses.dataclass(frozen=True)
+    @dataclasses.dataclass
     class Experiment[OptimizerT: (Adam, Sgd)]:
         path: Path
         opt: OptimizerT
diff --git a/docs/source/examples/overriding_configs.rst b/docs/source/examples/overriding_configs.rst
index 32e3c25b4..0b6212184 100644
--- a/docs/source/examples/overriding_configs.rst
+++ b/docs/source/examples/overriding_configs.rst
@@ -229,7 +229,7 @@ syntax.
 
     import tyro
 
-    @dataclass(frozen=True)
+    @dataclass
     class ExperimentConfig:
         # Dataset to run experiment on.
         dataset: Literal["mnist", "imagenet-50"]
diff --git a/docs/source/examples/subcommands.rst b/docs/source/examples/subcommands.rst
index 3d1d6aa57..217f95ee1 100644
--- a/docs/source/examples/subcommands.rst
+++ b/docs/source/examples/subcommands.rst
@@ -35,13 +35,13 @@ the union; arguments are then populated from the chosen type.
 
     import tyro
 
-    @dataclasses.dataclass(frozen=True)
+    @dataclasses.dataclass
     class Checkout:
         """Checkout a branch."""
 
         branch: str
 
-    @dataclasses.dataclass(frozen=True)
+    @dataclasses.dataclass
     class Commit:
         """Commit changes."""
 
@@ -149,13 +149,13 @@ struct types.
 
     import tyro
 
-    @dataclasses.dataclass(frozen=True)
+    @dataclasses.dataclass
     class Checkout:
         """Checkout a branch."""
 
         branch: str
 
-    @dataclasses.dataclass(frozen=True)
+    @dataclasses.dataclass
     class Commit:
         """Commit changes."""
 
diff --git a/examples/01_basics/07_unions.py b/examples/01_basics/07_unions.py
index 7f06ff1fe..f1b5aa5e9 100644
--- a/examples/01_basics/07_unions.py
+++ b/examples/01_basics/07_unions.py
@@ -26,7 +26,7 @@ class Color(enum.Enum):
     BLUE = enum.auto()
 
 
-@dataclasses.dataclass(frozen=True)
+@dataclasses.dataclass
 class Args:
     # Unions can be used to specify multiple allowable types.
     union_over_types: int | str = 0
diff --git a/examples/03_subcommands/01_subcommands.py b/examples/03_subcommands/01_subcommands.py
index 0f184d347..0a9a98d3b 100644
--- a/examples/03_subcommands/01_subcommands.py
+++ b/examples/03_subcommands/01_subcommands.py
@@ -34,14 +34,14 @@
 import tyro
 
 
-@dataclasses.dataclass(frozen=True)
+@dataclasses.dataclass
 class Checkout:
     """Checkout a branch."""
 
     branch: str
 
 
-@dataclasses.dataclass(frozen=True)
+@dataclasses.dataclass
 class Commit:
     """Commit changes."""
 
diff --git a/examples/03_subcommands/02_subcommands_in_func.py b/examples/03_subcommands/02_subcommands_in_func.py
index 727d336eb..1c179d387 100644
--- a/examples/03_subcommands/02_subcommands_in_func.py
+++ b/examples/03_subcommands/02_subcommands_in_func.py
@@ -34,14 +34,14 @@
 import tyro
 
 
-@dataclasses.dataclass(frozen=True)
+@dataclasses.dataclass
 class Checkout:
     """Checkout a branch."""
 
     branch: str
 
 
-@dataclasses.dataclass(frozen=True)
+@dataclasses.dataclass
 class Commit:
     """Commit changes."""
 
diff --git a/examples/04_overriding_configs/03_choosing_base_configs.py b/examples/04_overriding_configs/03_choosing_base_configs.py
index fd532b677..2b95d50f3 100644
--- a/examples/04_overriding_configs/03_choosing_base_configs.py
+++ b/examples/04_overriding_configs/03_choosing_base_configs.py
@@ -37,7 +37,7 @@
 import tyro
 
 
-@dataclass(frozen=True)
+@dataclass
 class ExperimentConfig:
     # Dataset to run experiment on.
     dataset: Literal["mnist", "imagenet-50"]
diff --git a/examples/05_generics/02_generics.py b/examples/05_generics/02_generics.py
index a881d1158..ceaf28ec3 100644
--- a/examples/05_generics/02_generics.py
+++ b/examples/05_generics/02_generics.py
@@ -17,7 +17,7 @@
 ShapeType = TypeVar("ShapeType")
 
 
-@dataclasses.dataclass(frozen=True)
+@dataclasses.dataclass
 class Point3(Generic[ScalarType]):
     x: ScalarType
     y: ScalarType
@@ -25,14 +25,14 @@ class Point3(Generic[ScalarType]):
     frame_id: str
 
 
-@dataclasses.dataclass(frozen=True)
+@dataclasses.dataclass
 class Triangle:
     a: Point3[float]
     b: Point3[float]
     c: Point3[float]
 
 
-@dataclasses.dataclass(frozen=True)
+@dataclasses.dataclass
 class Args(Generic[ShapeType]):
     shape: ShapeType
 
diff --git a/examples/05_generics/03_generic_subcommands.py b/examples/05_generics/03_generic_subcommands.py
index 2c69adf00..007da7bac 100644
--- a/examples/05_generics/03_generic_subcommands.py
+++ b/examples/05_generics/03_generic_subcommands.py
@@ -32,7 +32,7 @@ class Adam:
     betas: tuple[float, float] = (0.9, 0.999)
 
 
-@dataclasses.dataclass(frozen=True)
+@dataclasses.dataclass
 class Experiment[OptimizerT: (Adam, Sgd)]:
     path: Path
     opt: OptimizerT

From edc24154ffac5bbaff55641687f4d7c54506b8e9 Mon Sep 17 00:00:00 2001
From: brentyi 
Date: Wed, 4 Dec 2024 15:41:46 +0000
Subject: [PATCH 488/491] Adjust behavior for string types in unions

---
 src/tyro/constructors/_primitive_spec.py        | 17 +++++++++++------
 tests/test_dcargs.py                            | 17 +++++++++++++++++
 tests/test_helptext.py                          | 10 +++++-----
 .../test_dcargs_generated.py                    | 17 +++++++++++++++++
 .../test_helptext_generated.py                  | 10 +++++-----
 5 files changed, 55 insertions(+), 16 deletions(-)

diff --git a/src/tyro/constructors/_primitive_spec.py b/src/tyro/constructors/_primitive_spec.py
index d3418c665..a515b569e 100644
--- a/src/tyro/constructors/_primitive_spec.py
+++ b/src/tyro/constructors/_primitive_spec.py
@@ -586,12 +586,16 @@ def union_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None:
         if type_info.type_origin not in (Union, _resolver.UnionType):
             return None
         options = list(get_args(type_info.type))
-        if type(None) in options:
-            # Move `None` types to the beginning.
-            # If we have `Optional[str]`, we want this to be parsed as
-            # `Union[NoneType, str]`.
-            options.remove(type(None))
-            options.insert(0, type(None))
+
+        # Heuristic: handle `str` types last, since any input can be captured as a string.
+        #
+        # Some examples where this behavior matters:
+        # - For `Optional[str]`, None will create a None object instead of a string.
+        # - For `str | bool`, True/False will create booleans instead of strings.
+        # - For `str | int`, integers will be created when possible instead of strings.
+        if str in options:
+            options.remove(str)
+            options.append(str)
 
         # General unions, eg Union[int, bool]. We'll try to convert these from left to
         # right.
@@ -600,6 +604,7 @@ def union_rule(type_info: PrimitiveTypeInfo) -> PrimitiveConstructorSpec | None:
         nargs: int | Literal["*"] = 1
         first = True
         registry = ConstructorRegistry._get_active_registry()
+
         for t in options:
             option_spec = registry.get_primitive_spec(
                 PrimitiveTypeInfo.make(
diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py
index b82543d1c..68dc81def 100644
--- a/tests/test_dcargs.py
+++ b/tests/test_dcargs.py
@@ -238,6 +238,23 @@ def main(x: Union[int, str]) -> Union[int, str]:
     assert tyro.cli(main, args=["--x", "five"]) == "five"
 
 
+def test_union_bool_str() -> None:
+    def main(x: Union[bool, str]) -> Any:
+        return x
+
+    assert tyro.cli(main, args=["--x", "5"]) == "5"
+    assert tyro.cli(main, args=["--x", "True"]) is True
+    assert tyro.cli(main, args=["--x", "Tru"]) == "Tru"
+
+    # Order should not matter.
+    def flipped(x: Union[str, bool]) -> Any:
+        return x
+
+    assert tyro.cli(flipped, args=["--x", "5"]) == "5"
+    assert tyro.cli(flipped, args=["--x", "True"]) is True
+    assert tyro.cli(flipped, args=["--x", "Tru"]) == "Tru"
+
+
 def test_union_with_list() -> None:
     def main(x: Union[int, str, List[bool]]) -> Any:
         return x
diff --git a/tests/test_helptext.py b/tests/test_helptext.py
index 768da2f6f..f935d92bf 100644
--- a/tests/test_helptext.py
+++ b/tests/test_helptext.py
@@ -275,7 +275,7 @@ class Config:
         """An optional variable."""
 
     helptext = get_helptext_with_checks(Config)
-    assert "--x {None}|INT" in helptext
+    assert "--x INT|{None}" in helptext
     assert "An optional variable. (default: None)" in helptext
 
 
@@ -411,7 +411,7 @@ class OptionalLiteralHelptext:
         """A number."""
 
     helptext = get_helptext_with_checks(OptionalLiteralHelptext)
-    assert "--x {None,1,2,3}" in helptext
+    assert "--x {1,2,3,None}" in helptext
     assert "A number. (default: None)" in helptext
 
 
@@ -555,9 +555,9 @@ class OptionalHelptext:
     helptext = get_helptext_with_checks(OptionalHelptext)
     assert cast(str, cast(str, OptionalHelptext.__doc__)[:20]) in helptext
     assert "2% milk" in helptext
-    assert "--x {None}|INT" in helptext
-    assert "--y [{None}|INT [{None}|INT ...]]" in helptext
-    assert "[--z {None}|INT]" in helptext
+    assert "--x INT|{None}" in helptext
+    assert "--y [INT|{None} [INT|{None} ...]]" in helptext
+    assert "[--z INT|{None}]" in helptext
 
 
 def test_metavar_0() -> None:
diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py
index 909b3a038..49a0afbec 100644
--- a/tests/test_py311_generated/test_dcargs_generated.py
+++ b/tests/test_py311_generated/test_dcargs_generated.py
@@ -240,6 +240,23 @@ def main(x: int | str) -> int | str:
     assert tyro.cli(main, args=["--x", "five"]) == "five"
 
 
+def test_union_bool_str() -> None:
+    def main(x: bool | str) -> Any:
+        return x
+
+    assert tyro.cli(main, args=["--x", "5"]) == "5"
+    assert tyro.cli(main, args=["--x", "True"]) is True
+    assert tyro.cli(main, args=["--x", "Tru"]) == "Tru"
+
+    # Order should not matter.
+    def flipped(x: str | bool) -> Any:
+        return x
+
+    assert tyro.cli(flipped, args=["--x", "5"]) == "5"
+    assert tyro.cli(flipped, args=["--x", "True"]) is True
+    assert tyro.cli(flipped, args=["--x", "Tru"]) == "Tru"
+
+
 def test_union_with_list() -> None:
     def main(x: int | str | List[bool]) -> Any:
         return x
diff --git a/tests/test_py311_generated/test_helptext_generated.py b/tests/test_py311_generated/test_helptext_generated.py
index 92eac7b57..e50b41f58 100644
--- a/tests/test_py311_generated/test_helptext_generated.py
+++ b/tests/test_py311_generated/test_helptext_generated.py
@@ -287,7 +287,7 @@ class Config:
         """An optional variable."""
 
     helptext = get_helptext_with_checks(Config)
-    assert "--x {None}|INT" in helptext
+    assert "--x INT|{None}" in helptext
     assert "An optional variable. (default: None)" in helptext
 
 
@@ -423,7 +423,7 @@ class OptionalLiteralHelptext:
         """A number."""
 
     helptext = get_helptext_with_checks(OptionalLiteralHelptext)
-    assert "--x {None,1,2,3}" in helptext
+    assert "--x {1,2,3,None}" in helptext
     assert "A number. (default: None)" in helptext
 
 
@@ -567,9 +567,9 @@ class OptionalHelptext:
     helptext = get_helptext_with_checks(OptionalHelptext)
     assert cast(str, cast(str, OptionalHelptext.__doc__)[:20]) in helptext
     assert "2% milk" in helptext
-    assert "--x {None}|INT" in helptext
-    assert "--y [{None}|INT [{None}|INT ...]]" in helptext
-    assert "[--z {None}|INT]" in helptext
+    assert "--x INT|{None}" in helptext
+    assert "--y [INT|{None} [INT|{None} ...]]" in helptext
+    assert "[--z INT|{None}]" in helptext
 
 
 def test_metavar_0() -> None:

From 201335e4b2dcb3749e155ffd96bbe2a12087270a Mon Sep 17 00:00:00 2001
From: brentyi 
Date: Wed, 4 Dec 2024 15:52:47 +0000
Subject: [PATCH 489/491] Test adjustments

---
 tests/test_dcargs.py                                            | 2 +-
 tests/test_new_style_annotations_min_py310.py                   | 2 +-
 tests/test_new_style_annotations_unions.py                      | 2 +-
 tests/test_py311_generated/test_dcargs_generated.py             | 2 +-
 .../test_new_style_annotations_min_py310_generated.py           | 2 +-
 .../test_new_style_annotations_unions_generated.py              | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/test_dcargs.py b/tests/test_dcargs.py
index 68dc81def..37185c85a 100644
--- a/tests/test_dcargs.py
+++ b/tests/test_dcargs.py
@@ -262,7 +262,7 @@ def main(x: Union[int, str, List[bool]]) -> Any:
     assert tyro.cli(main, args=["--x", "5"]) == 5
     assert tyro.cli(main, args=["--x", "6"]) == 6
     assert tyro.cli(main, args=["--x", "five"]) == "five"
-    assert tyro.cli(main, args=["--x", "True"]) == "True"
+    assert tyro.cli(main, args=["--x", "True"]) == [True]
     assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False]
 
 
diff --git a/tests/test_new_style_annotations_min_py310.py b/tests/test_new_style_annotations_min_py310.py
index 3e2e90a55..7caa0558c 100644
--- a/tests/test_new_style_annotations_min_py310.py
+++ b/tests/test_new_style_annotations_min_py310.py
@@ -31,7 +31,7 @@ def main(x: int | str | list[bool]) -> Any:
     assert tyro.cli(main, args=["--x", "5"]) == 5
     assert tyro.cli(main, args=["--x", "6"]) == 6
     assert tyro.cli(main, args=["--x", "five"]) == "five"
-    assert tyro.cli(main, args=["--x", "True"]) == "True"
+    assert tyro.cli(main, args=["--x", "True"]) == [True]
     assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False]
 
 
diff --git a/tests/test_new_style_annotations_unions.py b/tests/test_new_style_annotations_unions.py
index 7927af133..a40f8c99f 100644
--- a/tests/test_new_style_annotations_unions.py
+++ b/tests/test_new_style_annotations_unions.py
@@ -24,7 +24,7 @@ def main(x: int | str | list[bool]) -> Any:
     assert tyro.cli(main, args=["--x", "5"]) == 5
     assert tyro.cli(main, args=["--x", "6"]) == 6
     assert tyro.cli(main, args=["--x", "five"]) == "five"
-    assert tyro.cli(main, args=["--x", "True"]) == "True"
+    assert tyro.cli(main, args=["--x", "True"]) == [True]
     assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False]
 
 
diff --git a/tests/test_py311_generated/test_dcargs_generated.py b/tests/test_py311_generated/test_dcargs_generated.py
index 49a0afbec..3cbb7d5c2 100644
--- a/tests/test_py311_generated/test_dcargs_generated.py
+++ b/tests/test_py311_generated/test_dcargs_generated.py
@@ -264,7 +264,7 @@ def main(x: int | str | List[bool]) -> Any:
     assert tyro.cli(main, args=["--x", "5"]) == 5
     assert tyro.cli(main, args=["--x", "6"]) == 6
     assert tyro.cli(main, args=["--x", "five"]) == "five"
-    assert tyro.cli(main, args=["--x", "True"]) == "True"
+    assert tyro.cli(main, args=["--x", "True"]) == [True]
     assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False]
 
 
diff --git a/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py
index 3e2e90a55..7caa0558c 100644
--- a/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py
+++ b/tests/test_py311_generated/test_new_style_annotations_min_py310_generated.py
@@ -31,7 +31,7 @@ def main(x: int | str | list[bool]) -> Any:
     assert tyro.cli(main, args=["--x", "5"]) == 5
     assert tyro.cli(main, args=["--x", "6"]) == 6
     assert tyro.cli(main, args=["--x", "five"]) == "five"
-    assert tyro.cli(main, args=["--x", "True"]) == "True"
+    assert tyro.cli(main, args=["--x", "True"]) == [True]
     assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False]
 
 
diff --git a/tests/test_py311_generated/test_new_style_annotations_unions_generated.py b/tests/test_py311_generated/test_new_style_annotations_unions_generated.py
index 19cf034fe..f84336db2 100644
--- a/tests/test_py311_generated/test_new_style_annotations_unions_generated.py
+++ b/tests/test_py311_generated/test_new_style_annotations_unions_generated.py
@@ -23,7 +23,7 @@ def main(x: int | str | list[bool]) -> Any:
     assert tyro.cli(main, args=["--x", "5"]) == 5
     assert tyro.cli(main, args=["--x", "6"]) == 6
     assert tyro.cli(main, args=["--x", "five"]) == "five"
-    assert tyro.cli(main, args=["--x", "True"]) == "True"
+    assert tyro.cli(main, args=["--x", "True"]) == [True]
     assert tyro.cli(main, args=["--x", "True", "False"]) == [True, False]
 
 

From 275431febe88164d573d0c9ad422fc0660028159 Mon Sep 17 00:00:00 2001
From: brentyi 
Date: Wed, 4 Dec 2024 16:01:18 +0000
Subject: [PATCH 490/491] Remove forward reference

---
 tests/test_py311_generated/test_pydantic_generated.py | 2 +-
 tests/test_pydantic.py                                | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/test_py311_generated/test_pydantic_generated.py b/tests/test_py311_generated/test_pydantic_generated.py
index 02320270a..266c40496 100644
--- a/tests/test_py311_generated/test_pydantic_generated.py
+++ b/tests/test_py311_generated/test_pydantic_generated.py
@@ -177,7 +177,7 @@ class Inside(BaseModel):
         x: int = 1
 
     class Outside(BaseModel):
-        i: "Inside" = Inside(x=2)
+        i: Inside = Inside(x=2)
 
     assert tyro.cli(Outside, args=[]).i.x == 2, (
         "Expected x value from the default instance",
diff --git a/tests/test_pydantic.py b/tests/test_pydantic.py
index a64ca28cf..1015c021e 100644
--- a/tests/test_pydantic.py
+++ b/tests/test_pydantic.py
@@ -7,12 +7,12 @@
 from typing import cast
 
 import pytest
-from helptext_utils import get_helptext_with_checks
+import tyro
+import tyro._strings
 from pydantic import BaseModel, Field, v1
 from typing_extensions import Annotated
 
-import tyro
-import tyro._strings
+from helptext_utils import get_helptext_with_checks
 
 
 def test_pydantic() -> None:
@@ -178,7 +178,7 @@ class Inside(BaseModel):
         x: int = 1
 
     class Outside(BaseModel):
-        i: "Inside" = Inside(x=2)
+        i: Inside = Inside(x=2)
 
     assert tyro.cli(Outside, args=[]).i.x == 2, (
         "Expected x value from the default instance",

From 37dc485f850d01c60eaf722cc20a823c75cbfa6f Mon Sep 17 00:00:00 2001
From: brentyi 
Date: Wed, 4 Dec 2024 18:34:52 +0000
Subject: [PATCH 491/491] ruff

---
 tests/test_pydantic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_pydantic.py b/tests/test_pydantic.py
index 1015c021e..183e1319d 100644
--- a/tests/test_pydantic.py
+++ b/tests/test_pydantic.py
@@ -7,12 +7,12 @@
 from typing import cast
 
 import pytest
-import tyro
-import tyro._strings
+from helptext_utils import get_helptext_with_checks
 from pydantic import BaseModel, Field, v1
 from typing_extensions import Annotated
 
-from helptext_utils import get_helptext_with_checks
+import tyro
+import tyro._strings
 
 
 def test_pydantic() -> None: