diff --git a/README.md b/README.md index 98e51c2d..dcade88c 100644 --- a/README.md +++ b/README.md @@ -189,27 +189,26 @@ problemtools' configuration: 1. `languages.yaml`. Use it to override problemtools' default programming language configuration. For instance, while the - problemtools default is to use the CPython `/usr/bin/python3` - interpreter for Python 3, many contests, as well as the Kattis - online judge, use Pypy as the interpreter for Python 3. To change - this on your machine, you can simply place a file - `/etc/kattis/problemtools/languages.yaml` (or - `~/.config/problemtools/languages.yaml` if you only want to make the - change for your user) containing the following: + problemtools default is to use the PyPy `/usr/bin/pypy3` + interpreter for Python 3. If you prefer CPython, you can + use the following `languages.yaml`: ```yaml python3: - name: 'Python 3 w/Pypy' - run: '/usr/bin/pypy3 "{mainfile}"' + name: 'Python 3' + compile: '/usr/bin/python3 -m py_compile {files}' + run: '/usr/bin/python3 "{mainfile}"' ``` Here, overriding the name of the language is not strictly - necessary, but it is often helpful to clearly indicate that Pypy is - being used. + necessary, but the default configuration mentions PyPy. For more details on the format of the language specifications and what the default settings are, see the [default version of languages.yaml](problemtools/config/languages.yaml) + The file `languages.yaml` may also be placed as a sibling to the problem + package being checked (e.g., for contest-specific language configuration). + 2. `problem.yaml`. For most users, this should not be edited. If you are not sure whether you should use it, then you probably shouldn't. This file can be used to specify the system defaults for those diff --git a/problemtools/languages.py b/problemtools/languages.py index 2c4d54e9..60dfed08 100644 --- a/problemtools/languages.py +++ b/problemtools/languages.py @@ -6,10 +6,17 @@ import fnmatch import re import string +from collections.abc import Sequence from pathlib import Path +from typing import TypeVar from . import config +# Used to preserve the str-vs-Path type of file names through the +# file-list-filtering methods below. We eventually want to move +# to Path. +StrOrPath = TypeVar('StrOrPath', str, Path) + class LanguageConfigError(Exception): """Exception class for errors in language configuration.""" @@ -24,7 +31,15 @@ class Language: __VARIABLES = ['path', 'files', 'binary', 'mainfile', 'mainclass', 'Mainclass', 'memlim'] __MAINFILE_RE = re.compile(r'^main\.', re.IGNORECASE) - def __init__(self, lang_id, lang_spec): + name: str + priority: int + files: list[str] + run: str + shebang: re.Pattern[str] | None = None + shebang_files: list[str] | None = None + compile: str | None = None + + def __init__(self, lang_id: str, lang_spec: dict): """Construct language object Args: @@ -35,16 +50,9 @@ def __init__(self, lang_id, lang_spec): if not re.match('[a-z][a-z0-9]*', lang_id): raise LanguageConfigError('Invalid language ID "%s"' % lang_id) self.lang_id = lang_id - self.name = None - self.priority = None - self.files = None - self.shebang = None - self.shebang_files = None - self.compile = None - self.run = None self.update(lang_spec) - def get_source_files(self, file_list): + def get_source_files(self, file_list: Sequence[StrOrPath]) -> list[StrOrPath]: """Given a list of files, determine which ones would be considered source files for the language. @@ -54,9 +62,9 @@ def get_source_files(self, file_list): Args: file_list (list of str): list of file names """ - return [file_name for file_name in file_list if any(fnmatch.fnmatch(file_name, glob) for glob in self.files)] # type: ignore[union-attr] + return [file_name for file_name in file_list if any(fnmatch.fnmatch(file_name, glob) for glob in self.files)] - def get_source_files_for_detection(self, file_list): + def get_source_files_for_detection(self, file_list: Sequence[StrOrPath]) -> list[StrOrPath]: """Given a list of files, determine which ones count as positive evidence that a program is written in this language, for use when auto-detecting a program's language (see Languages.detect_language). @@ -72,7 +80,7 @@ def get_source_files_for_detection(self, file_list): """ return [file_name for file_name in self.get_source_files(file_list) if self.__passes_shebang_gate(file_name)] - def mainfile_candidates(self, files: list[str | Path]) -> list[str | Path]: + def mainfile_candidates(self, files: Sequence[StrOrPath]) -> list[StrOrPath]: """Given a list of files, determine which ones would be considered plausible mainfiles for the language, i.e. an entrypoint override. @@ -84,7 +92,10 @@ def mainfile_candidates(self, files: list[str | Path]) -> list[str | Path]: """ return [f for f in files if Language.__MAINFILE_RE.match(Path(f).name)] - def update(self, values): + # Update is no longer really needed - we only call it from the constructor. + # But cleaning that up doesn't simplify the code much, so we keep it around + # for now. + def update(self, values: dict) -> None: """Update a language specification with new values. Args: @@ -118,19 +129,20 @@ def update(self, values): self.__check() - def __check(self): + def __check(self) -> None: """Check that the language specification is valid (all mandatory fields provided, all metavariables used in compile/run commands valid, and uniquely defined entry point. """ - # Check that all mandatory fields are provided - if self.name is None: + # Check that all mandatory fields are provided. These attributes + # may not exist yet -- hence getattr() rather than a direct read. + if getattr(self, 'name', None) is None: raise LanguageConfigError(f'Language {self.lang_id} has no name') - if self.priority is None: + if getattr(self, 'priority', None) is None: raise LanguageConfigError(f'Language {self.lang_id} has no priority') - if self.files is None: + if getattr(self, 'files', None) is None: raise LanguageConfigError(f'Language {self.lang_id} has no files glob') - if self.run is None: + if getattr(self, 'run', None) is None: raise LanguageConfigError(f'Language {self.lang_id} has no run command') if (self.shebang is None) != (self.shebang_files is None): raise LanguageConfigError(f'Language {self.lang_id} must specify both "shebang" and "shebang_files", or neither') @@ -150,27 +162,28 @@ def __check(self): raise LanguageConfigError('More than one entry point type variable used for language %s' % self.lang_id) @staticmethod - def __variables_in_command(cmd): + def __variables_in_command(cmd: str) -> set[str]: """List all meta-variables appearing in a string.""" formatter = string.Formatter() return set(field for _, field, _, _ in formatter.parse(cmd) if field is not None) - def __passes_shebang_gate(self, filename): + def __passes_shebang_gate(self, filename: str | Path) -> bool: """Check if a file matched by shebang_files also matches shebang. Files not matched by shebang_files are unaffected by the gate.""" if self.shebang_files is None: return True + assert self.shebang is not None, '__check() guarantees shebang and shebang_files are set together' if not any(fnmatch.fnmatch(filename, glob) for glob in self.shebang_files): return True with open(filename, 'r') as f_in: shebang_line = f_in.readline() - return self.shebang.search(shebang_line) is not None # type: ignore[union-attr] + return self.shebang.search(shebang_line) is not None class Languages: """A set of languages.""" - def __init__(self, data=None): + def __init__(self, data: dict | None = None): """Create a set of languages from a dict. Args: @@ -178,11 +191,11 @@ def __init__(self, data=None): If None, resulting set of languages is empty. See documentation of update() method below for details. """ - self.languages = {} + self.languages: dict[str, Language] = {} if data is not None: self.update(data) - def detect_language(self, file_list): + def detect_language(self, file_list: Sequence[StrOrPath]) -> Language | None: """Auto-detect language for a set of files. Args: @@ -193,7 +206,7 @@ def detect_language(self, file_list): list of files did not match any language in the set. """ result = None - src: list[str] = [] + src: list[StrOrPath] = [] prio = 1e99 for lang in self.languages.values(): lang_src = lang.get_source_files_for_detection(file_list) @@ -203,12 +216,12 @@ def detect_language(self, file_list): prio = lang.priority return result - def get(self, lang_id): + def get(self, lang_id: str) -> Language | None: if not isinstance(lang_id, str): raise LanguageConfigError('Config file error: language IDs must be strings, but %s is %s.' % (lang_id, type(lang_id))) return self.languages.get(lang_id, None) - def update(self, data): + def update(self, data: dict) -> None: """Update the set with language configuration data from a dict. Args: @@ -239,7 +252,7 @@ def update(self, data): else: self.languages[lang_id].update(lang_spec) - priorities: dict[int, Language] = {} + priorities: dict[int, str] = {} for lang_id, lang in self.languages.items(): if lang.priority in priorities: raise LanguageConfigError( diff --git a/problemtools/run/source.py b/problemtools/run/source.py index e71cc475..9714395c 100644 --- a/problemtools/run/source.py +++ b/problemtools/run/source.py @@ -125,7 +125,6 @@ def get_runcmd(self, cwd=None, memlim=1024): subs['path'] = os.path.relpath(subs['path'], cwd) subs['binary'] = os.path.relpath(subs['binary'], cwd) subs['mainfile'] = os.path.relpath(subs['mainfile'], cwd) - assert self.language.run is not None, 'Language.__check() guarantees run is always set' return shlex.split(self.language.run.format(**subs)) def should_skip_memory_rlimit(self) -> bool: