Add/fix type annotations in languages.py (and fix outdated README example for language config)

README.md

@@ -189,27 +189,26 @@ problemtools' configuration:
 
 1. `languages.yaml`.  Use it to override problemtools' default
    programming language configuration.  For instance, while the
-   problemtools default is to use the CPython `/usr/bin/python3`
-   interpreter for Python 3, many contests, as well as the Kattis
-   online judge, use Pypy as the interpreter for Python 3.  To change
-   this on your machine, you can simply place a file
-   `/etc/kattis/problemtools/languages.yaml` (or
-   `~/.config/problemtools/languages.yaml` if you only want to make the
-   change for your user) containing the following:
+   problemtools default is to use the PyPy `/usr/bin/pypy3`
+   interpreter for Python 3. If you prefer  CPython, you can
+   use the following `languages.yaml`:
 
    ```yaml
    python3:
-      name: 'Python 3 w/Pypy'
-      run: '/usr/bin/pypy3 "{mainfile}"'
+      name: 'Python 3'
+      compile: '/usr/bin/python3 -m py_compile {files}'
+      run: '/usr/bin/python3 "{mainfile}"'
    ```
    Here, overriding the name of the language is not strictly
-   necessary, but it is often helpful to clearly indicate that Pypy is
-   being used.
+   necessary, but the default configuration mentions PyPy.
 
    For more details on the format of the language specifications and
    what the default settings are, see the [default version of
    languages.yaml](problemtools/config/languages.yaml)
 
+   The file `languages.yaml` may also be placed as a sibling to the problem
+   package being checked (e.g., for contest-specific language configuration).
+
 2. `problem.yaml`.  For most users, this should not be edited.  If you
    are not sure whether you should use it, then you probably shouldn't.
    This file can be used to specify the system defaults for those

problemtools/languages.py

@@ -6,10 +6,17 @@
 import fnmatch
 import re
 import string
+from collections.abc import Sequence
 from pathlib import Path
+from typing import TypeVar
 
 from . import config
 
+# Used to preserve the str-vs-Path type of file names through the
+# file-list-filtering methods below. We eventually want to move
+# to Path.
+StrOrPath = TypeVar('StrOrPath', str, Path)
+
 
 class LanguageConfigError(Exception):
     """Exception class for errors in language configuration."""
@@ -24,7 +31,15 @@ class Language:
     __VARIABLES = ['path', 'files', 'binary', 'mainfile', 'mainclass', 'Mainclass', 'memlim']
     __MAINFILE_RE = re.compile(r'^main\.', re.IGNORECASE)
 
-    def __init__(self, lang_id, lang_spec):
+    name: str
+    priority: int
+    files: list[str]
+    run: str
+    shebang: re.Pattern[str] | None = None
+    shebang_files: list[str] | None = None
+    compile: str | None = None
+
+    def __init__(self, lang_id: str, lang_spec: dict):
         """Construct language object
 
         Args:
@@ -35,16 +50,9 @@ def __init__(self, lang_id, lang_spec):
         if not re.match('[a-z][a-z0-9]*', lang_id):
             raise LanguageConfigError('Invalid language ID "%s"' % lang_id)
         self.lang_id = lang_id
-        self.name = None
-        self.priority = None
-        self.files = None
-        self.shebang = None
-        self.shebang_files = None
-        self.compile = None
-        self.run = None
         self.update(lang_spec)
 
-    def get_source_files(self, file_list):
+    def get_source_files(self, file_list: Sequence[StrOrPath]) -> list[StrOrPath]:
         """Given a list of files, determine which ones would be considered
         source files for the language.
 
@@ -54,9 +62,9 @@ def get_source_files(self, file_list):
         Args:
             file_list (list of str): list of file names
         """
-        return [file_name for file_name in file_list if any(fnmatch.fnmatch(file_name, glob) for glob in self.files)]  # type: ignore[union-attr]
+        return [file_name for file_name in file_list if any(fnmatch.fnmatch(file_name, glob) for glob in self.files)]
 
-    def get_source_files_for_detection(self, file_list):
+    def get_source_files_for_detection(self, file_list: Sequence[StrOrPath]) -> list[StrOrPath]:
         """Given a list of files, determine which ones count as positive
         evidence that a program is written in this language, for use when
         auto-detecting a program's language (see Languages.detect_language).
@@ -72,7 +80,7 @@ def get_source_files_for_detection(self, file_list):
         """
         return [file_name for file_name in self.get_source_files(file_list) if self.__passes_shebang_gate(file_name)]
 
-    def mainfile_candidates(self, files: list[str | Path]) -> list[str | Path]:
+    def mainfile_candidates(self, files: Sequence[StrOrPath]) -> list[StrOrPath]:
         """Given a list of files, determine which ones would be considered
         plausible mainfiles for the language, i.e. an entrypoint override.
 
@@ -84,7 +92,10 @@ def mainfile_candidates(self, files: list[str | Path]) -> list[str | Path]:
         """
         return [f for f in files if Language.__MAINFILE_RE.match(Path(f).name)]
 
-    def update(self, values):
+    # Update is no longer really needed - we only call it from the constructor.
+    # But cleaning that up doesn't simplify the code much, so we keep it around
+    # for now.
+    def update(self, values: dict) -> None:
         """Update a language specification with new values.
 
         Args:
@@ -118,19 +129,20 @@ def update(self, values):
 
         self.__check()
 
-    def __check(self):
+    def __check(self) -> None:
         """Check that the language specification is valid (all mandatory
         fields provided, all metavariables used in compile/run
         commands valid, and uniquely defined entry point.
         """
-        # Check that all mandatory fields are provided
-        if self.name is None:
+        # Check that all mandatory fields are provided. These attributes
+        # may not exist yet -- hence getattr() rather than a direct read.
+        if getattr(self, 'name', None) is None:
             raise LanguageConfigError(f'Language {self.lang_id} has no name')
-        if self.priority is None:
+        if getattr(self, 'priority', None) is None:
             raise LanguageConfigError(f'Language {self.lang_id} has no priority')
-        if self.files is None:
+        if getattr(self, 'files', None) is None:
             raise LanguageConfigError(f'Language {self.lang_id} has no files glob')
-        if self.run is None:
+        if getattr(self, 'run', None) is None:
             raise LanguageConfigError(f'Language {self.lang_id} has no run command')
         if (self.shebang is None) != (self.shebang_files is None):
             raise LanguageConfigError(f'Language {self.lang_id} must specify both "shebang" and "shebang_files", or neither')
@@ -150,39 +162,40 @@ def __check(self):
             raise LanguageConfigError('More than one entry point type variable used for language %s' % self.lang_id)
 
     @staticmethod
-    def __variables_in_command(cmd):
+    def __variables_in_command(cmd: str) -> set[str]:
         """List all meta-variables appearing in a string."""
         formatter = string.Formatter()
         return set(field for _, field, _, _ in formatter.parse(cmd) if field is not None)
 
-    def __passes_shebang_gate(self, filename):
+    def __passes_shebang_gate(self, filename: str | Path) -> bool:
         """Check if a file matched by shebang_files also matches shebang.
         Files not matched by shebang_files are unaffected by the gate."""
         if self.shebang_files is None:
             return True
+        assert self.shebang is not None, '__check() guarantees shebang and shebang_files are set together'
         if not any(fnmatch.fnmatch(filename, glob) for glob in self.shebang_files):
             return True
         with open(filename, 'r') as f_in:
             shebang_line = f_in.readline()
-        return self.shebang.search(shebang_line) is not None  # type: ignore[union-attr]
+        return self.shebang.search(shebang_line) is not None
 
 
 class Languages:
     """A set of languages."""
 
-    def __init__(self, data=None):
+    def __init__(self, data: dict | None = None):
         """Create a set of languages from a dict.
 
         Args:
             data (dict): dictonary containing configuration.
                 If None, resulting set of languages is empty.
                 See documentation of update() method below for details.
         """
-        self.languages = {}
+        self.languages: dict[str, Language] = {}
         if data is not None:
             self.update(data)
 
-    def detect_language(self, file_list):
+    def detect_language(self, file_list: Sequence[StrOrPath]) -> Language | None:
         """Auto-detect language for a set of files.
 
         Args:
@@ -193,7 +206,7 @@ def detect_language(self, file_list):
             list of files did not match any language in the set.
         """
         result = None
-        src: list[str] = []
+        src: list[StrOrPath] = []
         prio = 1e99
         for lang in self.languages.values():
             lang_src = lang.get_source_files_for_detection(file_list)
@@ -203,12 +216,12 @@ def detect_language(self, file_list):
                 prio = lang.priority
         return result
 
-    def get(self, lang_id):
+    def get(self, lang_id: str) -> Language | None:
         if not isinstance(lang_id, str):
             raise LanguageConfigError('Config file error: language IDs must be strings, but %s is %s.' % (lang_id, type(lang_id)))
         return self.languages.get(lang_id, None)
 
-    def update(self, data):
+    def update(self, data: dict) -> None:
         """Update the set with language configuration data from a dict.
 
         Args:
@@ -239,7 +252,7 @@ def update(self, data):
             else:
                 self.languages[lang_id].update(lang_spec)
 
-        priorities: dict[int, Language] = {}
+        priorities: dict[int, str] = {}
         for lang_id, lang in self.languages.items():
             if lang.priority in priorities:
                 raise LanguageConfigError(

problemtools/run/source.py

@@ -125,7 +125,6 @@ def get_runcmd(self, cwd=None, memlim=1024):
             subs['path'] = os.path.relpath(subs['path'], cwd)
             subs['binary'] = os.path.relpath(subs['binary'], cwd)
             subs['mainfile'] = os.path.relpath(subs['mainfile'], cwd)
-        assert self.language.run is not None, 'Language.__check() guarantees run is always set'
         return shlex.split(self.language.run.format(**subs))
 
     def should_skip_memory_rlimit(self) -> bool: