diff --git a/.github/workflows/indexer-sync.yml b/.github/workflows/indexer-sync.yml index 89a9eab79..d2642e615 100644 --- a/.github/workflows/indexer-sync.yml +++ b/.github/workflows/indexer-sync.yml @@ -56,7 +56,7 @@ jobs: uses: actions/checkout@v6 with: token: ${{ secrets.GITHUB_TOKEN }} - fetch-depth: 50 + fetch-depth: 0 fetch-tags: false - name: Cache Python dependencies @@ -105,7 +105,7 @@ jobs: echo "✅ Jackett repository fetched successfully" echo "🔄 Fetching prowlarr/indexers repository..." - git fetch origin master --depth=50 + git fetch origin master echo "✅ prowlarr/indexers repository fetched successfully" - name: Run indexer sync diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 930d8a966..590cf3922 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ This guide covers how to contribute to the Prowlarr Indexers repository, includi ## Prerequisites > [!IMPORTANT] -> **Python 3.11 or higher** is required - must be installed and accessible via `python3` command +> **Python 3.11 or higher** is required - must be installed and accessible via `python3` command (alternatively `python` command if it points to Python 3) - Git - Basic understanding of YAML and JSON Schema @@ -13,20 +13,29 @@ This guide covers how to contribute to the Prowlarr Indexers repository, includi ## Setup 1. Fork and clone the repository: -```bash -git clone https://github.com/YOUR_USERNAME/Indexers.git -cd Indexers -``` + ```bash + git clone https://github.com/YOUR_USERNAME/Indexers.git + cd Indexers + ``` 2. Set up Python environment: -```bash -# Create virtual environment (recommended) -python3 -m venv .venv -source .venv/bin/activate + (on Linux/Mac) + ```bash + # Create virtual environment (recommended) + python3 -m venv .venv + source .venv/bin/activate + ``` + (or on Windows) + ```bash + # Create virtual environment (recommended) + python -m venv .venv + source .venv/Scripts/activate + ``` -# Install dependencies -pip install -r requirements.txt -``` +3. Install dependencies: + ```bash + pip install -r requirements.txt + ``` ## Script Commands @@ -208,7 +217,7 @@ The repository uses several automated workflows to ensure code quality: - **Tools**: `yamllint` with GitHub annotations - **Runs**: On every change to definition files -#### 2. Python Validation (`python-validation.yml`) +#### 2. Python Validation (`python-validation.yml`) - **Triggers**: Push/PR to `master` on Python files in `scripts/` or `requirements.txt` - **Purpose**: Validates Python script syntax and functionality - **Tools**: `py_compile` syntax checking @@ -222,7 +231,7 @@ The repository uses several automated workflows to ensure code quality: #### 4. Indexer Sync Automation (`indexer-sync.yml`) - **Schedule**: 3 times daily (2 AM, 10 AM, 6 PM UTC) - **Purpose**: Automatically syncs indexers from Jackett repository -- **Features**: +- **Features**: - Automated PR creation for updates - Manual trigger with debug options - Caching for performance (Python deps + Jackett data) @@ -285,13 +294,13 @@ We sync indexer definitions from [Jackett](https://github.com/Jackett/Jackett). ```bash # Create virtual environment (recommended) python -m venv .venv - + # Activate virtual environment # On Linux/Mac: source .venv/bin/activate # On Windows: source .venv/Scripts/activate - + # Install Python dependencies pip install -r requirements.txt ``` diff --git a/requirements.txt b/requirements.txt index c5e7ba819..57287e661 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -jsonschema>=4.0.0 -PyYAML>=6.0 \ No newline at end of file +jsonschema>=4.26.0 +PyYAML>=6.0.3 \ No newline at end of file diff --git a/scripts/indexer-sync-v2.sh b/scripts/indexer-sync-v2.sh index 1fe950582..b5d1aef6f 100755 --- a/scripts/indexer-sync-v2.sh +++ b/scripts/indexer-sync-v2.sh @@ -245,7 +245,9 @@ initialize_script() { # Check for Python and virtual environment # Check for Python and determine command to use PYTHON_CMD="" - if command -v python3 &> /dev/null; then + if [[ -f ".venv/Scripts/python.exe" ]]; then + PYTHON_CMD=".venv/Scripts/python.exe" + elif command -v python3 &> /dev/null; then PYTHON_CMD="python3" elif command -v python &> /dev/null; then PYTHON_CMD="python" diff --git a/scripts/validate.py b/scripts/validate.py index f294b9224..153c88cbf 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -166,7 +166,7 @@ def validate_file_against_schema(yaml_path, schema, all_errors=False): except Exception as e: return False, f"Error validating {yaml_path}: {str(e)}" -def validate_files_in_directory(directory, schema_path, all_errors=False): +def validate_files_in_directory(directory, schema_path, all_errors=False, verbose=False): """Validate all YAML files in a directory against a single schema.""" success = True error_count = 0 @@ -199,7 +199,8 @@ def validate_files_in_directory(directory, schema_path, all_errors=False): success = False error_count += 1 else: - print(f"PASS: {os.path.basename(yaml_file)}") + if verbose: + print(f"PASS: {os.path.basename(yaml_file)}") print(f"\nValidation Summary:") print(f"Total files: {total_files}") @@ -208,92 +209,100 @@ def validate_files_in_directory(directory, schema_path, all_errors=False): return success -def validate_directory(definitions_dir, all_errors=False): - """Validate all YAML files in a definitions directory.""" +def _find_yaml_files_without_schema(definitions_dir): + """Handle the case where no version dirs or root schema exist.""" + print(f"No version directories or root schema found in {definitions_dir}") + print(f"Searching for YAML files without schema validation...") + yaml_files = [] + for extension in YAML_EXTENSIONS: + yaml_files.extend(glob.glob(os.path.join(definitions_dir, extension))) + if yaml_files: + print(f"Found {len(yaml_files)} YAML files but no schema for validation") + for yaml_file in sorted(yaml_files): + print(f"SKIP: {os.path.basename(yaml_file)} (no schema)") + else: + print(f"No YAML files found in {definitions_dir}") + return False + + +def _validate_version_dir(version_dir, all_errors=False, verbose=False): + """Validate all YAML files in a single version directory. Returns (success, error_count, total_files).""" + version_str = os.path.basename(version_dir)[1:] # Remove 'v' prefix + try: + version_num = int(version_str) + except ValueError: + version_num = 0 + + print(f"Validating {version_dir}") + + schema_path = os.path.join(version_dir, SCHEMA_FILENAME) + if not os.path.exists(schema_path): + if version_num >= MIN_SCHEMA_VERSION: + print(f"Warning: No schema.json found in {version_dir}") + return True, 0, 0 + + schema = load_json_schema(schema_path) + if schema is None: + print(f"Error: Failed to load schema from {schema_path}") + return False, 1, 0 + + yaml_files = [] + for extension in YAML_EXTENSIONS: + yaml_files.extend(glob.glob(os.path.join(version_dir, extension))) + + if not yaml_files: + if version_num >= MIN_SCHEMA_VERSION: + print(f"No YAML files found in {version_dir}") + return True, 0, 0 + success = True error_count = 0 - total_files = 0 - + for yaml_file in sorted(yaml_files): + is_valid, error_msg = validate_file_against_schema(yaml_file, schema, all_errors) + if not is_valid: + print(f"FAIL: {error_msg}") + success = False + error_count += 1 + elif verbose: + print(f"PASS: {os.path.basename(yaml_file)}") + + return success, error_count, len(yaml_files) + + +def validate_directory(definitions_dir, all_errors=False, verbose=False): + """Validate all YAML files in a definitions directory.""" # Check for schema.json in root directory first (Jackett-style) root_schema_path = os.path.join(definitions_dir, SCHEMA_FILENAME) if os.path.exists(root_schema_path): print(f"Found root schema, validating files in {definitions_dir}") - return validate_files_in_directory(definitions_dir, root_schema_path, all_errors) - + return validate_files_in_directory(definitions_dir, root_schema_path, all_errors, verbose) + # Find all version directories (Prowlarr-style) - version_dirs = glob.glob(os.path.join(definitions_dir, "v*")) - version_dirs.sort() - + version_dirs = sorted(glob.glob(os.path.join(definitions_dir, "v*"))) if not version_dirs: - print(f"No version directories or root schema found in {definitions_dir}") - print(f"Searching for YAML files without schema validation...") - yaml_files = [] - for extension in YAML_EXTENSIONS: - yaml_files.extend(glob.glob(os.path.join(definitions_dir, extension))) - - if yaml_files: - print(f"Found {len(yaml_files)} YAML files but no schema for validation") - for yaml_file in sorted(yaml_files): - print(f"SKIP: {os.path.basename(yaml_file)} (no schema)") - return False - else: - print(f"No YAML files found in {definitions_dir}") - return False - + return _find_yaml_files_without_schema(definitions_dir) + + success = True + error_count = 0 + total_files = 0 + for version_dir in version_dirs: if not os.path.isdir(version_dir): continue - - # Extract version number to check against minimum - version_str = os.path.basename(version_dir)[1:] # Remove 'v' prefix - try: - version_num = int(version_str) - except ValueError: - version_num = 0 - - print(f"Validating {version_dir}") - - schema_path = os.path.join(version_dir, SCHEMA_FILENAME) - if not os.path.exists(schema_path): - if version_num >= MIN_SCHEMA_VERSION: - print(f"Warning: No schema.json found in {version_dir}") - continue - - schema = load_json_schema(schema_path) - if schema is None: - print(f"Error: Failed to load schema from {schema_path}") + dir_success, dir_errors, dir_files = _validate_version_dir(version_dir, all_errors, verbose) + if not dir_success: success = False - continue - - # Find all YAML files in this version directory - yaml_files = [] - for extension in YAML_EXTENSIONS: - yaml_files.extend(glob.glob(os.path.join(version_dir, extension))) - - if not yaml_files: - # Only log "no files" for versions at or above minimum - if version_num >= MIN_SCHEMA_VERSION: - print(f"No YAML files found in {version_dir}") - continue - - for yaml_file in sorted(yaml_files): - total_files += 1 - is_valid, error_msg = validate_file_against_schema(yaml_file, schema, all_errors) - - if not is_valid: - print(f"FAIL: {error_msg}") - success = False - error_count += 1 - else: - print(f"PASS: {os.path.basename(yaml_file)}") - + error_count += dir_errors + total_files += dir_files + print(f"\nValidation Summary:") print(f"Total files: {total_files}") print(f"Errors: {error_count}") print(f"Success: {total_files - error_count}") - + return success + def find_best_schema_version(yaml_file, definitions_dir=DEFAULT_DEFINITIONS_DIR): """Find the best schema version for a YAML file.""" matched_version = 0 @@ -355,6 +364,14 @@ def main(): try: # Determine the definitions directory to use definitions_dir = args.definitions_dir_override or args.definitions_dir + + # Confine the CLI-provided path to the working tree before any filesystem + # access, so a crafted argument cannot escape into arbitrary directories. + _base_dir = os.path.realpath(os.getcwd()) + definitions_dir = os.path.realpath(definitions_dir) + if definitions_dir != _base_dir and not definitions_dir.startswith(_base_dir + os.sep): + print(f"Error: definitions directory must be within {_base_dir}", file=sys.stderr) + sys.exit(1) # Handle caching override if args.no_cache: @@ -387,7 +404,7 @@ def main(): if not os.path.exists(definitions_dir): print(f"Error: Definitions directory '{definitions_dir}' not found", file=sys.stderr) sys.exit(1) - success = validate_directory(definitions_dir, all_errors) + success = validate_directory(definitions_dir, all_errors, args.verbose) if args.single or not hasattr(args, 'find_best_version'): if success: