Luigi scheduler running with Py 3.12 and HelloWorldTask success on local scheduler

.gitignore

@@ -172,3 +172,4 @@ Icon
 Network Trash Folder
 Temporary Items
 .apdisk
+.venv

.mcp.json

@@ -0,0 +1,51 @@
+{
+  "mcpServers": {
+    "microsoft/markitdown": {
+      "type": "stdio",
+      "command": "uvx",
+      "args": [
+        "markitdown-mcp==0.0.1a4"
+      ],
+      "gallery": "https://api.mcp.github.com",
+      "version": "1.0.0"
+    },
+    "Context7": {
+      "type": "stdio",
+      "command": "npx",
+      "args": [
+        "-y",
+        "@upstash/context7-mcp@latest"
+      ]
+    },
+    "spark-history-server": {
+      "type": "http",
+      "url": "http://localhost:18888/mcp",
+      "env": {
+        "SHS_MCP_TRANSPORT": "stdio"
+      }
+    },
+    "memory-bank": {
+      "type": "stdio",
+      "command": "uvx",
+      "args": [
+        "--from",
+        "git+ssh://git@github.com/Affirm/ai-memory-bank-mcp",
+        "mcp_memory_bank_setup"
+      ]
+    },
+    "notion": {
+      "type": "stdio",
+      "command": "npx",
+      "args": [
+        "-y",
+        "mcp-remote",
+        "https://mcp.notion.com/mcp"
+      ]
+    },
+    "github": {
+      "type": "http",
+      "url": "https://api.githubcopilot.com/mcp/"
+    }
+  },
+  "inputs": []
+}

.python-version

@@ -0,0 +1 @@
+3.12.7

CLAUDE.md

@@ -0,0 +1,94 @@
+# Luigi Development Guide
+
+## Overview
+Luigi is a Python package for building complex pipelines of batch jobs. It handles dependency resolution, workflow management, visualization, and more.
+
+## Development Setup
+
+### Virtual Environment
+```bash
+source .venv/bin/activate
+```
+
+### Running Tests
+```bash
+# Run all tests and log results
+./run_tests.sh
+
+# Run specific test file
+python -m pytest test/some_test.py -v
+
+# Run specific test
+python -m pytest test/some_test.py::TestClass::test_method -v
+
+# Run with config (required for some tests)
+LUIGI_CONFIG_PATH=test/testconfig/luigi.cfg python -m pytest test/some_test.py -v
+```
+
+## Project Structure
+- `luigi/` - Main package source code
+- `test/` - Test files
+- `test/contrib/` - Tests for contrib modules (AWS, databases, etc.)
+- `test/testconfig/` - Test configuration files
+
+## Key Files
+- `luigi/worker.py` - Task execution worker
+- `luigi/scheduler.py` - Central scheduler
+- `luigi/task.py` - Base Task class
+- `luigi/parameter.py` - Parameter types
+- `luigi/contrib/` - Integration modules (S3, ECS, Batch, etc.)
+
+## Python 3.12 Compatibility Notes
+- `random.seed()` no longer accepts tuples - use `hash()` to convert
+- `pickle.dump()` requires binary mode (`"wb"`)
+- `nose` module uses removed `imp` module - use pytest marks instead
+- `logging.config.fileConfig()` raises `FileNotFoundError` for missing files (was `KeyError`)
+
+## Running Luigi
+
+### Quick Test with Local Scheduler (No Server)
+For quick testing without starting a server, use `--local-scheduler` which runs an in-memory scheduler:
+```bash
+# Run the hello world example with in-memory scheduler (no web UI)
+# Note: PYTHONPATH=. is needed to find the examples module from project root
+PYTHONPATH=. luigi --module examples.hello_world examples.HelloWorldTask --local-scheduler
+```
+
+### Central Scheduler with Web UI (luigid)
+The `luigid` daemon provides a central scheduler with web interface at http://localhost:8082
+
+#### Run in Foreground
+```bash
+# Create log directory first
+mkdir -p /tmp/luigi-logs
+
+# Start the scheduler with web UI (http://localhost:8082)
+luigid --logdir /tmp/luigi-logs
+
+# Or with state persistence (survives restarts)
+luigid --port 8082 --logdir /tmp/luigi-logs --state-path /tmp/luigi-state.pickle
+
+# In another terminal, run a task against the central scheduler
+PYTHONPATH=. luigi --module examples.hello_world examples.HelloWorldTask
+```
+
+#### Run in Background
+```bash
+mkdir -p /tmp/luigi-logs
+
+# Start scheduler in background
+luigid --background --logdir /tmp/luigi-logs --pidfile /tmp/luigi.pid
+
+# Run a task
+PYTHONPATH=. luigi --module examples.hello_world examples.HelloWorldTask
+
+# Kill the scheduler
+kill $(cat /tmp/luigi.pid)
+# Or if pidfile not used
+pkill -f luigid
+```
+
+## Common Test Issues
+- boto3 tests require AWS region configuration or proper mocking
+- SQLAlchemy tests need eager loading for relationships to avoid DetachedInstanceError
+- Process-related tests may need small delays for `/proc` filesystem to be ready

luigi/contrib/gcs.py

@@ -28,10 +28,9 @@
 except ImportError:
     from urllib.parse import urlsplit
 
+import io
 from luigi.contrib import gcp
 import luigi.target
-from luigi import six
-from luigi.six.moves import xrange
 
 logger = logging.getLogger('luigi-interface')
 
@@ -70,7 +69,7 @@ def _wait_for_consistency(checker):
     This is necessary for e.g. create/delete where the operation might return,
     but won't be reflected for a bit.
     """
-    for _ in xrange(EVENTUAL_CONSISTENCY_MAX_SLEEPS):
+    for _ in range(EVENTUAL_CONSISTENCY_MAX_SLEEPS):
         if checker():
             return
 
@@ -255,10 +254,10 @@ def put(self, filename, dest_path, mimetype=None, chunksize=None):
 
     def put_string(self, contents, dest_path, mimetype=None):
         mimetype = mimetype or mimetypes.guess_type(dest_path)[0] or DEFAULT_MIMETYPE
-        assert isinstance(mimetype, six.string_types)
-        if not isinstance(contents, six.binary_type):
+        assert isinstance(mimetype, str)
+        if not isinstance(contents, bytes):
             contents = contents.encode("utf-8")
-        media = http.MediaIoBaseUpload(six.BytesIO(contents), mimetype, resumable=bool(contents))
+        media = http.MediaIoBaseUpload(io.BytesIO(contents), mimetype, resumable=bool(contents))
         self._do_put(media, dest_path)
 
     def mkdir(self, path, parents=True, raise_if_exists=False):

luigi/contrib/hdfs/target.py

@@ -21,12 +21,11 @@
 import luigi
 import random
 import warnings
+from urllib import parse as urlparse
 from luigi.target import FileSystemTarget
 from luigi.contrib.hdfs.config import tmppath
 from luigi.contrib.hdfs import format as hdfs_format
 from luigi.contrib.hdfs import clients as hdfs_clients
-from luigi.six.moves.urllib import parse as urlparse
-from luigi.six.moves import range
 
 
 class HdfsTarget(FileSystemTarget):