diff --git a/CLAUDE.md b/CLAUDE.md index 2af94a6f..18d421e7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -231,8 +231,14 @@ All settings use the `PYLON_` prefix. Example: `PYLON_BITTENSOR_NETWORK=finney` - By default, data is cached for all subnets configured in identities - **Monitoring**: - - `sentry_dsn`: Sentry DSN for error tracking (optional) - - `sentry_environment`: Environment name for Sentry (default: "development") + - `environment` (in `pylon_commons/settings.py`, env `PYLON_ENVIRONMENT`): Deployment environment name; single source of truth for both Sentry and OTEL (default: "production") + - Sentry (`SentrySettings` in `pylon_service/settings.py`, env prefix `PYLON_SENTRY_`): + - `dsn` (`PYLON_SENTRY_DSN`): Sentry DSN for error tracking; empty disables Sentry (default: "") + - `environment` (`PYLON_SENTRY_ENVIRONMENT`): Sentry environment name; falls back to `PYLON_ENVIRONMENT` when left unset + - OpenTelemetry traces (`OtelSettings` in `pylon_service/settings.py`, env prefix `PYLON_OTEL_`): + - `collector_endpoint` (`PYLON_OTEL_COLLECTOR_ENDPOINT`): Base URL of an OTLP collector; empty disables traces (default: "") + - `deployment_environment` (`PYLON_OTEL_DEPLOYMENT_ENVIRONMENT`): OTEL `deployment.environment.name` attribute; falls back to `PYLON_ENVIRONMENT` when left unset + - `service_version` (`PYLON_OTEL_SERVICE_VERSION`): OTEL `service.version` attribute; injected automatically at Docker build time (default: "") - **Development**: - `docker_image_name`: Docker image name (default: "bittensor_pylon") diff --git a/pylon_commons/pylon_commons/settings.py b/pylon_commons/pylon_commons/settings.py index a46ee0ac..554fe8e5 100644 --- a/pylon_commons/pylon_commons/settings.py +++ b/pylon_commons/pylon_commons/settings.py @@ -54,6 +54,9 @@ class Settings(BaseSettings): max_request_timeout_seconds: float = 300.0 # evm + # WARNING: these URLs are recorded as-is in telemetry (OpenTelemetry spans, debug logs, and the + # Prometheus `rpc_url` metric label), so do NOT embed credentials in them (no `user:pass@` and no + # path/query API keys like `/v2/`). If a provider requires authentication, pass it out of band. evm_rpc_url: evm_types.RpcUrl = evm_types.RpcUrl("https://lite.chain.opentensor.ai") evm_archive_rpc_url: evm_types.RpcUrl = evm_types.RpcUrl("https://archive.chain.opentensor.ai") evm_archive_blocks_cutoff: ArchiveBlocksCutoff = ArchiveBlocksCutoff(300) diff --git a/pylon_service/README.md b/pylon_service/README.md index f5025de3..8660e651 100644 --- a/pylon_service/README.md +++ b/pylon_service/README.md @@ -466,6 +466,41 @@ That means: This keeps one execution path for public behavior. +## Observability + +The service ships optional, disabled-by-default integrations for error tracking and tracing. Both are enabled purely +by setting an endpoint/DSN — no separate feature flag. + +### Sentry + +- Enabled iff `PYLON_SENTRY_DSN` is set. Reports errors through the Litestar and asyncio integrations. + +### OpenTelemetry traces + +- Enabled iff `PYLON_OTEL_COLLECTOR_ENDPOINT` is set to the base URL of an OTLP collector (e.g. `http://alloy:4318`). + Traces are exported via OTLP HTTP/protobuf to `/v1/traces`. +- When enabled, auto-instrumentation covers: the Litestar HTTP server (incoming requests), `httpx` and `aiohttp` + (outgoing HTTP — chain RPC over `httpx`, web3/EVM RPC over `aiohttp`), and `SQLAlchemy` (database). The active span's + `trace_id` / `span_id` are injected into structured logs for log↔trace correlation. +- Outgoing HTTP URLs are recorded on spans verbatim, so do **not** embed credentials in the configured RPC URLs + (`PYLON_EVM_RPC_URL`, `PYLON_EVM_ARCHIVE_RPC_URL`, or an `http(s)://` `PYLON_BITTENSOR_NETWORK`) — see the warning in + `pylon_commons/settings.py`. The same URLs also appear in debug logs and the Prometheus `rpc_url` metric label. +- **Not traced:** the default Bittensor chain transport in `turbobt` is websockets, for which no OpenTelemetry + instrumentation exists — so chain RPC calls are not auto-traced. They are covered only when + `PYLON_BITTENSOR_NETWORK` points at an `http(s)://` URI, where `turbobt` falls back to `httpx`. +- The service does not ship a collector. Running and configuring Alloy (or any OTLP collector) at the configured + endpoint, including any tail-sampling or endpoint filtering, is the deployer's responsibility. +- **Long-lived on-chain submission spans:** background submission tasks (`apply_weights`, `set_commitment`, + `set_revealed_commitment`) emit one short, self-contained span per retry attempt, each with its own `trace_id` and + span links back to the originating request and the previous attempt — this keeps traces short across the (up to 200) + retries. A *single* attempt can still take up to 120s while waiting for extrinsic finalization (~12s per block, longer + under congestion). Backends that bound trace lifetime — notably Tempo's `max_trace_live` (default 30s) — will split + such an attempt's trace. If you use Tempo, set `max_trace_live` to at least 180s (a margin above the 120s submission + timeout) and `max_trace_idle` to at least 30s. +- Traces require the service to run as a single uvicorn process; both `--workers` and `WEB_CONCURRENCY` (other than + `1`) are rejected (see `uvicorn_entrypoint.py`) because the SDK is initialised once at import and would not survive + `fork()`. + ## Change checklist Before merging changes in `pylon_service`, verify: diff --git a/pylon_service/pylon_service/api/_unstable/tasks.py b/pylon_service/pylon_service/api/_unstable/tasks.py index 7066d3ab..ded9548d 100644 --- a/pylon_service/pylon_service/api/_unstable/tasks.py +++ b/pylon_service/pylon_service/api/_unstable/tasks.py @@ -1,7 +1,13 @@ import asyncio from abc import ABC, abstractmethod +from collections.abc import Iterator +from contextlib import contextmanager from typing import Any, ClassVar, TypeVar +from opentelemetry import trace +from opentelemetry.context import Context +from opentelemetry.trace import Link, SpanContext +from opentelemetry.util.types import AttributeValue import structlog from prometheus_client import Histogram from pylon_commons.models import Block, CommitReveal @@ -44,8 +50,10 @@ track_operation, ) from pylon_service.settings import settings +from pylon_service.tracing import TraceLinkType, get_current_valid_span_context logger = structlog.stdlib.get_logger(__name__) +_tracer = trace.get_tracer(__name__) class StopRetrying(Exception): @@ -79,10 +87,14 @@ def __init_subclass__( def __init__(self) -> None: self._running_task: asyncio.Task[ReturnT] | None = None + self._request_span_context: SpanContext | None = None + self._previous_attempt_context: SpanContext | None = None async def schedule(self) -> asyncio.Task[ReturnT]: await self._on_task_scheduled() + self._request_span_context = get_current_valid_span_context() + self._running_task = asyncio.create_task(self(), name=self.JOB_NAME) type(self).tasks_running.add(self) self._running_task.add_done_callback(self._on_task_done) @@ -91,15 +103,46 @@ async def schedule(self) -> asyncio.Task[ReturnT]: async def __call__(self) -> ReturnT: return await self._submit_with_retries() + @contextmanager + def _attempt_span(self, attempt_number: int) -> Iterator[None]: + links: list[Link] = [] + if self._request_span_context is not None: + links.append( + Link( + self._request_span_context, + attributes={TraceLinkType.ATTRIBUTE_KEY: TraceLinkType.ORIGINATING_REQUEST}, + ) + ) + if self._previous_attempt_context is not None: + links.append( + Link( + self._previous_attempt_context, + attributes={TraceLinkType.ATTRIBUTE_KEY: TraceLinkType.PREVIOUS_ATTEMPT}, + ) + ) + with _tracer.start_as_current_span( + f"{self.JOB_NAME}.attempt", + context=Context(), + links=links, + attributes={ + "attempt_number": attempt_number, + **self._attempt_span_attributes(), + }, + ): + self._previous_attempt_context = get_current_valid_span_context() + yield + async def _submit_with_retries(self) -> ReturnT: prepared = False + self._previous_attempt_context = None async def attempt() -> ReturnT: nonlocal prepared - if not prepared: - await self._prepare() - prepared = True - return await self._single_attempt() + with self._attempt_span(retrying.statistics["attempt_number"]): + if not prepared: + await self._prepare() + prepared = True + return await self._single_attempt() retrying = AsyncRetrying( stop=stop_after_attempt(self._retry_attempts + 1), @@ -149,6 +192,12 @@ def _on_task_done(self, task: asyncio.Task[ReturnT]) -> None: else: logger.info("task_finished", task=task, job_name=self.JOB_NAME) + def _attempt_span_attributes(self) -> dict[str, AttributeValue]: + """ + Per-task attributes attached to each attempt span; overridden by subclasses that have them. + """ + return {} + async def _on_task_scheduled(self) -> None: pass @@ -213,6 +262,12 @@ def _retry_attempts(self) -> int: def _retry_delay_seconds(self) -> int: return settings.weights_retry_delay_seconds + def _attempt_span_attributes(self) -> dict[str, AttributeValue]: + """ + Attach netuid and hotkey to each attempt span for trace filtering. + """ + return {"netuid": self._netuid, "hotkey": self._hotkey} + async def _on_task_scheduled(self) -> None: if self._is_rescheduled: return diff --git a/pylon_service/pylon_service/envs/test_env.template b/pylon_service/pylon_service/envs/test_env.template index 9cba1a29..99a6a9c4 100644 --- a/pylon_service/pylon_service/envs/test_env.template +++ b/pylon_service/pylon_service/envs/test_env.template @@ -38,6 +38,12 @@ PYLON_OTEL_SERVICE_VERSION= # To override, uncomment the line below and set a non-empty value (an empty value would disable the fallback). # PYLON_OTEL_DEPLOYMENT_ENVIRONMENT= +# OTLP traces exporter endpoint - base URL of the collector, e.g. http://alloy:4318. +# Leave empty to disable OpenTelemetry traces (disabled by default). When set, traces are exported via +# OTLP HTTP/protobuf to /v1/traces, and the Litestar HTTP server plus the httpx, aiohttp +# and SQLAlchemy libraries are auto-instrumented. You must run your own OTLP collector at this address. +PYLON_OTEL_COLLECTOR_ENDPOINT= + # Sentry Configuration # DSN address for pylon to use - leave empty to disable sentry integration. PYLON_SENTRY_DSN= diff --git a/pylon_service/pylon_service/logging.py b/pylon_service/pylon_service/logging.py index c3650ec1..ded26904 100644 --- a/pylon_service/pylon_service/logging.py +++ b/pylon_service/pylon_service/logging.py @@ -8,6 +8,7 @@ from pylon_service.middleware.request_id import current_request_id from pylon_service.settings import otel_settings, settings +from pylon_service.tracing import get_current_valid_span_context if TYPE_CHECKING: from structlog.typing import EventDict, WrappedLogger @@ -48,6 +49,19 @@ def add_request_id_to_structlog( return event_dict +def add_otel_context_to_structlog( + logger: WrappedLogger, + method_name: str, + event_dict: EventDict, +) -> EventDict: + """Structlog processor injecting the active span's trace_id and span_id into the log event.""" + ctx = get_current_valid_span_context() + if ctx is not None: + event_dict["trace_id"] = format(ctx.trace_id, "032x") + event_dict["span_id"] = format(ctx.span_id, "016x") + return event_dict + + def add_coro_name_to_structlog( logger: WrappedLogger, method_name: str, @@ -97,6 +111,7 @@ def add_coro_name_to_structlog( _JSON_RENDER_PROCESSORS = [ structlog.stdlib.ProcessorFormatter.remove_processors_meta, add_otel_resource_to_structlog, + add_otel_context_to_structlog, structlog.processors.format_exc_info, structlog.processors.JSONRenderer(), ] diff --git a/pylon_service/pylon_service/main.py b/pylon_service/pylon_service/main.py index 8665a557..f8fcbe24 100644 --- a/pylon_service/pylon_service/main.py +++ b/pylon_service/pylon_service/main.py @@ -1,6 +1,8 @@ from litestar import Litestar +from litestar.contrib.opentelemetry import OpenTelemetryConfig, OpenTelemetryPlugin from litestar.di import Provide from litestar.openapi.config import OpenAPIConfig +from litestar.plugins import PluginProtocol from litestar.plugins.prometheus import PrometheusConfig from pylon_service import dependencies, lifecycle @@ -11,10 +13,11 @@ from pylon_service.logging import configure_structlog, litestar_logging_config from pylon_service.middleware.request_id import RequestIdMiddleware from pylon_service.middleware.request_timeout import RequestTimeoutMiddleware +from pylon_service.otel_config import init_otel from pylon_service.prometheus_controller import AuthenticatedPrometheusController from pylon_service.schema import PylonSchemaPlugin from pylon_service.sentry_config import init_sentry -from pylon_service.settings import response_cache_config, settings +from pylon_service.settings import otel_settings, response_cache_config, settings from pylon_service.stores import stores @@ -27,6 +30,10 @@ def create_app() -> Litestar: group_path=True, # Group metrics by path template to avoid cardinality explosion ) + plugins: list[PluginProtocol] = [PylonSchemaPlugin()] + if otel_settings.traces_enabled: + plugins.append(OpenTelemetryPlugin(OpenTelemetryConfig())) + return Litestar( route_handlers=[ v1_router, @@ -49,7 +56,7 @@ def create_app() -> Litestar: lifecycle.reschedule_weight_tasks_on_startup, ], dependencies={"bt_contact_pool": Provide(dependencies.bt_contact_pool_dep, use_cache=True)}, - plugins=[PylonSchemaPlugin()], + plugins=plugins, exception_handlers={ArchiveFallbackException: archive_fallback_handler}, stores=stores, response_cache_config=response_cache_config, @@ -58,6 +65,7 @@ def create_app() -> Litestar: ) +init_otel() configure_structlog() init_sentry() app = create_app() diff --git a/pylon_service/pylon_service/otel_config.py b/pylon_service/pylon_service/otel_config.py new file mode 100644 index 00000000..bf64b1fb --- /dev/null +++ b/pylon_service/pylon_service/otel_config.py @@ -0,0 +1,33 @@ +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor +from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +from pylon_service.db.database import engine as db_engine +from pylon_service.settings import otel_settings + + +def init_otel() -> None: + """ + Initialize OpenTelemetry tracing if a traces endpoint is configured. + + Sets up a TracerProvider exporting via OTLP HTTP/protobuf and auto-instruments the httpx, + aiohttp, and SQLAlchemy libraries. A no-op when no endpoint is set. + """ + if not otel_settings.traces_enabled: + return + + resource = Resource.create(otel_settings.resource_attributes()) + provider = TracerProvider(resource=resource) + provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{otel_settings.normalized_collector_endpoint}/v1/traces")) + ) + trace.set_tracer_provider(provider) + + HTTPXClientInstrumentor().instrument() + AioHttpClientInstrumentor().instrument() + SQLAlchemyInstrumentor().instrument(engine=db_engine.sync_engine) diff --git a/pylon_service/pylon_service/settings.py b/pylon_service/pylon_service/settings.py index d072f4ec..fbc3487a 100644 --- a/pylon_service/pylon_service/settings.py +++ b/pylon_service/pylon_service/settings.py @@ -2,6 +2,8 @@ from typing import Self from litestar.config.response_cache import ResponseCacheConfig +from opentelemetry.semconv._incubating.attributes.deployment_attributes import DEPLOYMENT_ENVIRONMENT_NAME +from opentelemetry.semconv.resource import ResourceAttributes from pydantic import Field, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict from pylon_commons.settings import ENV_FILE, Settings @@ -82,6 +84,7 @@ class OtelSettings(BaseSettings): deployment_environment: str = Field(default_factory=lambda: settings.environment) service_instance_id: str = _DEFAULT_SERVICE_INSTANCE_ID service_version: str = "" + collector_endpoint: str = "" model_config = SettingsConfigDict( env_file=ENV_FILE, @@ -90,16 +93,31 @@ class OtelSettings(BaseSettings): extra="ignore", ) + @property + def normalized_collector_endpoint(self) -> str: + """ + Return the collector endpoint with surrounding whitespace and trailing slashes removed, + so signal paths can be appended without producing a double slash. + """ + return self.collector_endpoint.strip().rstrip("/") + + @property + def traces_enabled(self) -> bool: + """ + Return whether traces export is enabled (a non-empty endpoint is configured). + """ + return bool(self.normalized_collector_endpoint) + def resource_attributes(self) -> dict[str, str]: """Return OTEL resource attributes as dotted-key fields for log injection.""" attrs = { - "service.namespace": self.service_namespace, - "service.name": self.service_name, - "deployment.environment.name": self.deployment_environment, - "service.instance.id": self.service_instance_id, + ResourceAttributes.SERVICE_NAMESPACE: self.service_namespace, + ResourceAttributes.SERVICE_NAME: self.service_name, + DEPLOYMENT_ENVIRONMENT_NAME: self.deployment_environment, + ResourceAttributes.SERVICE_INSTANCE_ID: self.service_instance_id, } if self.service_version: - attrs["service.version"] = self.service_version + attrs[ResourceAttributes.SERVICE_VERSION] = self.service_version return attrs diff --git a/pylon_service/pylon_service/tracing.py b/pylon_service/pylon_service/tracing.py new file mode 100644 index 00000000..6d41d164 --- /dev/null +++ b/pylon_service/pylon_service/tracing.py @@ -0,0 +1,25 @@ +from enum import StrEnum, nonmember + +from opentelemetry import trace +from opentelemetry.trace import SpanContext + + +class TraceLinkType(StrEnum): + """ + Kind of span link attached to a retry attempt span, and the attribute key it is stored under. + """ + + ATTRIBUTE_KEY = nonmember("link.type") + + ORIGINATING_REQUEST = "originating_request" + PREVIOUS_ATTEMPT = "previous_attempt" + + +def get_current_valid_span_context() -> SpanContext | None: + """ + Return the active span's context, or None when there is no valid active span. + + The context is invalid when tracing is disabled or when running outside any span. + """ + ctx = trace.get_current_span().get_span_context() + return ctx if ctx.is_valid else None diff --git a/pylon_service/pylon_service/uvicorn_entrypoint.py b/pylon_service/pylon_service/uvicorn_entrypoint.py index beaf1d09..b71d2fcd 100644 --- a/pylon_service/pylon_service/uvicorn_entrypoint.py +++ b/pylon_service/pylon_service/uvicorn_entrypoint.py @@ -14,9 +14,21 @@ def main() -> None: # - ensuring that only one worker performs tasks rescheduling in its startup # and the other wait for it to finish # - prometheus instrumentation + # - the OpenTelemetry SDK (TracerProvider + BatchSpanProcessor's background exporter + # thread) is initialised once at module import and would not survive fork(), so multiple + # workers would lose spans and risk deadlocks; it would need per-worker post-fork setup if any(arg == "--workers" or arg.startswith("--workers=") for arg in sys.argv[1:]): raise RuntimeError("Passing --workers is not supported for pylon-service.") + # uvicorn also reads the worker count from the WEB_CONCURRENCY environment variable (common in + # container setups), which would silently fork workers without --workers ever being passed. + web_concurrency = os.environ.get("WEB_CONCURRENCY") + if web_concurrency is not None and web_concurrency.strip() not in ("", "1"): + raise RuntimeError( + f"WEB_CONCURRENCY={web_concurrency!r} is not supported for pylon-service; " + "it must run as a single process (set WEB_CONCURRENCY=1 or leave it unset)." + ) + host = os.environ.get("PYLON_UVICORN_HOST", "0.0.0.0") port = int(os.environ.get("PYLON_UVICORN_PORT", "8000")) auto_reload = settings.debug diff --git a/pylon_service/pyproject.toml b/pylon_service/pyproject.toml index effd1770..03df74a2 100644 --- a/pylon_service/pyproject.toml +++ b/pylon_service/pyproject.toml @@ -18,6 +18,15 @@ dependencies = [ "aiosqlite>=0.22.1", "web3>=7.16.0", "structlog>=25,<26", + # OpenTelemetry uses split versioning: the stable API/SDK/exporters track 1.x while the + # instrumentation packages track 0.Nb pre-releases, and both are released together (the 1.4x + # line pairs with 0.63b). The differing pin styles are intentional, not a mismatch. + "opentelemetry-sdk~=1.41", + "opentelemetry-exporter-otlp-proto-http~=1.41", + "opentelemetry-instrumentation-asgi~=0.63b1", + "opentelemetry-instrumentation-httpx~=0.63b1", + "opentelemetry-instrumentation-aiohttp-client~=0.63b1", + "opentelemetry-instrumentation-sqlalchemy~=0.63b1", ] [dependency-groups] diff --git a/pylon_service/tests/unit/test_otel_config.py b/pylon_service/tests/unit/test_otel_config.py new file mode 100644 index 00000000..820ccfa7 --- /dev/null +++ b/pylon_service/tests/unit/test_otel_config.py @@ -0,0 +1,103 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from pylon_service.otel_config import init_otel +from pylon_service.settings import OtelSettings + + +@pytest.mark.parametrize( + ("endpoint", "expected"), + [ + pytest.param("", False, id="empty_endpoint_disabled"), + pytest.param(" ", False, id="whitespace_only_disabled"), + pytest.param("http://alloy:4318", True, id="set_endpoint_enabled"), + ], +) +def test_traces_enabled_reflects_endpoint(endpoint, expected): + assert OtelSettings(collector_endpoint=endpoint).traces_enabled is expected + + +def test_init_otel_is_noop_when_disabled(): + with ( + patch("pylon_service.otel_config.otel_settings", OtelSettings(collector_endpoint="")), + patch("pylon_service.otel_config.trace.set_tracer_provider") as set_provider, + patch("pylon_service.otel_config.HTTPXClientInstrumentor") as httpx_instrumentor, + patch("pylon_service.otel_config.AioHttpClientInstrumentor") as aiohttp_instrumentor, + patch("pylon_service.otel_config.SQLAlchemyInstrumentor") as sqlalchemy_instrumentor, + ): + init_otel() + + set_provider.assert_not_called() + httpx_instrumentor.assert_not_called() + aiohttp_instrumentor.assert_not_called() + sqlalchemy_instrumentor.assert_not_called() + + +def test_init_otel_installs_provider_and_instruments_when_enabled(): + sync_engine = MagicMock() + db_engine = MagicMock(sync_engine=sync_engine) + + with ( + patch("pylon_service.otel_config.otel_settings", OtelSettings(collector_endpoint="http://alloy:4318")), + patch("pylon_service.otel_config.db_engine", db_engine), + patch("pylon_service.otel_config.trace.set_tracer_provider") as set_provider, + patch("pylon_service.otel_config.TracerProvider"), + patch("pylon_service.otel_config.BatchSpanProcessor"), + patch("pylon_service.otel_config.OTLPSpanExporter") as exporter, + patch("pylon_service.otel_config.HTTPXClientInstrumentor") as httpx_instrumentor, + patch("pylon_service.otel_config.AioHttpClientInstrumentor") as aiohttp_instrumentor, + patch("pylon_service.otel_config.SQLAlchemyInstrumentor") as sqlalchemy_instrumentor, + ): + init_otel() + + set_provider.assert_called_once() + exporter.assert_called_once_with(endpoint="http://alloy:4318/v1/traces") + httpx_instrumentor.return_value.instrument.assert_called_once_with() + aiohttp_instrumentor.return_value.instrument.assert_called_once_with() + sqlalchemy_instrumentor.return_value.instrument.assert_called_once_with(engine=sync_engine) + + +@pytest.mark.parametrize( + ("collector_endpoint", "expected_exporter_endpoint"), + [ + pytest.param("http://alloy:4318", "http://alloy:4318/v1/traces", id="bare_endpoint"), + pytest.param("http://alloy:4318/", "http://alloy:4318/v1/traces", id="trailing_slash"), + pytest.param(" http://alloy:4318/ ", "http://alloy:4318/v1/traces", id="whitespace_and_slash"), + ], +) +def test_init_otel_normalizes_collector_endpoint(collector_endpoint, expected_exporter_endpoint): + with ( + patch("pylon_service.otel_config.otel_settings", OtelSettings(collector_endpoint=collector_endpoint)), + patch("pylon_service.otel_config.db_engine", MagicMock()), + patch("pylon_service.otel_config.trace.set_tracer_provider"), + patch("pylon_service.otel_config.TracerProvider"), + patch("pylon_service.otel_config.BatchSpanProcessor"), + patch("pylon_service.otel_config.OTLPSpanExporter") as exporter, + patch("pylon_service.otel_config.HTTPXClientInstrumentor"), + patch("pylon_service.otel_config.AioHttpClientInstrumentor"), + patch("pylon_service.otel_config.SQLAlchemyInstrumentor"), + ): + init_otel() + + exporter.assert_called_once_with(endpoint=expected_exporter_endpoint) + + +@pytest.mark.parametrize( + ("traces_enabled", "expected_present"), + [ + pytest.param(True, True, id="enabled_plugin_present"), + pytest.param(False, False, id="disabled_plugin_absent"), + ], +) +def test_create_app_registers_otel_plugin_only_when_enabled(traces_enabled, expected_present): + from litestar.contrib.opentelemetry import OpenTelemetryPlugin + + from pylon_service.main import create_app + + with patch("pylon_service.main.otel_settings") as otel_settings_mock: + otel_settings_mock.traces_enabled = traces_enabled + app = create_app() + + has_plugin = any(isinstance(plugin, OpenTelemetryPlugin) for plugin in app.plugins) + assert has_plugin is expected_present diff --git a/pylon_service/tests/unit/test_request_id_context.py b/pylon_service/tests/unit/test_request_id_context.py index 5cf30381..8d90fed5 100644 --- a/pylon_service/tests/unit/test_request_id_context.py +++ b/pylon_service/tests/unit/test_request_id_context.py @@ -3,10 +3,12 @@ import pytest from litestar.types import HTTPRequestEvent, Message +from opentelemetry.sdk.trace import TracerProvider from pylon_service.logging import ( _get_current_coroutine_name, add_coro_name_to_structlog, + add_otel_context_to_structlog, add_otel_resource_to_structlog, add_request_id_to_structlog, ) @@ -112,3 +114,18 @@ def test_otel_resource_attributes_override_event_fields(): "service.instance.id": otel_settings.service_instance_id, "service.name": "pylon_service", } + + +def test_otel_context_processor_adds_nothing_without_active_span(): + assert add_otel_context_to_structlog(None, "info", {"event": "hello"}) == {"event": "hello"} + + +def test_otel_context_processor_injects_trace_and_span_ids(): + tracer = TracerProvider().get_tracer("test") + with tracer.start_as_current_span("test-span") as span: + ctx = span.get_span_context() + assert add_otel_context_to_structlog(None, "info", {"event": "hello"}) == { + "event": "hello", + "trace_id": format(ctx.trace_id, "032x"), + "span_id": format(ctx.span_id, "016x"), + } diff --git a/pylon_service/uv.lock b/pylon_service/uv.lock index 786d29ac..ee06d229 100644 --- a/pylon_service/uv.lock +++ b/pylon_service/uv.lock @@ -169,6 +169,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/74/f5/9373290775639cb67a2fce7f629a1c240dce9f12fe927bc32b2736e16dfc/argcomplete-3.6.3-py3-none-any.whl", hash = "sha256:f5007b3a600ccac5d25bbce33089211dfd49eab4a7718da3f10e3082525a92ce", size = 43846, upload-time = "2025-10-20T03:33:33.021Z" }, ] +[[package]] +name = "asgiref" +version = "3.11.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/63/40/f03da1264ae8f7cfdbf9146542e5e7e8100a4c66ab48e791df9a03d3f6c0/asgiref-3.11.1.tar.gz", hash = "sha256:5f184dc43b7e763efe848065441eac62229c9f7b0475f41f80e207a114eda4ce", size = 38550, upload-time = "2026-02-03T13:30:14.33Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/0a/a72d10ed65068e115044937873362e6e32fab1b7dce0046aeb224682c989/asgiref-3.11.1-py3-none-any.whl", hash = "sha256:e8667a091e69529631969fd45dc268fa79b99c92c5fcdda727757e52146ec133", size = 24345, upload-time = "2026-02-03T13:30:13.039Z" }, +] + [[package]] name = "attrs" version = "25.4.0" @@ -375,6 +384,12 @@ dependencies = [ { name = "bittensor-pylon-commons" }, { name = "bittensor-wallet" }, { name = "litestar", extra = ["prometheus", "standard"] }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-instrumentation-aiohttp-client" }, + { name = "opentelemetry-instrumentation-asgi" }, + { name = "opentelemetry-instrumentation-httpx" }, + { name = "opentelemetry-instrumentation-sqlalchemy" }, + { name = "opentelemetry-sdk" }, { name = "sentry-sdk", extra = ["litestar"] }, { name = "sqlalchemy" }, { name = "structlog" }, @@ -416,6 +431,12 @@ requires-dist = [ { name = "bittensor-pylon-commons", editable = "../pylon_commons" }, { name = "bittensor-wallet" }, { name = "litestar", extras = ["prometheus", "standard"] }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = "~=1.41" }, + { name = "opentelemetry-instrumentation-aiohttp-client", specifier = "~=0.63b1" }, + { name = "opentelemetry-instrumentation-asgi", specifier = "~=0.63b1" }, + { name = "opentelemetry-instrumentation-httpx", specifier = "~=0.63b1" }, + { name = "opentelemetry-instrumentation-sqlalchemy", specifier = "~=0.63b1" }, + { name = "opentelemetry-sdk", specifier = "~=1.41" }, { name = "sentry-sdk", extras = ["litestar"] }, { name = "sqlalchemy", specifier = ">=2.0.49" }, { name = "structlog", specifier = ">=25,<26" }, @@ -1210,6 +1231,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.75.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035, upload-time = "2026-05-07T08:04:49.423Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, +] + [[package]] name = "greenlet" version = "3.5.0" @@ -1658,6 +1691,175 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b9/34/434c594e0125a16b05a7bedaea33e63c90abbfbe47e5729a735a8a8a90ea/nox-2025.11.12-py3-none-any.whl", hash = "sha256:707171f9f63bc685da9d00edd8c2ceec8405b8e38b5fb4e46114a860070ef0ff", size = 74447, upload-time = "2025-11-12T18:39:01.575Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/1c/125e1c936c0873796771b7f04f6c93b9f1bf5d424cea90fda94a99f61da8/opentelemetry_api-1.42.1.tar.gz", hash = "sha256:56c63bea9f77b62856be8c47600474acad853b2924b99b1687c4cb6297166716", size = 72296, upload-time = "2026-05-21T16:32:49.335Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/ca/9520cc1f3dfbbd03ac5903bbf55833e257bc64b1cf30fa8b0d6df374d821/opentelemetry_api-1.42.1-py3-none-any.whl", hash = "sha256:51a69edacadbc03a8950ace1c4c21099cacc538820ac2c9e36277e78cebba714", size = 61311, upload-time = "2026-05-21T16:32:28.822Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/9c/216acfeaedadf2e1937f4373929b20f73197c5c4a2546d4f584b7fa63813/opentelemetry_exporter_otlp_proto_common-1.42.1.tar.gz", hash = "sha256:04f1f01fb597c4249dfcd7f8b861c902c2102369d376d9d346ff38de4469a2ee", size = 21433, upload-time = "2026-05-21T16:32:55.526Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/43/2375e7612e1121a4518c17603b6e0b03ad94f565aafad53f464dc5be2bf6/opentelemetry_exporter_otlp_proto_common-1.42.1-py3-none-any.whl", hash = "sha256:f48d395ab815b444da118868977e9798ea354c25737d5cf39578ae894011c140", size = 17327, upload-time = "2026-05-21T16:32:33.387Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/32/826bfa1d80ecea24f47808de03cd4a0d13c17ecc07712f45123f0f61e4ac/opentelemetry_exporter_otlp_proto_http-1.42.1.tar.gz", hash = "sha256:bf142a21035d7571ac3a09cb2e5639f49886f243972883cfe777ed3bf02b734d", size = 25406, upload-time = "2026-05-21T16:32:56.807Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/96/82cb223a1502f0787d4bbff12907f5f8d870a50731febcd5818d93ef9555/opentelemetry_exporter_otlp_proto_http-1.42.1-py3-none-any.whl", hash = "sha256:00a16da1b312a1d6c7233d600d557c91df71125af73020f3b9a7765bd699d59d", size = 21793, upload-time = "2026-05-21T16:32:35.277Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "packaging" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/6d/4de72d97ff54db1ed270c7a59c9b904b917c0ac7af429c086c388b824ddb/opentelemetry_instrumentation-0.63b1.tar.gz", hash = "sha256:32368d6ae52c8de20aa790a6ad86b10a76f09956092337ae37d675773990e541", size = 41081, upload-time = "2026-05-21T16:36:14.206Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/a1/9314e621c143e4d82a5bf7a43c2ff7a745d31023506336857607c8c543cc/opentelemetry_instrumentation-0.63b1-py3-none-any.whl", hash = "sha256:f1986716d52cc316ea5f60189098726a9071d8ecc0eee96c9ed110be08bade9c", size = 35577, upload-time = "2026-05-21T16:34:56.818Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-aiohttp-client" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-http" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/47/6f/e7105760ec528b465238a06a05f8e6c358063e00ad53fed76fd625c6230c/opentelemetry_instrumentation_aiohttp_client-0.63b1.tar.gz", hash = "sha256:ec97399c02a7e278359efffdf16e93d59a7103b16f66790cda9b9496b171b136", size = 19041, upload-time = "2026-05-21T16:36:15.62Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/f8/f18666128e4b602601316ee73f35986c0a42ce44a615fd6b0f566c15e282/opentelemetry_instrumentation_aiohttp_client-0.63b1-py3-none-any.whl", hash = "sha256:5259c2c5103a5919941e0c45f2c95b055a50eb2ab39dc252f4b1e41ce6d984bb", size = 13675, upload-time = "2026-05-21T16:34:59.263Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-asgi" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asgiref" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-http" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a0/b5/7ea3a9fd1b80e89786c14250bfaecf32a753c3fd08232690f4da8dc16e29/opentelemetry_instrumentation_asgi-0.63b1.tar.gz", hash = "sha256:267b422416d768f3c7f4054883b41d9c3a7c943d86d20032b738c99a3dbb5862", size = 26151, upload-time = "2026-05-21T16:36:18.368Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/7e/83986f27b421de04fab1e1a84e892621dac42e6432a9c66779505f4d1381/opentelemetry_instrumentation_asgi-0.63b1-py3-none-any.whl", hash = "sha256:1a22453dfa965f14799b10a674b8acbcb897a8a75c79136060af54214cc7886e", size = 15906, upload-time = "2026-05-21T16:35:04.162Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-httpx" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-http" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/27/c2b4335bca030e893acbe5ff2b4f434868773bf94508be7e6bf5af981b24/opentelemetry_instrumentation_httpx-0.63b1.tar.gz", hash = "sha256:f41ec82f25c3abcdada621052db3e5fd648e3b43d55eec4b9c0c5d3ecb7b4ff4", size = 23557, upload-time = "2026-05-21T16:36:34.583Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/b8/f536780996195c3b9f2354998554671e05a7a262df8c043f63fe9e5a6f0b/opentelemetry_instrumentation_httpx-0.63b1-py3-none-any.whl", hash = "sha256:14df6e99d81be9a8cd238f6639b6fa52404c4d3ce219058fcb5dc8c0f2211f86", size = 16336, upload-time = "2026-05-21T16:35:32.221Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-sqlalchemy" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "packaging" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/97/e5cb3ad027aebf7128faadeefe4d4cb0fc07ed32ef95e8fc9d828a077a85/opentelemetry_instrumentation_sqlalchemy-0.63b1.tar.gz", hash = "sha256:621f9eb800ea24a98b4eda968373e3909bfede0ff47f77b96f8b8a18bc2a2a1a", size = 18006, upload-time = "2026-05-21T16:36:46.855Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/bc/c0984c4c51da64cc2c37ce031b4fb7fab61d223f2188a6bc6b5f18035ae3/opentelemetry_instrumentation_sqlalchemy-0.63b1-py3-none-any.whl", hash = "sha256:d417414f6517963e9c1ee91ec971b94938b46904499114d035a43937bd62b6a1", size = 14410, upload-time = "2026-05-21T16:35:53.342Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/55/63eac3e1089b768ba014091fdd2ae8a9a440c821ef5e2b786909c94c8836/opentelemetry_proto-1.42.1.tar.gz", hash = "sha256:c6a51e6b4f05ae63565f3a113217f3d2bfaec68f78c02d7a6c85f9010d1cfca6", size = 45839, upload-time = "2026-05-21T16:33:03.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/9d/171c02c84a76940b7e601805b3bb536985aded9168fbcc9ba52f0a730fa2/opentelemetry_proto-1.42.1-py3-none-any.whl", hash = "sha256:dedb74cba2886c59c7789b227a7a670613025a07489040050aedff6e5c0fb43c", size = 71782, upload-time = "2026-05-21T16:32:44.867Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/f7/b390bd9bfd703bf98a68fea1f27786c6872331fd617164a54b8a59bdc008/opentelemetry_sdk-1.42.1.tar.gz", hash = "sha256:8c834e8f8c9ba4171d4ec843d0cb8a67e4c7394d3f9e9297e582cbd9456ddbf7", size = 239262, upload-time = "2026-05-21T16:33:04.641Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/6b/4287766cfbde577ae2272e8884abac325aeaac0d64f41c61d5b8cc595105/opentelemetry_sdk-1.42.1-py3-none-any.whl", hash = "sha256:083cd4bbfaa5aa7b5a9e552430d9951219967cfb27aa61feb13a77aba1fc839d", size = 170907, upload-time = "2026-05-21T16:32:45.894Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/99/4d7dd6df64795951413ce6e815f8cf1eb191daf7196ae86574589643d5f3/opentelemetry_semantic_conventions-0.63b1.tar.gz", hash = "sha256:3daf963611334b365e98a57438183eb012d3bfb40b2d931a9af613476b8701a9", size = 148340, upload-time = "2026-05-21T16:33:05.455Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/7a/7fe66f5f3682b1dd47d88cc4e11f1c6c0966b737de2d16671146e23c39a5/opentelemetry_semantic_conventions-0.63b1-py3-none-any.whl", hash = "sha256:dfe5ef4dee82586b746f522b818ceb298d00b3d59f660042bd79404bff8d0682", size = 203713, upload-time = "2026-05-21T16:32:47.016Z" }, +] + +[[package]] +name = "opentelemetry-util-http" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/d8/7bf5e4cec0578ac3c28c18eb7b88f34279139cbc8c568d6aa02b9c5ae53e/opentelemetry_util_http-0.63b1.tar.gz", hash = "sha256:ba1268f00922ee522dba2ae38458060f99486e7385a8056985901ca9685adfff", size = 11102, upload-time = "2026-05-21T16:36:56.675Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/f1/34e047e8f6a3c67e5220acf1af7b9f62868c25d77791bca74457bd2180a6/opentelemetry_util_http-0.63b1-py3-none-any.whl", hash = "sha256:6284194028c59cd439f8acfe388145069a6127f11dc077e1344a2094adacc3f8", size = 8205, upload-time = "2026-05-21T16:36:09.736Z" }, +] + [[package]] name = "packaging" version = "25.0" @@ -1847,6 +2049,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/5a/bc7b4a4ef808fa59a816c17b20c4bef6884daebbdf627ff2a161da67da19/propcache-0.4.1-py3-none-any.whl", hash = "sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237", size = 13305, upload-time = "2025-10-08T19:49:00.792Z" }, ] +[[package]] +name = "protobuf" +version = "6.33.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, +] + [[package]] name = "pycparser" version = "3.0"