diff --git a/airbyte-integrations/connectors/source-s3/metadata.yaml b/airbyte-integrations/connectors/source-s3/metadata.yaml index 1191a956bbf4..14871c0b58e3 100644 --- a/airbyte-integrations/connectors/source-s3/metadata.yaml +++ b/airbyte-integrations/connectors/source-s3/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: file connectorType: source definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 - dockerImageTag: 4.15.4 + dockerImageTag: 4.15.5 dockerRepository: airbyte/source-s3 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 externalDocumentationUrls: diff --git a/airbyte-integrations/connectors/source-s3/poetry.lock b/airbyte-integrations/connectors/source-s3/poetry.lock index fd9311ae85b6..35ca06161b1b 100644 --- a/airbyte-integrations/connectors/source-s3/poetry.lock +++ b/airbyte-integrations/connectors/source-s3/poetry.lock @@ -2,14 +2,14 @@ [[package]] name = "airbyte-cdk" -version = "7.19.1" +version = "7.19.2" description = "A framework for writing Airbyte Connectors." optional = false python-versions = "<3.14,>=3.10" groups = ["main"] files = [ - {file = "airbyte_cdk-7.19.1-py3-none-any.whl", hash = "sha256:982410e582174cda84a3424c783c65dca38b4370ecaf660ac5c6c8b2b298c273"}, - {file = "airbyte_cdk-7.19.1.tar.gz", hash = "sha256:882b127409193f329d91f8d9671f385d5a6b3440ee333ebfd5d235b3947eb7d8"}, + {file = "airbyte_cdk-7.19.2-py3-none-any.whl", hash = "sha256:3ff64c854e2b04351037def1e92bd105affb459bc8333d35631b58f5d06982d0"}, + {file = "airbyte_cdk-7.19.2.tar.gz", hash = "sha256:9d55df2cfbb21d83b9b98d66784e9409e753e8f62afaea2cfbbfb56a22dc00fc"}, ] [package.dependencies] diff --git a/airbyte-integrations/connectors/source-s3/pyproject.toml b/airbyte-integrations/connectors/source-s3/pyproject.toml index a4048cec6f33..1a2e70cfc56d 100644 --- a/airbyte-integrations/connectors/source-s3/pyproject.toml +++ b/airbyte-integrations/connectors/source-s3/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "4.15.4" +version = "4.15.5" name = "source-s3" description = "Source implementation for S3." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/__init__.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/__init__.py index 0869a3773d36..2c96d9029534 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/__init__.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/__init__.py @@ -8,4 +8,5 @@ from .source import SourceS3 from .stream_reader import SourceS3StreamReader + __all__ = ["Config", "Cursor", "LegacyConfigTransformer", "SourceS3", "SourceS3StreamReader"] diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py index 04a23079910b..d7b81be1ad96 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py @@ -8,6 +8,15 @@ from source_s3.v4 import Config, SourceS3, SourceS3StreamReader +from airbyte_cdk.models import AirbyteStream, SyncMode +from airbyte_cdk.sources.concurrent_source.concurrent_read_processor import ConcurrentReadProcessor +from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager +from airbyte_cdk.sources.message import MessageRepository +from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream +from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer +from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader +from airbyte_cdk.sources.utils.slice_logger import SliceLogger + _V3_FIELDS = ["dataset", "format", "path_pattern", "provider", "schema"] TEST_FILES_FOLDER = Path(__file__).resolve().parent.parent.joinpath("sample_files") @@ -52,3 +61,31 @@ def test_when_spec_then_v3_fields_descriptions_are_prefixed_with_deprecation_war def test_when_spec_then_v3_nested_fields_are_not_required(self) -> None: spec = self._source.spec() assert not spec.connectionSpecification["properties"]["provider"]["required"] + + +def test_airbyte_cdk_limits_concurrent_partition_generators() -> None: + stream = Mock(spec=AbstractStream) + stream.name = "stream" + stream.block_simultaneous_read = "" + stream.as_airbyte_stream.return_value = AirbyteStream( + name="stream", + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ) + thread_pool_manager = Mock(spec=ThreadPoolManager) + handler = ConcurrentReadProcessor( + [stream], + Mock(spec=PartitionEnqueuer), + thread_pool_manager, + Mock(), + Mock(spec=SliceLogger), + Mock(spec=MessageRepository), + Mock(spec=PartitionReader), + max_concurrent_partition_generators=1, + ) + handler._streams_currently_generating_partitions.append("active_stream") + + status_message = handler.start_next_partition_generator() + + assert status_message is None + thread_pool_manager.submit.assert_not_called() diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md index a3c4522f5b48..8566d892e60e 100644 --- a/docs/integrations/sources/s3.md +++ b/docs/integrations/sources/s3.md @@ -356,6 +356,7 @@ This connector utilizes the open source [Unstructured](https://unstructured-io.g | Version | Date | Pull Request | Subject | |:------------|:-----------|:----------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------| +| 4.15.5 | 2026-05-20 | [TBD](https://github.com/airbytehq/airbyte/pull/TBD) | Improved completion reliability for large full refresh syncs. | | 4.15.4 | 2026-05-14 | [78081](https://github.com/airbytehq/airbyte/pull/78081) | Updates the source-s3 Poetry lock file to use airbyte-cdk 7.19.1 for CSV column error. | | 4.15.3 | 2026-04-08 | [76162](https://github.com/airbytehq/airbyte/pull/76162) | Add opt-in `skip_full_check_for_parquet` stream option to skip the full parse check for Parquet files during CHECK, avoiding OOM on large files | | 4.15.2 | 2025-11-11 | [69268](https://github.com/airbytehq/airbyte/pull/69268) | Update dependencies |