From c740f8d478271709496c2d2f01796acc1da576b4 Mon Sep 17 00:00:00 2001 From: Mariia Borodii Date: Tue, 1 Sep 2026 13:07:35 +0000 Subject: [PATCH 1/4] Return nested recipe's dataframe from the recipe write connector When a recipe is used as a write: - recipe: step, its own write: - dataframe: step can now shape the outer recipe's returned dataframe, instead of being silently discarded. Addresses the second scenario in #1157. --- tests/connectors/test_recipe.py | 31 +++++++++++++++++++++++++++++++ wrangles/connectors/recipe.py | 7 +++++-- wrangles/recipe.py | 9 +++++++-- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/tests/connectors/test_recipe.py b/tests/connectors/test_recipe.py index 10799612b..fbfd5aca0 100644 --- a/tests/connectors/test_recipe.py +++ b/tests/connectors/test_recipe.py @@ -46,6 +46,37 @@ def test_write_recipe_connector(): assert df['Find2'].iloc[0] == 'brg' +def test_write_recipe_connector_returns_nested_dataframe(): + """ + A nested recipe used as a write: - recipe: step can shape the + return value of the outer recipe via its own write: - dataframe: step. + https://github.com/wrangleworks/WranglesPY/issues/1157 + """ + recipe = """ + read: + - test: + rows: 1 + values: + Col1: Val1 + Col2: Val2 + + write: + - recipe: + wrangles: + - create.column: + output: Col3 + value: Val3 + write: + - dataframe: + columns: + - Col2 + - Col3 + """ + df = wrangles.recipe.run(recipe) + assert list(df.columns) == ['Col2', 'Col3'] + assert df['Col3'].iloc[0] == 'Val3' + + # Running recipe def test_run_recipe_connector(): recipe = """ diff --git a/wrangles/connectors/recipe.py b/wrangles/connectors/recipe.py index 711818482..0df956655 100644 --- a/wrangles/connectors/recipe.py +++ b/wrangles/connectors/recipe.py @@ -109,7 +109,7 @@ def write( columns: list = None, functions: _Union[_types.FunctionType, list] = [], **kwargs -) -> None: +) -> _pd.DataFrame: """ Run a recipe, from a recipe! Recipe-ception. This will trigger a new recipe with the contents of the current recipe. @@ -121,6 +121,9 @@ def write( :param variables: (Optional) A dictionary of custom variables to override placeholders in the recipe. Variables can be indicated as ${MY_VARIABLE}. Variables can also be overwritten by Environment Variables. :param columns: (Optional) A list of the columns to pass to the recipe. If omitted, all columns will be included. :param functions: Pass in a custom function or list of custom functions that can be called in the recipe. + :return: The nested recipe's resulting dataframe, so it can be used to \ + shape the return value of the outer recipe (e.g. via the nested \ + recipe's own write: - dataframe: step). """ if variables is None: variables = {} @@ -130,7 +133,7 @@ def write( columns = _wildcard_expansion(df.columns, columns) df = df[columns] - _recipe.run(name, dataframe=df, variables=variables, functions=functions) + return _recipe.run(name, dataframe=df, variables=variables, functions=functions) _schema['write'] = """ diff --git a/wrangles/recipe.py b/wrangles/recipe.py index c35f19e49..993287474 100644 --- a/wrangles/recipe.py +++ b/wrangles/recipe.py @@ -1122,8 +1122,13 @@ def _write_data( # Validate with a placeholder for df _validate_function_args(func, {"df": None, **args}, export_type) - # Execute the function - func(df_temp, **args) + # Execute the function. If it returns a dataframe + # (e.g. the recipe connector reflecting a nested + # write: - dataframe: step), use that as the + # dataframe returned by this recipe. + result = func(df_temp, **args) + if result is not None: + df_return = result except Exception as e: # Wrap with enhanced error information _wrap_and_raise('WRITE', export_type, None, e) From 60be6d7d9df43f268170e7ae86230c5093494aaf Mon Sep 17 00:00:00 2001 From: Mariia Borodii Date: Tue, 1 Sep 2026 13:32:21 +0000 Subject: [PATCH 2/4] skip schema generation --- .github/workflows/deploy-dev.yml | 94 ++++++++++++++++---------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index bd3b82635..14d47cee9 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -124,61 +124,61 @@ jobs: pip install . wrangles.recipe tests/samples/generate-data.wrgl.yml - # ── 3. Generate and test schema ────────────────────────────────────────── - test-generate-schema: - name: Generate and Test Schema - runs-on: ubuntu-latest - needs: [compute-version] - permissions: - contents: read - steps: - - name: Checkout Repository - uses: actions/checkout@v5 + # # ── 3. Generate and test schema ────────────────────────────────────────── + # test-generate-schema: + # name: Generate and Test Schema + # runs-on: ubuntu-latest + # needs: [compute-version] + # permissions: + # contents: read + # steps: + # - name: Checkout Repository + # uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.13' + # - name: Set up Python + # uses: actions/setup-python@v6 + # with: + # python-version: '3.13' - - name: Install Dependencies - run: | - pip install -r requirements-full.txt - pip install jsonschema + # - name: Install Dependencies + # run: | + # pip install -r requirements-full.txt + # pip install jsonschema - - name: Generate and Test Schema - run: cd schema && python generate_recipe_schema.py + # - name: Generate and Test Schema + # run: cd schema && python generate_recipe_schema.py - - name: Save schema as schema_dev.json - run: cp schema/schema.json schema/schema_dev.json + # - name: Save schema as schema_dev.json + # run: cp schema/schema.json schema/schema_dev.json - - name: Checkout wrangleworks.github.io - uses: actions/checkout@v5 - with: - repository: wrangleworks/wrangleworks.github.io - token: ${{ secrets.CROSS_REPO_PAT_V2 }} - path: wrangleworks.github.io + # - name: Checkout wrangleworks.github.io + # uses: actions/checkout@v5 + # with: + # repository: wrangleworks/wrangleworks.github.io + # token: ${{ secrets.CROSS_REPO_PAT_V2 }} + # path: wrangleworks.github.io - - name: Copy schema_dev.json to wrangleworks.github.io - run: cp schema/schema_dev.json wrangleworks.github.io/schema/recipes/schema_dev.json + # - name: Copy schema_dev.json to wrangleworks.github.io + # run: cp schema/schema_dev.json wrangleworks.github.io/schema/recipes/schema_dev.json - - name: Commit and push schema_dev.json - working-directory: wrangleworks.github.io - run: | - git config user.name "mborodii-prog" - git config user.email "mborodii@binariks.com" - git add schema/recipes/schema_dev.json - if git diff --cached --quiet; then - echo "No changes to schema_dev.json, skipping commit." - else - git commit -m "Update dev recipe schema" - git push - fi + # - name: Commit and push schema_dev.json + # working-directory: wrangleworks.github.io + # run: | + # git config user.name "mborodii-prog" + # git config user.email "mborodii@binariks.com" + # git add schema/recipes/schema_dev.json + # if git diff --cached --quiet; then + # echo "No changes to schema_dev.json, skipping commit." + # else + # git commit -m "Update dev recipe schema" + # git push + # fi - - name: Save Schema as Artifact - uses: actions/upload-artifact@v6 - with: - name: schema - path: schema/schema_dev.json + # - name: Save Schema as Artifact + # uses: actions/upload-artifact@v6 + # with: + # name: schema + # path: schema/schema_dev.json # ── 4. Publish Python package to CodeArtifact ──────────────────────────── publish-codeartifact: From 2cd6214f03abd6094caa7bf2bf4c06b0de1fc8ed Mon Sep 17 00:00:00 2001 From: Mariia Borodii Date: Tue, 1 Sep 2026 13:33:39 +0000 Subject: [PATCH 3/4] skip schhema --- .github/workflows/deploy-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 14d47cee9..c1ec29ae9 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -184,7 +184,7 @@ jobs: publish-codeartifact: name: Publish RC Package runs-on: ubuntu-latest - needs: [compute-version, test-pip-install, test-generate-schema] + needs: [compute-version, test-pip-install] permissions: contents: read id-token: write # required for OIDC AssumeRoleWithWebIdentity From 41c887b0bab8316c9ab690665b1f3909d790a573 Mon Sep 17 00:00:00 2001 From: Mariia Borodii Date: Wed, 2 Sep 2026 06:57:03 +0000 Subject: [PATCH 4/4] Restore schema generation job in deploy-dev workflow --- .github/workflows/deploy-dev.yml | 96 ++++++++++++++++---------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index c1ec29ae9..bd3b82635 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -124,67 +124,67 @@ jobs: pip install . wrangles.recipe tests/samples/generate-data.wrgl.yml - # # ── 3. Generate and test schema ────────────────────────────────────────── - # test-generate-schema: - # name: Generate and Test Schema - # runs-on: ubuntu-latest - # needs: [compute-version] - # permissions: - # contents: read - # steps: - # - name: Checkout Repository - # uses: actions/checkout@v5 + # ── 3. Generate and test schema ────────────────────────────────────────── + test-generate-schema: + name: Generate and Test Schema + runs-on: ubuntu-latest + needs: [compute-version] + permissions: + contents: read + steps: + - name: Checkout Repository + uses: actions/checkout@v5 - # - name: Set up Python - # uses: actions/setup-python@v6 - # with: - # python-version: '3.13' + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' - # - name: Install Dependencies - # run: | - # pip install -r requirements-full.txt - # pip install jsonschema + - name: Install Dependencies + run: | + pip install -r requirements-full.txt + pip install jsonschema - # - name: Generate and Test Schema - # run: cd schema && python generate_recipe_schema.py + - name: Generate and Test Schema + run: cd schema && python generate_recipe_schema.py - # - name: Save schema as schema_dev.json - # run: cp schema/schema.json schema/schema_dev.json + - name: Save schema as schema_dev.json + run: cp schema/schema.json schema/schema_dev.json - # - name: Checkout wrangleworks.github.io - # uses: actions/checkout@v5 - # with: - # repository: wrangleworks/wrangleworks.github.io - # token: ${{ secrets.CROSS_REPO_PAT_V2 }} - # path: wrangleworks.github.io + - name: Checkout wrangleworks.github.io + uses: actions/checkout@v5 + with: + repository: wrangleworks/wrangleworks.github.io + token: ${{ secrets.CROSS_REPO_PAT_V2 }} + path: wrangleworks.github.io - # - name: Copy schema_dev.json to wrangleworks.github.io - # run: cp schema/schema_dev.json wrangleworks.github.io/schema/recipes/schema_dev.json + - name: Copy schema_dev.json to wrangleworks.github.io + run: cp schema/schema_dev.json wrangleworks.github.io/schema/recipes/schema_dev.json - # - name: Commit and push schema_dev.json - # working-directory: wrangleworks.github.io - # run: | - # git config user.name "mborodii-prog" - # git config user.email "mborodii@binariks.com" - # git add schema/recipes/schema_dev.json - # if git diff --cached --quiet; then - # echo "No changes to schema_dev.json, skipping commit." - # else - # git commit -m "Update dev recipe schema" - # git push - # fi + - name: Commit and push schema_dev.json + working-directory: wrangleworks.github.io + run: | + git config user.name "mborodii-prog" + git config user.email "mborodii@binariks.com" + git add schema/recipes/schema_dev.json + if git diff --cached --quiet; then + echo "No changes to schema_dev.json, skipping commit." + else + git commit -m "Update dev recipe schema" + git push + fi - # - name: Save Schema as Artifact - # uses: actions/upload-artifact@v6 - # with: - # name: schema - # path: schema/schema_dev.json + - name: Save Schema as Artifact + uses: actions/upload-artifact@v6 + with: + name: schema + path: schema/schema_dev.json # ── 4. Publish Python package to CodeArtifact ──────────────────────────── publish-codeartifact: name: Publish RC Package runs-on: ubuntu-latest - needs: [compute-version, test-pip-install] + needs: [compute-version, test-pip-install, test-generate-schema] permissions: contents: read id-token: write # required for OIDC AssumeRoleWithWebIdentity