diff --git a/.changes/unreleased/added-20260401-135149.yaml b/.changes/unreleased/added-20260401-135149.yaml new file mode 100644 index 00000000..83cc5fa8 --- /dev/null +++ b/.changes/unreleased/added-20260401-135149.yaml @@ -0,0 +1,8 @@ +kind: added +body: Add feature flag for shortcut smart diff publishing +time: 2026-04-01T13:51:49.2071983+02:00 +custom: + Author: bckstrm + AuthorLink: https://github.com/bckstrm + Issue: "904" + IssueLink: https://github.com/microsoft/fabric-cicd/issues/904 diff --git a/src/fabric_cicd/_items/_lakehouse.py b/src/fabric_cicd/_items/_lakehouse.py index 772a2c0f..f33a9e06 100644 --- a/src/fabric_cicd/_items/_lakehouse.py +++ b/src/fabric_cicd/_items/_lakehouse.py @@ -3,6 +3,7 @@ """Functions to process and deploy Lakehouse item.""" +import copy import json import logging @@ -57,16 +58,26 @@ def check_sqlendpoint_provision_status(fabric_workspace_obj: FabricWorkspace, it iteration += 1 -def list_deployed_shortcuts(fabric_workspace_obj: FabricWorkspace, item_obj: Item) -> list: +def _get_shortcut_path(shortcut: dict) -> str: """ - Lists all deployed shortcut paths + Build a unique shortcut path from shortcut metadata. + + Args: + shortcut: The shortcut definition dictionary + """ + return f"{shortcut['path']}/{shortcut['name']}" + + +def list_deployed_shortcuts(fabric_workspace_obj: FabricWorkspace, item_obj: Item) -> dict[str, dict]: + """ + Lists all deployed shortcut by path Args: fabric_workspace_obj: The FabricWorkspace object containing the items to be published item_obj: The item object to list the shortcuts for """ request_url = f"{fabric_workspace_obj.base_api_url}/items/{item_obj.guid}/shortcuts" - deployed_shortcut_paths = [] + deployed_shortcuts_by_path = {} while request_url: # https://learn.microsoft.com/en-us/rest/api/fabric/core/onelake-shortcuts/list-shortcuts @@ -74,11 +85,12 @@ def list_deployed_shortcuts(fabric_workspace_obj: FabricWorkspace, item_obj: Ite # Handle cases where the response body is empty shortcuts = response["body"].get("value", []) - deployed_shortcut_paths.extend(f"{shortcut['path']}/{shortcut['name']}" for shortcut in shortcuts) + for shortcut in shortcuts: + deployed_shortcuts_by_path[_get_shortcut_path(shortcut)] = shortcut request_url = response["header"].get("continuationUri", None) - return deployed_shortcut_paths + return deployed_shortcuts_by_path def replace_default_lakehouse_id(shortcut: dict, item_obj: Item) -> dict: @@ -165,6 +177,43 @@ def _unpublish_shortcuts(self, shortcut_paths: list) -> None: url=f"{self.fabric_workspace_obj.base_api_url}/items/{self.item_obj.guid}/shortcuts/{deployed_shortcut_path}", ) + @staticmethod + def _is_shortcut_subset_match(repository_shortcut: object, deployed_shortcut: object) -> bool: + """ + Recursively compare shortcut fields while ignoring extra deployed fields. + + This allows server-side schema extensions without forcing unnecessary republish. + """ + if isinstance(repository_shortcut, dict): + if not isinstance(deployed_shortcut, dict): + return False + for key, repository_value in repository_shortcut.items(): + if key not in deployed_shortcut: + return False + if not ShortcutPublisher._is_shortcut_subset_match(repository_value, deployed_shortcut[key]): + return False + return True + + if isinstance(repository_shortcut, list): + if not isinstance(deployed_shortcut, list) or len(repository_shortcut) != len(deployed_shortcut): + return False + return all( + ShortcutPublisher._is_shortcut_subset_match(repository_item, deployed_item) + for repository_item, deployed_item in zip(repository_shortcut, deployed_shortcut) + ) + + return repository_shortcut == deployed_shortcut + + def _is_shortcut_changed(self, shortcut: dict, deployed_shortcut: dict) -> bool: + """ + Determine whether a shortcut needs publishing. + + Returns: + True when the shortcut differs from deployed state and should be published. + """ + shortcut_for_compare = replace_default_lakehouse_id(copy.deepcopy(shortcut), self.item_obj) + return not self._is_shortcut_subset_match(shortcut_for_compare, deployed_shortcut) + def publish_one(self, _shortcut_name: str, shortcut: dict) -> None: """ Publish a single shortcut. @@ -230,12 +279,29 @@ def publish_all(self) -> None: ) logger.info(f"{constants.INDENT}Excluded shortcuts: {excluded_shortcuts}") - shortcuts_to_publish = {f"{shortcut['path']}/{shortcut['name']}": shortcut for shortcut in shortcuts} + shortcuts_to_publish = {_get_shortcut_path(shortcut): shortcut for shortcut in shortcuts} if shortcuts_to_publish: logger.info(f"Publishing Lakehouse '{self.item_obj.name}' Shortcuts") - shortcut_paths_to_unpublish = [path for path in deployed_shortcuts if path not in shortcuts_to_publish] + shortcut_paths_to_unpublish = [ + path for path in list(deployed_shortcuts.keys()) if path not in shortcuts_to_publish + ] self._unpublish_shortcuts(shortcut_paths_to_unpublish) - # Deploy and overwrite shortcuts + if FeatureFlag.ENABLE_SHORTCUT_SMART_DIFF.value in constants.FEATURE_FLAG: + shortcuts_with_change = {} + unchanged_shortcut_count = 0 + for shortcut_path, shortcut in shortcuts_to_publish.items(): + deployed_shortcut = deployed_shortcuts.get(shortcut_path) + if deployed_shortcut and not self._is_shortcut_changed(shortcut, deployed_shortcut): + unchanged_shortcut_count += 1 + continue + shortcuts_with_change[shortcut_path] = shortcut + shortcuts_to_publish = shortcuts_with_change + if unchanged_shortcut_count: + logger.info( + f"{constants.INDENT}Skipped {unchanged_shortcut_count} unchanged shortcut(s) via smart diff" + ) + + # Deploy and overwrite changed/new shortcuts for shortcut_path, shortcut in shortcuts_to_publish.items(): self.publish_one(shortcut_path, shortcut) diff --git a/src/fabric_cicd/constants.py b/src/fabric_cicd/constants.py index c2424c64..54a53937 100644 --- a/src/fabric_cicd/constants.py +++ b/src/fabric_cicd/constants.py @@ -117,6 +117,8 @@ class FeatureFlag(str, Enum): """Set to enable the deletion of KQL Databases (attached to Eventhouses).""" ENABLE_SHORTCUT_PUBLISH = "enable_shortcut_publish" """Set to enable deploying shortcuts with the lakehouse.""" + ENABLE_SHORTCUT_SMART_DIFF = "enable_shortcut_smart_diff" + """Set to enable change comparision before deploying shortcuts""" DISABLE_WORKSPACE_FOLDER_PUBLISH = "disable_workspace_folder_publish" """Set to disable deploying workspace sub folders.""" CONTINUE_ON_SHORTCUT_FAILURE = "continue_on_shortcut_failure" diff --git a/tests/test_deploy_with_config.py b/tests/test_deploy_with_config.py index 26497464..a28e133b 100644 --- a/tests/test_deploy_with_config.py +++ b/tests/test_deploy_with_config.py @@ -267,6 +267,7 @@ def test_extract_parameter_file_path_missing(self): settings = extract_workspace_settings(config, "dev") assert "parameter_file_path" not in settings + class TestPublishSettingsExtraction: """Test publish settings extraction from config.""" @@ -1001,6 +1002,7 @@ def test_deploy_with_config_loads_parameter_when_field_present(self, tmp_path): call_kwargs = mock_fabric_ws.call_args[1] assert call_kwargs.get("skip_parameterization") is False + class TestConfigIntegration: """Integration tests for config functionality.""" diff --git a/tests/test_shortcut_exclude.py b/tests/test_shortcut_exclude.py index a21861c3..1bf2926b 100644 --- a/tests/test_shortcut_exclude.py +++ b/tests/test_shortcut_exclude.py @@ -8,6 +8,7 @@ import pytest +from fabric_cicd import constants from fabric_cicd._common._item import Item from fabric_cicd._items._lakehouse import ShortcutPublisher from fabric_cicd.fabric_workspace import FabricWorkspace @@ -56,6 +57,39 @@ def create_shortcut_file(shortcuts_data): return file_obj +def set_deployed_shortcuts(mock_fabric_workspace, deployed_shortcuts): + """Configure endpoint behavior to return specific deployed shortcuts.""" + + def mock_invoke(method, url, **_kwargs): + if method == "GET" and "shortcuts" in url: + return {"body": {"value": deployed_shortcuts}, "header": {}} + if method == "POST" and "shortcuts" in url: + return {"body": {"id": "mock-shortcut-id"}} + if method == "DELETE" and "shortcuts" in url: + return {"body": {}} + return {"body": {}} + + mock_fabric_workspace.endpoint.invoke.side_effect = mock_invoke + + +def get_shortcut_post_calls(mock_fabric_workspace): + """Return all shortcut create/overwrite API calls.""" + return [ + call + for call in mock_fabric_workspace.endpoint.invoke.call_args_list + if call[1].get("method") == "POST" and "shortcuts" in call[1].get("url", "") + ] + + +@pytest.fixture +def restore_feature_flags(): + """Ensure feature flags are restored after test.""" + original_flags = constants.FEATURE_FLAG.copy() + yield + constants.FEATURE_FLAG.clear() + constants.FEATURE_FLAG.update(original_flags) + + def test_process_shortcuts_with_exclude_regex_filters_shortcuts(mock_fabric_workspace, mock_item): """Test that shortcut_exclude_regex correctly filters shortcuts from deployment.""" @@ -304,3 +338,140 @@ def test_process_shortcuts_with_complex_regex_pattern(mock_fabric_workspace, moc # Verify the published shortcut is the prod one published_shortcut = post_calls[0][1]["body"] assert published_shortcut["name"] == "prod_shortcut" + + +def test_shortcut_smart_diff_skips_unchanged_shortcuts(mock_fabric_workspace, mock_item): + """Smart diff should skip publish for unchanged shortcuts.""" + shortcuts_data = [ + { + "name": "shortcut1", + "path": "/Tables", + "target": {"type": "OneLake", "oneLake": {"path": "Tables/s1", "itemId": "item-1"}}, + }, + { + "name": "shortcut2", + "path": "/Files", + "target": {"type": "OneLake", "oneLake": {"path": "Files/s2", "itemId": "item-2"}}, + }, + ] + deployed_shortcuts = [ + { + "name": "shortcut1", + "path": "/Tables", + "target": {"type": "OneLake", "oneLake": {"path": "Tables/s1", "itemId": "item-1"}}, + "serverManaged": {"createdBy": "system"}, + }, + { + "name": "shortcut2", + "path": "/Files", + "target": {"type": "OneLake", "oneLake": {"path": "Files/s2", "itemId": "item-2"}}, + "serverManaged": {"createdBy": "system"}, + }, + ] + + mock_item.item_files = [create_shortcut_file(shortcuts_data)] + set_deployed_shortcuts(mock_fabric_workspace, deployed_shortcuts) + constants.FEATURE_FLAG.add("enable_shortcut_smart_diff") + try: + ShortcutPublisher(mock_fabric_workspace, mock_item).publish_all() + post_calls = get_shortcut_post_calls(mock_fabric_workspace) + assert len(post_calls) == 0 + finally: + constants.FEATURE_FLAG.discard("enable_shortcut_smart_diff") + + +def test_shortcut_smart_diff_publishes_only_changed_shortcut(mock_fabric_workspace, mock_item): + """Smart diff should publish only shortcuts with changed fields.""" + shortcuts_data = [ + { + "name": "shortcut1", + "path": "/Tables", + "target": {"type": "OneLake", "oneLake": {"path": "Tables/s1", "itemId": "item-1"}}, + }, + { + "name": "shortcut2", + "path": "/Files", + "target": {"type": "OneLake", "oneLake": {"path": "Files/s2-new", "itemId": "item-2"}}, + }, + ] + deployed_shortcuts = [ + { + "name": "shortcut1", + "path": "/Tables", + "target": {"type": "OneLake", "oneLake": {"path": "Tables/s1", "itemId": "item-1"}}, + }, + { + "name": "shortcut2", + "path": "/Files", + "target": {"type": "OneLake", "oneLake": {"path": "Files/s2-old", "itemId": "item-2"}}, + }, + ] + + mock_item.item_files = [create_shortcut_file(shortcuts_data)] + set_deployed_shortcuts(mock_fabric_workspace, deployed_shortcuts) + constants.FEATURE_FLAG.add("enable_shortcut_smart_diff") + try: + ShortcutPublisher(mock_fabric_workspace, mock_item).publish_all() + + post_calls = get_shortcut_post_calls(mock_fabric_workspace) + assert len(post_calls) == 1 + assert post_calls[0][1]["body"]["name"] == "shortcut2" + finally: + constants.FEATURE_FLAG.discard("enable_shortcut_smart_diff") + + +def test_shortcut_smart_diff_publishes_client_side_new_field(mock_fabric_workspace, mock_item): + """Smart diff should publish when repo contains a new field missing from deployed shortcut.""" + shortcuts_data = [ + { + "name": "shortcut1", + "path": "/Tables", + "target": {"type": "OneLake", "oneLake": {"path": "Tables/s1", "itemId": "item-1"}}, + "clientField": {"owner": "team-a"}, + } + ] + deployed_shortcuts = [ + { + "name": "shortcut1", + "path": "/Tables", + "target": {"type": "OneLake", "oneLake": {"path": "Tables/s1", "itemId": "item-1"}}, + } + ] + + mock_item.item_files = [create_shortcut_file(shortcuts_data)] + set_deployed_shortcuts(mock_fabric_workspace, deployed_shortcuts) + constants.FEATURE_FLAG.add("enable_shortcut_smart_diff") + try: + ShortcutPublisher(mock_fabric_workspace, mock_item).publish_all() + + post_calls = get_shortcut_post_calls(mock_fabric_workspace) + assert len(post_calls) == 1 + assert post_calls[0][1]["body"]["name"] == "shortcut1" + finally: + constants.FEATURE_FLAG.discard("enable_shortcut_smart_diff") + + +def test_shortcut_smart_diff_disabled_keeps_publish_all_behavior(mock_fabric_workspace, mock_item): + """When smart diff is disabled, unchanged shortcuts are still published.""" + shortcuts_data = [ + { + "name": "shortcut1", + "path": "/Tables", + "target": {"type": "OneLake", "oneLake": {"path": "Tables/s1", "itemId": "item-1"}}, + } + ] + deployed_shortcuts = [ + { + "name": "shortcut1", + "path": "/Tables", + "target": {"type": "OneLake", "oneLake": {"path": "Tables/s1", "itemId": "item-1"}}, + } + ] + + mock_item.item_files = [create_shortcut_file(shortcuts_data)] + set_deployed_shortcuts(mock_fabric_workspace, deployed_shortcuts) + + ShortcutPublisher(mock_fabric_workspace, mock_item).publish_all() + + post_calls = get_shortcut_post_calls(mock_fabric_workspace) + assert len(post_calls) == 1