config updates from json to yaml

.azure-pipelines/templates/configure_azureml_agent.yml

@@ -16,7 +16,7 @@ steps:
 
       python -m pip install --upgrade pip
       python -m pip install -r .azure-pipelines/requirements/execute_job_requirements.txt
-      python -m pip install promptflow promptflow-tools promptflow-sdk jinja2 promptflow[azure] openai promptflow-sdk[builtins]
+      python -m pip install promptflow promptflow-tools promptflow-sdk jinja2 promptflow[azure] openai promptflow-sdk[builtins] PyPDF2 faiss-cpu
 
       az version
 

.azure-pipelines/web_classification_pf_in_aml_pipeline_workflow.yml

@@ -0,0 +1,26 @@
+parameters:
+ - name: env_name
+   displayName: "Execution Environment"
+   default: "dev"
+ - name: use_case_base_path
+   displayName: "Base path of model to execute"
+   default: "web_classification"
+
+stages:
+    - stage: execute_training_job
+      displayName: execute_training_job
+      jobs:
+      - job: Execute_ml_Job_Pipeline
+        steps:
+        - template: templates/get_connection_details.yml
+
+        - template: templates/configure_azureml_agent.yml
+
+        - template: templates/execute_python_code.yml
+          parameters:
+            step_name: "Execute PF IN AML Pipeline"
+            script_parameter: |
+                  python -m pf_aml_pipeline.promptflow_in_aml_pipeline \
+                  --subscription_id "$(SUBSCRIPTION_ID)" \
+                  --env_name ${{ parameters.env_name }} \
+                  --base_path ${{ parameters.use_case_base_path }}

.devcontainer/devcontainer.json

@@ -0,0 +1,25 @@
+{
+    "name": "llmops_promptflow_container",
+    "image": "mcr.microsoft.com/azureml/promptflow/promptflow-runtime-stable",
+
+    "customizations": {
+		"vscode": {
+            "extensions": [
+		        "ms-python.python",
+            "ms-python.debugpy",
+		        "ms-toolsai.vscode-ai",
+		        "ms-toolsai.jupyter",
+		        "redhat.vscode-yaml",
+		        "prompt-flow.prompt-flow"
+	        ]
+        }
+    },
+
+    "features": {
+      "ghcr.io/devcontainers/features/azure-cli:1": {
+        "version": "latest"
+      }
+    },
+    "containerEnv": { "SUBSCRIPTION_ID": "${localEnv:SUBSCRIPTION_ID}" },
+    "postCreateCommand": "pip3 install -r /workspaces/llmops-promptflow-template/.devcontainer/requirements.txt"    
+  }

.devcontainer/requirements.txt

@@ -0,0 +1,14 @@
+promptflow-tools 
+promptflow-sdk 
+promptflow-evals
+promptflow-tracing
+jinja2 
+promptflow-azure
+openai 
+promptflow-sdk 
+python-dotenv
+azure-ai-ml
+azure-identity
+azure-keyvault-secrets
+keyrings.alt
+bs4

.env.sample

@@ -0,0 +1,9 @@
+
+# Azure ML configuration
+SUBSCRIPTION_ID=""
+RESOURCE_GROUP_NAME=""
+WORKSPACE_NAME=""
+AZURE_OPENAI_KEY=""
+AZURE_OPENAI_API_VERSION = ""
+AZURE_OPENAI_ENDPOINT=""
+experiment_name=""

.github/actions/configure_azureml_agent/action.yml

@@ -25,6 +25,6 @@ runs:
 
         python -m pip install --upgrade pip
         python -m pip install -r .github/requirements/execute_job_requirements.txt
-        python -m pip install promptflow promptflow-tools promptflow-sdk jinja2 promptflow[azure] openai promptflow-sdk[builtins]
+        python -m pip install promptflow promptflow-tools promptflow-sdk jinja2 promptflow[azure] openai promptflow-sdk[builtins] PyPDF2 faiss-cpu
 
         az version

.github/workflows/build_validation_workflow.yml

@@ -29,6 +29,11 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -r .github/requirements/build_validation_requirements.txt
           az version
+      - name: Export Secrets 
+        uses: oNaiPs/secrets-to-env-action@v1
+        with:
+          secrets: ${{ toJSON(secrets) }}
+          convert: upper
       - name: Azure login
         uses: azure/login@v1
         with:

.github/workflows/chat_with_pdf_ci_dev_workflow.yml

@@ -0,0 +1,44 @@
+name: chat_with_pdf_ci_dev_workflow
+
+on:
+  workflow_call:
+  workflow_dispatch:
+    inputs:
+      env_name:
+        type: string
+        description: "Execution Environment"
+        required: true
+        default: "dev"
+      use_case_base_path:
+        type: string
+        description: "The flow usecase to execute"
+        required: true
+        default: "chat_with_pdf"
+      deployment_type:
+        type: string
+        description: "Determine type of deployment - aml, aks, docker, webapp"
+        required: true
+  push:
+    branches:
+      - main
+      - development
+    paths:
+      - '.github/**'
+      - 'llmops/**'
+      - 'chat_with_pdf/**'
+
+
+#=====================================
+# Execute platform_ci_dev_workflow workflow for experiment, evaluation and deployment of flows
+#=====================================
+jobs:
+  execute-platform-flow-ci:
+    uses: ./.github/workflows/platform_ci_dev_workflow.yml
+    with:
+      env_name: ${{ inputs.env_name || 'dev'}}
+      use_case_base_path: ${{ inputs.use_case_base_path || 'chat_with_pdf' }}
+      deployment_type: ${{ inputs.deployment_type|| 'aml' }}
+    secrets:
+      azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
+      connection_details: ${{ secrets.COMMON_DEV_CONNECTIONS }}
+      registry_details: ${{ secrets.DOCKER_IMAGE_REGISTRY }}

.github/workflows/chat_with_pdf_pr_dev_workflow.yml

@@ -0,0 +1,35 @@
+name: chat_with_pdf_pr_dev_workflow
+
+on:
+  workflow_call:
+    inputs:
+      env_name:
+        type: string
+        description: "Execution Environment"
+        required: true
+        default: "dev"
+      use_case_base_path:
+        type: string
+        description: "The flow usecase to execute"
+        required: true
+        default: "chat_with_pdf"
+  pull_request:
+    branches:
+      - main
+      - development
+    paths:
+      - '.github/**'
+      - 'llmops/**'
+      - 'chat_with_pdf/**'
+
+#=====================================
+# Execute platform_pr_dev_workflow workflow for experiment, evaluation and deployment of flows
+#=====================================
+jobs:
+  execute-platform-pr-workflow:
+    uses: ./.github/workflows/platform_pr_dev_workflow.yml
+    with:
+      env_name: ${{ inputs.env_name || 'pr'}}
+      use_case_base_path: ${{ inputs.use_case_base_path || 'chat_with_pdf' }}
+    secrets:
+      azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}

.github/workflows/named_entity_recognition_data_aml_cd_workflow.yml

@@ -0,0 +1,114 @@
+name: named_entity_recognition_data_aml_pipeline
+on:
+  # workflow_call allows reusable workflow that can be called by other workflows
+  workflow_call:
+    inputs:
+      subscription_id:
+        description: Azure subscription id
+        type: string
+        required: true      
+      resource_group_name:
+        description: Azure resource group name
+        type: string
+        required: true
+      workspace_name:
+          description: Azure ML workspace name
+          type: string
+          required: true
+      aml_env_name:
+            description: Environment name
+            type: string
+            required: true
+      config_path_root_dir:
+              description: Root dir for config file
+              type: string
+              required: true
+              default: "named_entity_recognition"
+
+  # workflow_dispatch allows to run workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      subscription_id:
+        description: Azure subscription id
+        type: string
+        required: true      
+      resource_group_name:
+        description: Azure resource group name
+        type: string
+        required: true
+      workspace_name:
+          description: Azure ML workspace name
+          type: string
+          required: true
+      aml_env_name:
+            description: Environment name
+            type: string
+            required: true
+      config_path_root_dir:
+              description: Root dir for config file
+              type: string
+              required: true
+              default: "named_entity_recognition"
+
+jobs:
+  deploy_aml_data_pipeline:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout current repository
+        uses: actions/checkout@v3.3.0
+
+      - name: Set up python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Azure login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Configure Azure ML Agent
+        uses: ./.github/actions/configure_azureml_agent
+
+      - name: Load the current Azure subscription details
+        id: subscription_details
+        shell: bash
+        run: |
+          export subscriptionId=$(az account show --query id -o tsv)
+          echo "SUBSCRIPTION_ID=$subscriptionId" >> $GITHUB_OUTPUT
+
+      - name: Deploy data pipeline
+        uses: ./.github/actions/execute_script
+        with:
+          step_name: "Deploy data pipeline"
+          script_parameter: |
+            python -m dataops.common.aml_pipeline \
+            --subscription_id ${{ inputs.subscription_id }} \
+            --resource_group_name ${{ inputs.resource_group_name }} \
+            --workspace_name ${{ inputs.workspace_name }} \
+            --aml_env_name ${{ inputs.aml_env_name }} \
+            --config_path_root_dir ${{ inputs.config_path_root_dir }} 
+
+      - name: Create data store
+        uses: ./.github/actions/execute_script
+        with:
+          step_name: "Create data store"
+          script_parameter: |
+            python -m dataops.common.aml_data_store \
+            --subscription_id ${{ inputs.subscription_id }} \
+            --resource_group_name ${{ inputs.resource_group_name }} \
+            --workspace_name ${{ inputs.workspace_name }} \
+            --config_path_root_dir ${{ inputs.config_path_root_dir }} \
+            --sa_key ${{ secrets.SA_KEY }} 
+
+      - name: Register data asset
+        uses: ./.github/actions/execute_script
+        with:
+          step_name: "Register data asset"
+          script_parameter: |
+            python -m dataops.common.aml_data_asset \
+            --subscription_id ${{ inputs.subscription_id }} \
+            --resource_group_name ${{ inputs.resource_group_name }} \
+            --workspace_name ${{ inputs.workspace_name }} \
+            --config_path_root_dir ${{ inputs.config_path_root_dir }}      

.github/workflows/web_classification_pf_in_aml_pipeline_workflow.yml

@@ -0,0 +1,61 @@
+name: web_classification_pf_in_aml_pipeline_workflow.yml
+
+on:
+  workflow_call:
+    inputs:
+      env_name:
+        type: string
+        description: "Execution Environment"
+        required: true
+        default: "dev"
+      use_case_base_path:
+        type: string
+        description: "The base path of the flow use-case to execute"
+        required: true
+        default: "web_classification"
+    secrets:
+      azure_credentials:
+        description: "service principal authentication to Azure"
+        required: true
+jobs:
+  flow-experiment-and_evaluation:
+    name: prompt flow experiment and evaluation job in Azure ML
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.env_name }}
+    env:
+      RESOURCE_GROUP_NAME: ${{ vars.RESOURCE_GROUP_NAME }}
+      WORKSPACE_NAME: ${{ vars.WORKSPACE_NAME }}
+      COMPUTE_TARGET: ${{ vars.COMPUTE_TARGET }}
+    steps:
+      - name: Checkout Actions
+        uses: actions/checkout@v4
+
+      - name: Azure login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.azure_credentials }}
+
+      - name: Configure Azure ML Agent
+        uses: ./.github/actions/configure_azureml_agent
+
+      - name: load the current Azure subscription details
+        id: subscription_details
+        shell: bash
+        run: |
+          export subscriptionId=$(az account show --query id -o tsv)
+          echo "SUBSCRIPTION_ID=$subscriptionId" >> $GITHUB_OUTPUT
+
+        #=====================================
+        # Run Promptflow in AML Pipeline
+        #=====================================      
+      - name: Run Promptflow in AML Pipeline
+        uses: ./.github/actions/execute_script
+        with:
+            step_name: "Run Promptflow in AML Pipeline"
+            script_parameter: |
+                python -m pf_aml_pipeline.promptflow_in_aml_pipeline \
+                --subscription_id ${{ steps.subscription_details.outputs.SUBSCRIPTION_ID }} \
+                --env_name ${{ inputs.env_name || 'dev' }} \
+                --base_path ${{ inputs.use_case_base_path || 'web_classification'}} \
+