diff --git a/.azure-pipelines/templates/configure_azureml_agent.yml b/.azure-pipelines/templates/configure_azureml_agent.yml index 6bc9cda96..5be2c74f0 100644 --- a/.azure-pipelines/templates/configure_azureml_agent.yml +++ b/.azure-pipelines/templates/configure_azureml_agent.yml @@ -16,7 +16,7 @@ steps: python -m pip install --upgrade pip python -m pip install -r .azure-pipelines/requirements/execute_job_requirements.txt - python -m pip install promptflow promptflow-tools promptflow-sdk jinja2 promptflow[azure] openai promptflow-sdk[builtins] + python -m pip install promptflow promptflow-tools promptflow-sdk jinja2 promptflow[azure] openai promptflow-sdk[builtins] PyPDF2 faiss-cpu az version diff --git a/.azure-pipelines/web_classification_pf_in_aml_pipeline_workflow.yml b/.azure-pipelines/web_classification_pf_in_aml_pipeline_workflow.yml new file mode 100644 index 000000000..eb201c3a8 --- /dev/null +++ b/.azure-pipelines/web_classification_pf_in_aml_pipeline_workflow.yml @@ -0,0 +1,26 @@ +parameters: + - name: env_name + displayName: "Execution Environment" + default: "dev" + - name: use_case_base_path + displayName: "Base path of model to execute" + default: "web_classification" + +stages: + - stage: execute_training_job + displayName: execute_training_job + jobs: + - job: Execute_ml_Job_Pipeline + steps: + - template: templates/get_connection_details.yml + + - template: templates/configure_azureml_agent.yml + + - template: templates/execute_python_code.yml + parameters: + step_name: "Execute PF IN AML Pipeline" + script_parameter: | + python -m pf_aml_pipeline.promptflow_in_aml_pipeline \ + --subscription_id "$(SUBSCRIPTION_ID)" \ + --env_name ${{ parameters.env_name }} \ + --base_path ${{ parameters.use_case_base_path }} diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..cad34b099 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,25 @@ +{ + "name": "llmops_promptflow_container", + "image": "mcr.microsoft.com/azureml/promptflow/promptflow-runtime-stable", + + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.debugpy", + "ms-toolsai.vscode-ai", + "ms-toolsai.jupyter", + "redhat.vscode-yaml", + "prompt-flow.prompt-flow" + ] + } + }, + + "features": { + "ghcr.io/devcontainers/features/azure-cli:1": { + "version": "latest" + } + }, + "containerEnv": { "SUBSCRIPTION_ID": "${localEnv:SUBSCRIPTION_ID}" }, + "postCreateCommand": "pip3 install -r /workspaces/llmops-promptflow-template/.devcontainer/requirements.txt" + } \ No newline at end of file diff --git a/.devcontainer/requirements.txt b/.devcontainer/requirements.txt new file mode 100644 index 000000000..b01747437 --- /dev/null +++ b/.devcontainer/requirements.txt @@ -0,0 +1,14 @@ +promptflow-tools +promptflow-sdk +promptflow-evals +promptflow-tracing +jinja2 +promptflow-azure +openai +promptflow-sdk +python-dotenv +azure-ai-ml +azure-identity +azure-keyvault-secrets +keyrings.alt +bs4 \ No newline at end of file diff --git a/.env.sample b/.env.sample new file mode 100644 index 000000000..212306d40 --- /dev/null +++ b/.env.sample @@ -0,0 +1,9 @@ + +# Azure ML configuration +SUBSCRIPTION_ID="" +RESOURCE_GROUP_NAME="" +WORKSPACE_NAME="" +AZURE_OPENAI_KEY="" +AZURE_OPENAI_API_VERSION = "" +AZURE_OPENAI_ENDPOINT="" +experiment_name="" diff --git a/.github/actions/configure_azureml_agent/action.yml b/.github/actions/configure_azureml_agent/action.yml index 5b3e42cff..558a71df6 100644 --- a/.github/actions/configure_azureml_agent/action.yml +++ b/.github/actions/configure_azureml_agent/action.yml @@ -25,6 +25,6 @@ runs: python -m pip install --upgrade pip python -m pip install -r .github/requirements/execute_job_requirements.txt - python -m pip install promptflow promptflow-tools promptflow-sdk jinja2 promptflow[azure] openai promptflow-sdk[builtins] + python -m pip install promptflow promptflow-tools promptflow-sdk jinja2 promptflow[azure] openai promptflow-sdk[builtins] PyPDF2 faiss-cpu az version \ No newline at end of file diff --git a/.github/workflows/build_validation_workflow.yml b/.github/workflows/build_validation_workflow.yml index 60c58a3f4..c36dad468 100644 --- a/.github/workflows/build_validation_workflow.yml +++ b/.github/workflows/build_validation_workflow.yml @@ -29,6 +29,11 @@ jobs: python -m pip install --upgrade pip python -m pip install -r .github/requirements/build_validation_requirements.txt az version + - name: Export Secrets + uses: oNaiPs/secrets-to-env-action@v1 + with: + secrets: ${{ toJSON(secrets) }} + convert: upper - name: Azure login uses: azure/login@v1 with: diff --git a/.github/workflows/chat_with_pdf_ci_dev_workflow.yml b/.github/workflows/chat_with_pdf_ci_dev_workflow.yml new file mode 100644 index 000000000..f6770bc79 --- /dev/null +++ b/.github/workflows/chat_with_pdf_ci_dev_workflow.yml @@ -0,0 +1,44 @@ +name: chat_with_pdf_ci_dev_workflow + +on: + workflow_call: + workflow_dispatch: + inputs: + env_name: + type: string + description: "Execution Environment" + required: true + default: "dev" + use_case_base_path: + type: string + description: "The flow usecase to execute" + required: true + default: "chat_with_pdf" + deployment_type: + type: string + description: "Determine type of deployment - aml, aks, docker, webapp" + required: true + push: + branches: + - main + - development + paths: + - '.github/**' + - 'llmops/**' + - 'chat_with_pdf/**' + + +#===================================== +# Execute platform_ci_dev_workflow workflow for experiment, evaluation and deployment of flows +#===================================== +jobs: + execute-platform-flow-ci: + uses: ./.github/workflows/platform_ci_dev_workflow.yml + with: + env_name: ${{ inputs.env_name || 'dev'}} + use_case_base_path: ${{ inputs.use_case_base_path || 'chat_with_pdf' }} + deployment_type: ${{ inputs.deployment_type|| 'aml' }} + secrets: + azure_credentials: ${{ secrets.AZURE_CREDENTIALS }} + connection_details: ${{ secrets.COMMON_DEV_CONNECTIONS }} + registry_details: ${{ secrets.DOCKER_IMAGE_REGISTRY }} diff --git a/.github/workflows/chat_with_pdf_pr_dev_workflow.yml b/.github/workflows/chat_with_pdf_pr_dev_workflow.yml new file mode 100644 index 000000000..5935a9b50 --- /dev/null +++ b/.github/workflows/chat_with_pdf_pr_dev_workflow.yml @@ -0,0 +1,35 @@ +name: chat_with_pdf_pr_dev_workflow + +on: + workflow_call: + inputs: + env_name: + type: string + description: "Execution Environment" + required: true + default: "dev" + use_case_base_path: + type: string + description: "The flow usecase to execute" + required: true + default: "chat_with_pdf" + pull_request: + branches: + - main + - development + paths: + - '.github/**' + - 'llmops/**' + - 'chat_with_pdf/**' + +#===================================== +# Execute platform_pr_dev_workflow workflow for experiment, evaluation and deployment of flows +#===================================== +jobs: + execute-platform-pr-workflow: + uses: ./.github/workflows/platform_pr_dev_workflow.yml + with: + env_name: ${{ inputs.env_name || 'pr'}} + use_case_base_path: ${{ inputs.use_case_base_path || 'chat_with_pdf' }} + secrets: + azure_credentials: ${{ secrets.AZURE_CREDENTIALS }} diff --git a/.github/workflows/named_entity_recognition_data_aml_cd_workflow.yml b/.github/workflows/named_entity_recognition_data_aml_cd_workflow.yml new file mode 100644 index 000000000..638823f00 --- /dev/null +++ b/.github/workflows/named_entity_recognition_data_aml_cd_workflow.yml @@ -0,0 +1,114 @@ +name: named_entity_recognition_data_aml_pipeline +on: + # workflow_call allows reusable workflow that can be called by other workflows + workflow_call: + inputs: + subscription_id: + description: Azure subscription id + type: string + required: true + resource_group_name: + description: Azure resource group name + type: string + required: true + workspace_name: + description: Azure ML workspace name + type: string + required: true + aml_env_name: + description: Environment name + type: string + required: true + config_path_root_dir: + description: Root dir for config file + type: string + required: true + default: "named_entity_recognition" + + # workflow_dispatch allows to run workflow manually from the Actions tab + workflow_dispatch: + inputs: + subscription_id: + description: Azure subscription id + type: string + required: true + resource_group_name: + description: Azure resource group name + type: string + required: true + workspace_name: + description: Azure ML workspace name + type: string + required: true + aml_env_name: + description: Environment name + type: string + required: true + config_path_root_dir: + description: Root dir for config file + type: string + required: true + default: "named_entity_recognition" + +jobs: + deploy_aml_data_pipeline: + runs-on: ubuntu-latest + + steps: + - name: Checkout current repository + uses: actions/checkout@v3.3.0 + + - name: Set up python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Azure login + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Configure Azure ML Agent + uses: ./.github/actions/configure_azureml_agent + + - name: Load the current Azure subscription details + id: subscription_details + shell: bash + run: | + export subscriptionId=$(az account show --query id -o tsv) + echo "SUBSCRIPTION_ID=$subscriptionId" >> $GITHUB_OUTPUT + + - name: Deploy data pipeline + uses: ./.github/actions/execute_script + with: + step_name: "Deploy data pipeline" + script_parameter: | + python -m dataops.common.aml_pipeline \ + --subscription_id ${{ inputs.subscription_id }} \ + --resource_group_name ${{ inputs.resource_group_name }} \ + --workspace_name ${{ inputs.workspace_name }} \ + --aml_env_name ${{ inputs.aml_env_name }} \ + --config_path_root_dir ${{ inputs.config_path_root_dir }} + + - name: Create data store + uses: ./.github/actions/execute_script + with: + step_name: "Create data store" + script_parameter: | + python -m dataops.common.aml_data_store \ + --subscription_id ${{ inputs.subscription_id }} \ + --resource_group_name ${{ inputs.resource_group_name }} \ + --workspace_name ${{ inputs.workspace_name }} \ + --config_path_root_dir ${{ inputs.config_path_root_dir }} \ + --sa_key ${{ secrets.SA_KEY }} + + - name: Register data asset + uses: ./.github/actions/execute_script + with: + step_name: "Register data asset" + script_parameter: | + python -m dataops.common.aml_data_asset \ + --subscription_id ${{ inputs.subscription_id }} \ + --resource_group_name ${{ inputs.resource_group_name }} \ + --workspace_name ${{ inputs.workspace_name }} \ + --config_path_root_dir ${{ inputs.config_path_root_dir }} \ No newline at end of file diff --git a/.github/workflows/web_classification_pf_in_aml_pipeline_workflow.yml b/.github/workflows/web_classification_pf_in_aml_pipeline_workflow.yml new file mode 100644 index 000000000..5e74eac4f --- /dev/null +++ b/.github/workflows/web_classification_pf_in_aml_pipeline_workflow.yml @@ -0,0 +1,61 @@ +name: web_classification_pf_in_aml_pipeline_workflow.yml + +on: + workflow_call: + inputs: + env_name: + type: string + description: "Execution Environment" + required: true + default: "dev" + use_case_base_path: + type: string + description: "The base path of the flow use-case to execute" + required: true + default: "web_classification" + secrets: + azure_credentials: + description: "service principal authentication to Azure" + required: true +jobs: + flow-experiment-and_evaluation: + name: prompt flow experiment and evaluation job in Azure ML + runs-on: ubuntu-latest + environment: + name: ${{ inputs.env_name }} + env: + RESOURCE_GROUP_NAME: ${{ vars.RESOURCE_GROUP_NAME }} + WORKSPACE_NAME: ${{ vars.WORKSPACE_NAME }} + COMPUTE_TARGET: ${{ vars.COMPUTE_TARGET }} + steps: + - name: Checkout Actions + uses: actions/checkout@v4 + + - name: Azure login + uses: azure/login@v1 + with: + creds: ${{ secrets.azure_credentials }} + + - name: Configure Azure ML Agent + uses: ./.github/actions/configure_azureml_agent + + - name: load the current Azure subscription details + id: subscription_details + shell: bash + run: | + export subscriptionId=$(az account show --query id -o tsv) + echo "SUBSCRIPTION_ID=$subscriptionId" >> $GITHUB_OUTPUT + + #===================================== + # Run Promptflow in AML Pipeline + #===================================== + - name: Run Promptflow in AML Pipeline + uses: ./.github/actions/execute_script + with: + step_name: "Run Promptflow in AML Pipeline" + script_parameter: | + python -m pf_aml_pipeline.promptflow_in_aml_pipeline \ + --subscription_id ${{ steps.subscription_details.outputs.SUBSCRIPTION_ID }} \ + --env_name ${{ inputs.env_name || 'dev' }} \ + --base_path ${{ inputs.use_case_base_path || 'web_classification'}} \ + diff --git a/README.md b/README.md index 187cc8a4b..fe8fa923f 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ As LLMs rapidly evolve, the importance of Prompt Engineering becomes increasingl - LLM-infused applications are designed to understand and generate human-like text based on the input they receive. They comprise of prompts that need engineering cadence and rigour. - Prompt flow is a powerful feature that simplifies and streamlines the Prompt Engineering process for LLM-infused applications. It enables users to create, evaluate, and deploy high-quality flows with ease and efficiency. - How do we best augment LLM-infused applications with LLMOps and engineering rigour? This template aims to assist in the development of those types of applications using Prompt flow and LLMOps. +- Bringing discipline to the data preparation for LLM app development by following DataOps best practices. # Solution @@ -57,6 +58,7 @@ Each use case (set of Prompt flow standard and evaluation flows) should follow t - environment : It contains a dockerfile used for running containers with flows for inferencing on Azure webapps. - flows : It should contain minimally two folder - one for standard Prompt flow related files and another for Evaluation flow related file. There can be multiple evaluation flow related folders. - tests : contains unit tests for the flows +- data-pipelines : It contains the data pipelines to generate the datasets (experimentation, evaluation etc.) necessary for the flows. This folder will have sub-folders specific to the data engineering tool - Microsoft Fabric, Azure ML etc. Additionally, there is a `experiment.yaml` file that configures the use-case (see file [description](./docs/the_experiment_file.md) and [specs](./docs/experiment.yaml) for more details). There is also a sample-request.json file containing test data for testing endpoints after deployment. @@ -70,6 +72,8 @@ Additionally, there is a `experiment.yaml` file that configures the use-case (se - The 'llmops' folder contains all the code related to flow execution, evaluation and deployment. +- The 'dataops' folder contains all the code related to data pipeline deployment. + - The 'local_execution' folder contains python scripts for executing both the standard and evaluation flow locally. # Documentation @@ -113,13 +117,19 @@ To harness the capabilities of the **local execution**, follow these installatio git clone https://github.com/microsoft/llmops-promptflow-template.git ``` -2. **setup env file**: create .env file at top folder level and provide information for items mentioned. Add as many connection names as needed. All the flow examples in this repo uses AzureOpenAI connection named `aoai`. Add a line `aoai={"api_key": "","api_base": "","api_type": "azure","api_version": "2023-03-15-preview"}` with updated values for api_key and api_base. If additional connections with different names are used in your flows, they should be added accordingly. Currently, flow with AzureOpenAI as provider as supported. +2. **setup env file**: create .env file at top folder level and provide information for items mentioned. Add as many connection names as needed. + +All the flow examples in this repo uses AzureOpenAI connection named `aoai`. connections are configured in llmops_config.yaml. If additional connections with different names are used in your flows, they should be added accordingly. Currently, flow with AzureOpenAI as provider as supported. ```bash -experiment_name= -connection_name_1={ "api_key": "","api_base": "","api_type": "azure","api_version": "2023-03-15-preview"} -connection_name_2={ "api_key": "","api_base": "","api_type": "azure","api_version": "2023-03-15-preview"} +SUBSCRIPTION_ID="" +RESOURCE_GROUP_NAME="" +WORKSPACE_NAME="" +AZURE_OPENAI_KEY="" +AZURE_OPENAI_API_VERSION="" +AZURE_OPENAI_ENDPOINT="" +experiment_name="" ``` 3. Prepare the local conda or virtual environment to install the dependencies. @@ -133,6 +143,14 @@ python -m pip install promptflow promptflow-tools promptflow-sdk jinja2 promptfl 5. Write python scripts similar to the provided examples in local_execution folder. +# DataOps + +DataOps combines aspects of DevOps, agile methodologies, and data management practices to streamline the process of collecting, processing, and analyzing data. DataOps can help to bring discipline in building the datasets (training, experimentation, evaluation etc.) necessary for LLM app development. + +The data pipelines are kept seperate from the prompt engineering flows. Data pipelines create the datasets and the datasets are registered as data assets in Azure ML for the flows to consume. This approach helps to scale and troubleshoot independently different parts of the system. + +For details on how to get started with DataOps, please follow this document - [How to Configure DataOps](./docs/how_to_configure_dataops.md). + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/chat_with_pdf/.azure-pipelines/chat_with_pdf_ci_dev_pipeline.yml b/chat_with_pdf/.azure-pipelines/chat_with_pdf_ci_dev_pipeline.yml new file mode 100644 index 000000000..d313d2c4f --- /dev/null +++ b/chat_with_pdf/.azure-pipelines/chat_with_pdf_ci_dev_pipeline.yml @@ -0,0 +1,45 @@ +pr: none +trigger: + branches: + include: + - main + - development + paths: + include: + - .azure-pipelines/* + - llmops/* + - chat_with_pdf/* + - environment/* + + +pool: + vmImage: ubuntu-latest + +variables: +- group: llmops_platform_dev_vg + +parameters: + - name: env_name + displayName: "Execution Environment" + default: "dev" + - name: use_case_base_path + displayName: "flow to execute" + default: "chat_with_pdf" + - name: deployment_type + displayName: "Determine type of deployment - aml, aks, docker, webapp" + default: "aml" + +#===================================== +# Execute platform_ci_dev_pipeline pipeline for experiment, evaluation and deployment of flows +#===================================== +stages: + - template: ../../.azure-pipelines/platform_ci_dev_pipeline.yml + parameters: + RESOURCE_GROUP_NAME: $(rg_name) # Expected in llmops_platform_dev_vg + WORKSPACE_NAME: $(ws_name) # Expected in llmops_platform_dev_vg + KEY_VAULT_NAME: $(kv_name) # Expected in llmops_platform_dev_vg + exec_environment: ${{ parameters.env_name }} + use_case_base_path: ${{ parameters.use_case_base_path }} + deployment_type: ${{ lower(parameters.deployment_type) }} + connection_details: '$(COMMON_DEV_CONNECTIONS)' + registry_details: '$(DOCKER_IMAGE_REGISTRY)' diff --git a/chat_with_pdf/.azure-pipelines/chat_with_pdf_pr_dev_pipeline.yml b/chat_with_pdf/.azure-pipelines/chat_with_pdf_pr_dev_pipeline.yml new file mode 100644 index 000000000..3993a8595 --- /dev/null +++ b/chat_with_pdf/.azure-pipelines/chat_with_pdf_pr_dev_pipeline.yml @@ -0,0 +1,36 @@ +trigger: none +pr: + branches: + include: + - main + - development + paths: + include: + - .azure-pipelines/* + - llmops/* + - chat_with_pdf/* + +pool: + vmImage: ubuntu-latest + +variables: +- group: llmops_platform_dev_vg + +parameters: + - name: env_name + displayName: "Execution Environment" + default: "pr" + - name: use_case_base_path + displayName: "flow to execute" + default: "chat_with_pdf" + +#===================================== +# Execute platform_pr_dev_pipeline pipeline for experiment, evaluation and deployment of flows +#===================================== +stages: + - template: ../../.azure-pipelines/platform_pr_dev_pipeline.yml + parameters: + RESOURCE_GROUP_NAME: $(rg_name) # Expected in llmops_platform_dev_vg + WORKSPACE_NAME: $(ws_name) # Expected in llmops_platform_dev_vg + exec_environment: ${{ parameters.env_name }} + use_case_base_path: ${{ parameters.use_case_base_path }} \ No newline at end of file diff --git a/chat_with_pdf/data/bert-paper-qna-1-line.jsonl b/chat_with_pdf/data/bert-paper-qna-1-line.jsonl new file mode 100644 index 000000000..0993b5867 --- /dev/null +++ b/chat_with_pdf/data/bert-paper-qna-1-line.jsonl @@ -0,0 +1 @@ +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf","chat_history":[],"question":"What is the name of the new language representation model introduced in the document?","answer":"BERT","context":"We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers.","config":{"EMBEDDING_MODEL_DEPLOYMENT_NAME":"text-embedding-ada-002","CHAT_MODEL_DEPLOYMENT_NAME":"aoai","PROMPT_TOKEN_LIMIT":2000,"MAX_COMPLETION_TOKENS":256,"CHUNK_SIZE":1024,"CHUNK_OVERLAP":64}} diff --git a/chat_with_pdf/data/bert-paper-qna-3-line.jsonl b/chat_with_pdf/data/bert-paper-qna-3-line.jsonl new file mode 100644 index 000000000..8f7f63e3f --- /dev/null +++ b/chat_with_pdf/data/bert-paper-qna-3-line.jsonl @@ -0,0 +1,3 @@ +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "What is the main difference between BERT and previous language representation models?", "groundtruth": "BERT is designed to pretrain deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers.", "context": "Unlike recent language representation models (Peters et al., 2018a; Radford et al., 2018), BERT is designed to pretrain deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers.", "config":{"EMBEDDING_MODEL_DEPLOYMENT_NAME":"text-embedding-ada-002","CHAT_MODEL_DEPLOYMENT_NAME":"aoai","PROMPT_TOKEN_LIMIT":2000,"MAX_COMPLETION_TOKENS":256,"CHUNK_SIZE":1024,"CHUNK_OVERLAP":64}} +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "What is the size of the vocabulary used by BERT?", "groundtruth": "30,000", "context": "We use WordPiece embeddings (Wu et al., 2016) with a 30,000 token vocabulary.", "config":{"EMBEDDING_MODEL_DEPLOYMENT_NAME":"text-embedding-ada-002","CHAT_MODEL_DEPLOYMENT_NAME":"aoai","PROMPT_TOKEN_LIMIT":2000,"MAX_COMPLETION_TOKENS":256,"CHUNK_SIZE":1024,"CHUNK_OVERLAP":64}} +{"pdf_url":"https://grs.pku.edu.cn/docs/2018-03/20180301083100898652.pdf", "chat_history":[], "question": "论文写作中论文引言有什么注意事项？", "groundtruth":"", "context":"", "config":{"EMBEDDING_MODEL_DEPLOYMENT_NAME":"text-embedding-ada-002","CHAT_MODEL_DEPLOYMENT_NAME":"aoai","PROMPT_TOKEN_LIMIT":2000,"MAX_COMPLETION_TOKENS":256,"CHUNK_SIZE":1024,"CHUNK_OVERLAP":64}} \ No newline at end of file diff --git a/chat_with_pdf/data/bert-paper-qna.jsonl b/chat_with_pdf/data/bert-paper-qna.jsonl new file mode 100644 index 000000000..32b107711 --- /dev/null +++ b/chat_with_pdf/data/bert-paper-qna.jsonl @@ -0,0 +1,11 @@ +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "What is the name of the new language representation model introduced in the document?", "answer": "BERT", "context": "We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers."} +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "What is the main difference between BERT and previous language representation models?", "answer": "BERT is designed to pretrain deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers.", "context": "Unlike recent language representation models (Peters et al., 2018a; Radford et al., 2018), BERT is designed to pretrain deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers."} +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "What is the advantage of fine-tuning BERT over using feature-based approaches?", "answer": "Fine-tuning BERT reduces the need for many heavily-engineered taskspecific architectures and transfers all parameters to initialize end-task model parameters.", "context": "We show that pre-trained representations reduce the need for many heavily-engineered taskspecific architectures. BERT is the first finetuning based representation model that achieves state-of-the-art performance on a large suite of sentence-level and token-level tasks, outperforming many task-specific architectures."} +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "What are the two unsupervised tasks used to pre-train BERT?", "answer": "Masked LM and next sentence prediction", "context": "In order to train a deep bidirectional representation, we simply mask some percentage of the input tokens at random, and then predict those masked tokens. We refer to this procedure as a \"masked LM\" (MLM), although it is often referred to as a Cloze task in the literature (Taylor, 1953). In addition to the masked language model, we also use a \"next sentence prediction\" task that jointly pretrains text-pair representations."} +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "How does BERT handle single sentence and sentence pair inputs?", "answer": "It uses a special classification token ([CLS]) at the beginning of every input sequence and a special separator token ([SEP]) to separate sentences or mark the end of a sequence.", "context": "To make BERT handle a variety of down-stream tasks, our input representation is able to unambiguously represent both a single sentence and a pair of sentences (e.g., h Question, Answeri) in one token sequence. The first token of every sequence is always a special classification token ([CLS]). The final hidden state corresponding to this token is used as the aggregate sequence representation for classification tasks. Sentence pairs are packed together into a single sequence. We differentiate the sentences in two ways. First, we separate them with a special token ([SEP]). Second, we add a learned embedding to every token indicating whether it belongs to sentence A or sentence B."} +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "What are the three types of embeddings used to construct the input representation for BERT?", "answer": "Token embeddings, segment embeddings and position embeddings", "context": "For a given token, its input representation is constructed by summing the corresponding token, segment, and position embeddings. A visualization of this construction can be seen in Figure 2."} +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "What is the size of the vocabulary used by BERT?", "answer": "30,000", "context": "We use WordPiece embeddings (Wu et al., 2016) with a 30,000 token vocabulary."} +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "What are the two model sizes reported in the paper for BERT?", "answer": "BERTBASE (L=12, H=768, A=12, Total Parameters=110M) and BERTLARGE (L=24, H=1024, A=16, Total Parameters=340M)", "context": "We primarily report results on two model sizes: BERTBASE (L=12, H=768, A=12, Total Parameters=110M) and BERTLARGE (L=24, H=1024, A=16, Total Parameters=340M)."} +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "How does BERT predict the start and end positions of an answer span in SQuAD?", "answer": "It uses two vectors S and E whose dot products with the final hidden vectors of each token denote scores for start and end positions.", "context": "We only introduce a start vector S ∈ R H and an end vector E ∈ R H during fine-tuning. The probability of word i being the start of the answer span is computed as a dot product between Ti and S followed by a softmax over all of the words in the paragraph: Pi = e S·Ti P j e S·Tj . The analogous formula is used for the end of the answer span."} +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "What is the main benefit of using a masked language model over a standard left-to-right or right-to-left language model?", "answer": "It enables the representation to fuse the left and the right context, which allows to pretrain a deep bidirectional Transformer.", "context": "Unlike left-to-right language model pre-training, the MLM objective enables the representation to fuse the left and the right context, which allows us to pretrain a deep bidirectional Transformer."} +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "How much does GPT4 API cost?", "answer": "I don't know"} \ No newline at end of file diff --git a/chat_with_pdf/data/data.jsonl b/chat_with_pdf/data/data.jsonl new file mode 100644 index 000000000..02f9fac52 --- /dev/null +++ b/chat_with_pdf/data/data.jsonl @@ -0,0 +1 @@ +{"question": "What is the purpose of the LLM Grounding Score, and what does a higher score mean in this context?", "groundtruth": "The LLM Grounding Score is a metric used in the context of in-context learning with large-scale pretrained language models (LLMs) [doc1]. It measures the ability of the LLM to understand and connect with the provided context or demonstrations during the learning process.\n\nA higher LLM Grounding Score indicates that the LLM has a better understanding and connection with the context or demonstrations [doc1]. It means that the LLM is able to effectively utilize the provided examples and generate more accurate and relevant responses based on that context.\n\nPlease note that the LLM Grounding Score is specific to the in-context learning framework and may not be applicable in other contexts or tasks.\n\n[doc1]: In-Context Learning with Large-Scale Pretrained Language Models: How Far Are We? (2022) - zelin, English.", "documents": [{"[doc1]":{"title": "In-Context Learning with Large-Scale Pretrained Language Models", "content": "In-Context Learning with Large-Scale Pretrained Language Models\nConcepts\nFew-Shot Learning : the model learns a new task from a small amount of training examples.\nIn-Context Learning : large-scale pretrained language models learn a new task simply by conditioning on a few training examples and predicting which tokens best complete a test input. In-context learning is entirely different from few-shot learning: the language models does receive only a few training examples, but the overall system may still require a large number of training examples.\nFew-shot learning with Large Language Models (LLMs)\nGPT-3 ( paper link (https://arxiv.org/pdf/2005.14165.pdf) paper link ) introduced the idea of adapting models to new task without fine-tuning or gradient update. The approach is to elicit the LLM with text interaction to work with new tasks with accuracy close to many finetuned models. This ability of LLMs to work with few demonstrations or even just a task description is demonstrated when scale of the model crosses a threshold. Image below shows the difference between different ways to do in-context learning.\nUsually, giving few demonstrations (Few-shot) shows better performance than giving just an instruction (Zero-shot). Varying the number of in-context examples also affects performance.\nHow to actually prompt the language models\nDifferent prompt orders have different performance\nRelevant Paper: Fantastically Ordered Prompts and Where to Find Them: Overcoming Few-Shot Prompt Order Sensitivity (https://arxiv.org/abs/2104.08786) Fantastically Ordered Prompts and Where to Find Them: Overcoming Few-Shot Prompt Order Sensitivity\nThis paper gives few major insights:\n1. Even with a fixed set of few-shot prompts, different orders lead to different performance. This means the models are sensitive to permutations of demonstrations we provide them. \n2. There are certain ordering that are \"fantastic\" and we might need to discover them through \"probing\". You can find the details in the paper above.\nFinding similar train examples for prompt improves performance\nRelevant Paper: Making pre-trained language models better few-shot learners (https://arxiv.org/pdf/2012.15723.pdf) Making pre-trained language models better few-shot learners\nIntuitively, we can guess that if we give model demonstrations that look like the final inference task, the model might show better performance. This paper demonstrates improvement in performance by using pre-trained sentence embeddings to select the closest prompt examples to the given input instance (for text classification tasks).\nMain Takeaway : Based on above two papers, we can conclude that, for improving in-context performance for any task:\n1. We need to find \"great\" few-shot examples. This can be done using models pretrained on other tasks. We call such models \"Retrievers\".\n2. Order in which we select the samples matter.\nIn-Context Learning with Retrievers\nRelevant Paper: What Makes Good In-Context Examples for GPT-3? (https://arxiv.org/pdf/2101.06804.pdf) What Makes Good In-Context Examples for GPT-3? 1. This paper demonstrates how using nearest neighbor samples based on semantic similarity to test samples improves performance by benchmarking on various NLU and NLG tasks.\n2. They also show that using models finetuned on semantic similarity tasks (like Roberta finetuned on NLI) shows even better performance. This implies that certain models are better retrievers than others for in-context learning.\nAdvanced: Improve retrievers by finetuning on downstream tasks\nDTE-finetuned (01_dte.md) DTE-finetuned\nTarget Similarity Tuning (02_tst.md) Target Similarity Tuning\nSynchromesh: Reliable Code Generation from Pre-Trained Language Models (https://arxiv.org/pdf/2201.11227.pdf) Synchromesh: Reliable Code Generation from Pre-Trained Language Models\nSuppose that we already have the assumptions that the training examples with the same SQL template as the test input are \"fantastic\" examples, we can extract SQL templates for the training examples and fine-tune the retriever to make the training examples sharing the same SQL template together.\nGenerally, we can finetune the retrievers to learn to embed examples with similar output closer. This seem to work well for code-generation task but we need to benchmark this across tasks.\nLimitation: the assumption that the training examples with the same SQL template as the test input are \"fantastic\" examples . We need different implementations (or even different assumptions) for different downstream tasks."}}]} \ No newline at end of file diff --git a/chat_with_pdf/data/invalid-data-missing-column.jsonl b/chat_with_pdf/data/invalid-data-missing-column.jsonl new file mode 100644 index 000000000..785e9295a --- /dev/null +++ b/chat_with_pdf/data/invalid-data-missing-column.jsonl @@ -0,0 +1 @@ +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf"} diff --git a/chat_with_pdf/environment/Dockerfile b/chat_with_pdf/environment/Dockerfile new file mode 100644 index 000000000..a57a78f41 --- /dev/null +++ b/chat_with_pdf/environment/Dockerfile @@ -0,0 +1,34 @@ +# syntax=docker/dockerfile:1 +FROM docker.io/continuumio/miniconda3:latest + +WORKDIR / + +COPY ./flow/requirements.txt /flow/requirements.txt + +# gcc is for build psutil in MacOS +RUN apt-get update && apt-get install -y runit gcc + +# create conda environment +RUN conda create -n promptflow-serve python=3.9.16 pip=23.0.1 -q -y && \ + conda run -n promptflow-serve \ + pip install -r /flow/requirements.txt && \ + conda run -n promptflow-serve pip install keyrings.alt && \ + conda run -n promptflow-serve pip install gunicorn==20.1.0 && \ + conda run -n promptflow-serve pip cache purge && \ + conda clean -a -y + +COPY ./flow /flow + + +EXPOSE 8080 + +COPY ./connections/* /connections/ + +# reset runsvdir +RUN rm -rf /var/runit +COPY ./runit /var/runit +# grant permission +RUN chmod -R +x /var/runit + +COPY ./start.sh / +CMD ["bash", "./start.sh"] diff --git a/chat_with_pdf/environment/run b/chat_with_pdf/environment/run new file mode 100644 index 000000000..52fabf3e7 --- /dev/null +++ b/chat_with_pdf/environment/run @@ -0,0 +1,11 @@ +#! /bin/bash + +CONDA_ENV_PATH="$(conda info --base)/envs/promptflow-serve" +export PATH="$CONDA_ENV_PATH/bin:$PATH" + +ls +ls /connections +pf connection create --file /connections/aoai.yaml --set api_key=$AOAI_API_KEY +echo "start promptflow serving with worker_num: 8, worker_threads: 1" +cd /flow +gunicorn -w 8 --threads 1 -b "0.0.0.0:8080" --timeout 300 "promptflow._sdk._serving.app:create_app()" \ No newline at end of file diff --git a/chat_with_pdf/flows/evaluation/README.md b/chat_with_pdf/flows/evaluation/README.md new file mode 100644 index 000000000..cc9445aa0 --- /dev/null +++ b/chat_with_pdf/flows/evaluation/README.md @@ -0,0 +1,84 @@ +# Q&A Evaluation: + +This is a flow evaluating the Q&A RAG (Retrieval Augmented Generation) systems by leveraging the state-of-the-art Large Language Models (LLM) to measure the quality and safety of responses. Utilizing GPT model to assist with measurements aims to achieve a high agreement with human evaluations compared to traditional mathematical measurements. + +## What you will learn + +The Q&A RAG evaluation flow allows you to assess and evaluate your model with the LLM-assisted metrics: + + +__gpt_retrieval_score__: Measures the relevance between the retrieved documents and the potential answer to the given question in the range of 1 to 5: + +* 1 means that none of the document is relevant to the question at all +* 5 means that either one of the documents or combination of a few documents is ideal for answering the given question. + + +__gpt_groundedness__ : Measures how grounded the factual information in the answers is against the fact from the retrieved documents. Even if answers is true, if not verifiable against context, then such answers are considered ungrounded. + +Grounding score is scored on a scale of 1 to 5, with 1 being the worst and 5 being the best. + +__gpt_relevance__: Measures the answer quality against the preference answer generated by LLm with the retrieved documents in the range of 1 to 5: + +* 1 means the provided answer is completely irrelevant to the reference answer. +* 5 means the provided answer includes all information necessary to answer the question based on the reference answer. +If the reference answer is can not be generated since no relevant document were retrieved, the answer would be rated as 5. + + +## Prerequisites + +- Connection: Azure OpenAI or OpenAI connection. +- Data input: Evaluating the Coherence metric requires you to provide data inputs including a question, an answer, and documents in json format. + +## Tools used in this flow +- `Python` tool +- `LLM` tool + +## 0. Setup connection +Prepare your Azure Open AI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one. + +```bash +# Override keys with --set to avoid yaml file changes +pf connection create --file ../../../connections/azure_openai.yml --set api_key= api_base= +``` + +## 1. Test flow/node +```bash +# test with default input value in flow.dag.yaml +pf flow test --flow . +``` + +## 2. Create flow run with multi line data and selected metrics +```bash +pf run create --flow . --data ./data.jsonl --column-mapping question='${data.question}' answer='${data.answer}' documents='${data.documents}' metrics='gpt_groundedness' --stream +``` +You can also skip providing `column-mapping` if provided data has same column name as the flow. +Reference [here](https://aka.ms/pf/column-mapping) for default behavior when `column-mapping` not provided in CLI. + +## 3. Run and Evaluate your flow with this Q&A RAG evaluation flow +After you develop your flow, you may want to run and evaluate it with this evaluation flow. + +Here we use the flow [basic_chat](../../chat/basic-chat/) as the main flow to evaluate. It is a flow demonstrating how to create a chatbot with LLM. The chatbot can remember previous interactions and use the conversation history to generate next message, given a question. +### 3.1 Create a batch run of your flow +```bash +pf run create --flow ../../chat/basic-chat --data data.jsonl --column-mapping question='${data.question}' --name basic_chat_run --stream +``` +Please note that `column-mapping` is a mapping from flow input name to specified values. Please refer to [Use column mapping](https://aka.ms/pf/column-mapping) for more details. + +The flow run is named by specifying `--name basic_chat_run` in the above command. You can view the run details with its run name using the command: +```bash +pf run show-details -n basic_chat_run +``` + +### 3.2 Evaluate your flow +You can use this evaluation flow to measure the quality and safety of your flow responses. + +After the chat flow run is finished, you can this evaluation flow to the run: +```bash +pf run create --flow . --data data.jsonl --column-mapping answer='${run.outputs.answer}' documents='{${data.documents}}' question='${data.question}' metrics='gpt_groundedness,gpt_relevance,gpt_retrieval_score' --run basic_chat_run --stream --name evaluation_qa_rag +``` +Please note the flow run to be evaluated is specified with `--run basic_chat_run`. Also same as previous run, the evaluation run is named with `--name evaluation_qa_rag`. +You can view the evaluation run details with: +```bash +pf run show-details -n evaluation_qa_rag +pf run show-metrics -n evaluation_qa_rag +``` \ No newline at end of file diff --git a/chat_with_pdf/flows/evaluation/aggregate_variants_results.py b/chat_with_pdf/flows/evaluation/aggregate_variants_results.py new file mode 100644 index 000000000..fcff09f5a --- /dev/null +++ b/chat_with_pdf/flows/evaluation/aggregate_variants_results.py @@ -0,0 +1,29 @@ +from typing import List +from promptflow import tool, log_metric +import numpy as np + + +@tool +def aggregate_variants_results(results: List[dict], metrics: List[str]): + aggregate_results = {} + for result in results: + for name, value in result.items(): + if name not in aggregate_results.keys(): + aggregate_results[name] = [] + try: + float_val = float(value) + except Exception: + float_val = np.nan + aggregate_results[name].append(float_val) + + for name, value in aggregate_results.items(): + if name in metrics[0]: + metric_name = name + aggregate_results[name] = np.nanmean(value) + if 'pass_rate' in metric_name: + metric_name = metric_name + "(%)" + aggregate_results[name] = aggregate_results[name] * 100.0 + aggregate_results[name] = round(aggregate_results[name], 2) + log_metric(metric_name, aggregate_results[name]) + + return aggregate_results diff --git a/chat_with_pdf/flows/evaluation/concat_scores.py b/chat_with_pdf/flows/evaluation/concat_scores.py new file mode 100644 index 000000000..955c83511 --- /dev/null +++ b/chat_with_pdf/flows/evaluation/concat_scores.py @@ -0,0 +1,30 @@ +from promptflow import tool +import numpy as np + + +@tool +def concat_results(rag_retrieval_score: dict = None, + rag_grounding_score: dict = None, rag_generation_score: dict = None): + + load_list = [{'name': 'gpt_groundedness', 'result': rag_grounding_score}, + {'name': 'gpt_retrieval_score', 'result': rag_retrieval_score}, + {'name': 'gpt_relevance', 'result': rag_generation_score}] + score_list = [] + errors = [] + for item in load_list: + if item['result']: + try: + score = float(item['result']["quality_score"]) + except Exception as e: + score = np.nan + errors.append({"name": item["name"], "msg": str(e), "data": item['result']}) + reasoning = item['result']['quality_reasoning'] + else: + score = np.nan + reasoning = None + score_list.append({"name": item["name"], "score": score, "quality_reasoning": reasoning}) + variant_level_result = {} + for item in score_list: + item_name = str(item["name"]) + variant_level_result[item_name] = item["score"] + return variant_level_result diff --git a/chat_with_pdf/flows/evaluation/flow.dag.yaml b/chat_with_pdf/flows/evaluation/flow.dag.yaml new file mode 100644 index 000000000..703ce0bbf --- /dev/null +++ b/chat_with_pdf/flows/evaluation/flow.dag.yaml @@ -0,0 +1,523 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json +environment: + python_requirements_txt: requirements.txt +inputs: + metrics: + type: string + default: gpt_groundedness,gpt_relevance,gpt_retrieval_score + is_chat_input: false + answer: + type: string + default: Of the tents mentioned in the retrieved documents, the Alpine Explorer + Tent has the highest waterproof rating of 3000mm for its rainfly. + is_chat_input: false + question: + type: string + default: Which tent is the most waterproof? + is_chat_input: false + documents: + type: string + default: "{\"documents\"： [{\"content\":\"

Information about + product item_number: 1

\\n

TrailMaster X4 Tent, price + $250,

\\n

Brand

\\n

OutdoorLiving

\\n

Category

\\n

Tents

\\n

Features

\\n\\n

Technical + Specs

\\n

Best Use: Camping
\\nCapacity: 4-person
\\nSeason + Rating: 3-season
\\nSetup: Freestanding
\\nMaterial: Polyester
\\nWaterproof: Yes
\\nFloor Area: + 80 square feet
\\nPeak Height: 6 feet
\\nNumber of Doors: 2
\\nColor: + Green
\\nRainfly: Included
\\nRainfly + Waterproof Rating: 2000mm
\\nTent Poles: + Aluminum
\\nPole Diameter: 9mm
\\nVentilation: Mesh panels and adjustable vents
\\nInterior Pockets: Yes (4 pockets)
\\nGear Loft: Included
\\nFootprint: Sold separately
\\nGuy + Lines: Reflective
\\nStakes: Aluminum
\\nCarry Bag: Included
\\nDimensions: 10ft x 8ft x 6ft (length x width x peak + height)
\\nPacked Size: 24 inches x 8 inches
\\nWeight: 12 lbs

\\n

TrailMaster X4 Tent User + Guide

\\n

Introduction

\\n

Thank you + for choosing the TrailMaster X4 Tent. This user guide provides + instructions on how to set up, use, and maintain your tent effectively. + Please read this guide thoroughly before using the tent.

\\n

Package Contents

\\n

Ensure that the + package includes the following components:

\\n\\n

If any components are missing + or damaged, please contact our customer support immediately.

\\n

Tent Setup

\\n

Step 1: Selecting a + Suitable Location

\\n\\n

Step 2: Unpacking + and Organizing Components

\\n\\n

Step + 3: Assembling the Tent Poles

\\n\\n

Step 4: Setting up the Tent + Body

\\n\\n

Step 5: Attaching + the Rainfly (if applicable)

\\n\\n

Step 6: Securing the + Tent

\\n\\n

Tent Takedown and Storage

\\n

Step 1: Removing Stakes + and Guy Lines

\\n\",\"id\":null,\"title\":\"Information + about product item_number: + 1\",\"filepath\":\"product_info_1.md\",\"url\":\"https://amipateldemo.blo\ + b.core.windows.net/fileupload-my-product-info/product_info_1.md\",\"metad\ + ata\":{\"chunking\":\"orignal document size=1544. Scores=3.739763Org + Highlight count=75.\"},\"chunk_id\":\"1\"},{\"content\":\"

Information about + product item_number: 8

\\n

Alpine Explorer Tent, price + $350,

\\n

Brand

\\n

AlpineGear

\\n

Category

\\n

Tents

\\n

Features

\\n\\n

Technical Specs

\\n

Best + Use: Camping
\\nCapacity: 8-person
\\nSeason Rating: 3-season
\\nSetup: Freestanding
\\nMaterial: Polyester
\\nWaterproof: Yes
\\nFloor Area: + 120 square feet
\\nPeak Height: 6.5 feet
\\nNumber of Doors: 2
\\nColor: + Orange
\\nRainfly: Included
\\nRainfly + Waterproof Rating: 3000mm
\\nTent Poles: + Aluminum
\\nPole Diameter: 12mm
\\nVentilation: Mesh panels and adjustable vents
\\nInterior Pockets: 4 pockets
\\nGear + Loft: Included
\\nFootprint: Sold + separately
\\nGuy Lines: Reflective
\\nStakes: Aluminum
\\nCarry Bag: + Included
\\nDimensions: 12ft x 10ft x 7ft (Length x + Width x Peak Height)
\\nPacked Size: 24 inches x 10 + inches
\\nWeight: 17 lbs

\\n

Alpine Explorer Tent User + Guide

\\n

Thank you for choosing the Alpine Explorer Tent. This user + guide provides instructions on how to set up, use, and maintain your tent + effectively. Please read this guide thoroughly before using the + tent.

\\n

Package + Contents

\\n

Ensure that the package includes the following + components:

\\n\\n

If any components are missing + or damaged, please contact our customer support immediately.

\\n

Tent Setup

\\n

Step 1: Selecting a + Suitable Location

\\n\\n

Step 2: Unpacking + and Organizing Components

\\n\\n

Step 3: Assembling the Tent + Poles

\\n\\n

Step 4: Setting up the Tent + Body

\\n\\n

Step 5: + Attaching the Rainfly

\\n\\n

Step 6: Securing the + Tent

\\n\\n

Tent + Takedown and Storage

\\n

Step 1: Removing Stakes and Guy + Lines

\\n\\n

Step 2: Taking Down the + Tent Body

\\n\",\"id\":null,\"title\":\"Information about product + item_number: + 8\",\"filepath\":\"product_info_8.md\",\"url\":\"https://amipateldemo.blo\ + b.core.windows.net/fileupload-my-product-info/product_info_8.md\",\"metad\ + ata\":{\"chunking\":\"orignal document size=1419. Scores=3.8508284Org + Highlight count=77.\"},\"chunk_id\":\"1\"},{\"content\":\"

Information about + product item_number: 15

\\n

SkyView 2-Person Tent, price + $200,

\\n

Brand

\\n

OutdoorLiving

\\n

Category

\\n

Tents

\\n

Features

\\n\\n

Technical + Specs

\\n\\n

User + Guide/Manual

\\n
    \\n
  1. Tent Components
    2. \\n
\\n

The + SkyView 2-Person Tent includes the following components:\\n- Tent body\\n- + Rainfly\\n- Aluminum tent poles\\n- Tent stakes\\n- Guy lines\\n- Tent + bag

\\n
    \\n
  1. Tent Setup
    2. \\n
\\n

Follow + these steps to set up your SkyView 2-Person Tent:

\\n

Step 1: Find a + suitable camping site with a level ground and clear of debris.\\nStep 2: + Lay out the tent body on the ground, aligning the doors and vestibules as + desired.\\nStep 3: Assemble the tent poles and insert them into the + corresponding pole sleeves or grommets on the tent body.\\nStep 4: Attach + the rainfly over the tent body, ensuring a secure fit.\\nStep 5: Stake + down the tent and rainfly using the provided tent stakes, ensuring a taut + pitch.\\nStep 6: Adjust the guy lines as needed to enhance stability and + ventilation.\\nStep 7: Once the tent is properly set up, organize your + gear inside and enjoy your camping experience.

\\n
    \\n
  1. Tent Takedown
    2. \\n
\\n

To dismantle and + pack up your SkyView 2-Person Tent, follow these steps:

\\n

Step 1: + Remove all gear and belongings from the tent.\\nStep 2: Remove the stakes + and guy lines from the ground.\\nStep 3: Detach the rainfly from the tent + body.\\nStep 4: Disassemble the tent poles and remove them from the tent + body.\\nStep 5: Fold and roll up the tent body, rainfly, and poles + separately.\\nStep 6: Place all components back into the tent bag, + ensuring a compact and organized packing.

\\n
    \\n
  1. Tent Care and Maintenance
    2. \\n
\\n

To + extend the lifespan of your SkyView 2-Person Tent, follow these care and + maintenance guidelines:

\\n\\n
    \\n
  1. Safety Precautions
    2. \\n
\\n\",\"id\":null,\"title\":\"Information about product + item_number: + 15\",\"filepath\":\"product_info_15.md\",\"url\":\"https://amipateldemo.b\ + lob.core.windows.net/fileupload-my-product-info/product_info_15.md\",\"me\ + tadata\":{\"chunking\":\"orignal document size=1342. Scores=3.4607773Org + Highlight + count=70.\"},\"chunk_id\":\"1\"},{\"content\":\"\\n

Reviews

\\n

36) + Rating: 5\\n Review: The Alpine Explorer + Tent is amazing! It's easy to set up, has excellent ventilation, and the + room divider is a great feature for added privacy. Highly recommend it for + family camping trips!

\\n

37) Rating: 4\\n + Review: I bought the Alpine Explorer Tent, and while it's + waterproof and spacious, I wish it had more storage pockets. Overall, it's + a good tent for camping.

\\n

38) Rating: 5\\n + Review: The Alpine Explorer Tent is perfect for my + family's camping adventures. It's easy to set up, has great ventilation, + and the gear loft is an excellent addition. Love it!

\\n

39) + Rating: 4\\n Review: I like the Alpine + Explorer Tent, but I wish it came with a footprint. It's comfortable and + has many useful features, but a footprint would make it even better. + Overall, it's a great tent.

\\n

40) Rating: 5\\n + Review: This tent is perfect for our family camping + trips. It's spacious, easy to set up, and the room divider is a great + feature for added privacy. The gear loft is a nice bonus for extra + storage.

\\n

FAQ

\\n

34) How easy is it to set + up the Alpine Explorer Tent?\\n The Alpine Explorer Tent features a quick + and easy setup, thanks to color-coded poles and intuitive design. Most + users can set it up in just a few minutes.

\\n

35) Can the Alpine + Explorer Tent accommodate two queen-sized air mattresses?\\n Yes, the + Alpine Explorer Tent is spacious enough to accommodate two queen-sized air + mattresses, making it an ideal choice for comfortable family + camping.

\\n

36) What is the purpose of the room divider in the + Alpine Explorer Tent?\\n The room divider in the Alpine Explorer Tent + allows you to create separate sleeping and living spaces, providing + privacy and organization for your camping experience.

\\n

37) How + does the gear loft in the Alpine Explorer Tent work?\\n The gear loft in + the Alpine Explorer Tent is a suspended mesh shelf that provides + additional storage space for small items, keeping them organized and + easily accessible.

\\n

38) Can the Alpine Explorer Tent be used in + snowy conditions?\\n The Alpine Explorer Tent is designed primarily for + three-season use. While it can withstand light snowfall, it may not + provide adequate structural support and insulation during heavy snow or + extreme winter conditions.

\",\"id\":null,\"title\":\"Information about + product item_number: + 8\",\"filepath\":\"product_info_8.md\",\"url\":\"https://amipateldemo.blo\ + b.core.windows.net/fileupload-my-product-info/product_info_8.md\",\"metad\ + ata\":{\"chunking\":\"orignal document size=906. Scores=5.568323Org + Highlight count=85.\"},\"chunk_id\":\"0\"},{\"content\":\"

If you have + any questions or need further assistance, please contact our customer + support:

\\n\\n

Return + Policy

\\n\\n

Reviews

\\n

1) + Rating: 5\\n Review: I am extremely + happy with my TrailMaster X4 Tent! It's spacious, easy to set up, and kept + me dry during a storm. The UV protection is a great addition too. Highly + recommend it to anyone who loves camping!

\\n

2) + Rating: 3\\n Review: I bought the + TrailMaster X4 Tent, and while it's waterproof and has a spacious + interior, I found it a bit difficult to set up. It's a decent tent, but I + wish it were easier to assemble.

\\n

3) Rating: 5\\n + Review: The TrailMaster X4 Tent is a fantastic investment + for any serious camper. The easy setup and spacious interior make it + perfect for extended trips, and the waterproof design kept us dry in heavy + rain.

\\n

4) Rating: 4\\n Review: I + like the TrailMaster X4 Tent, but I wish it came in more colors. It's + comfortable and has many useful features, but the green color just isn't + my favorite. Overall, it's a good tent.

\\n

5) + Rating: 5\\n Review: This tent is + perfect for my family camping trips. The spacious interior and convenient + storage pocket make it easy to stay organized. It's also super easy to set + up, making it a great addition to our gear.

\\n

FAQ

\\n

1) Can the TrailMaster X4 Tent be used in + winter conditions?\\n The TrailMaster X4 Tent is designed for 3-season use + and may not be suitable for extreme winter conditions with heavy snow and + freezing temperatures.

\\n

2) How many people can comfortably sleep + in the TrailMaster X4 Tent?\\n The TrailMaster X4 Tent can comfortably + accommodate up to 4 people with room for their gear.

\\n

3) Is there + a warranty on the TrailMaster X4 Tent?\\n Yes, the TrailMaster X4 Tent + comes with a 2-year limited warranty against manufacturing + defects.

\\n

4) Are there any additional accessories included with + the TrailMaster X4 Tent?\\n The TrailMaster X4 Tent includes a rainfly, + tent stakes, guy lines, and a carry bag for easy transport.

\\n

5) + Can the TrailMaster X4 Tent be easily carried during hikes?\\n Yes, the + TrailMaster X4 Tent weighs just 12lbs, and when packed in its carry bag, + it can be comfortably carried during + hikes.

\",\"id\":null,\"title\":\"Information about product + item_number: + 1\",\"filepath\":\"product_info_1.md\",\"url\":\"https://amipateldemo.blo\ + b.core.windows.net/fileupload-my-product-info/product_info_1.md\",\"metad\ + ata\":{\"chunking\":\"orignal document size=981. Scores=4.0350547Org + Highlight count=74.\"},\"chunk_id\":\"0\"}]}" + is_chat_input: false +outputs: + gpt_relevance: + type: string + reference: ${concat_scores.output.gpt_relevance} + gpt_groundedness: + type: string + reference: ${concat_scores.output.gpt_groundedness} + gpt_retrieval_score: + type: string + reference: ${concat_scores.output.gpt_retrieval_score} +nodes: +- name: concat_scores + type: python + source: + type: code + path: concat_scores.py + inputs: + rag_generation_score: ${parse_generation_score.output} + rag_grounding_score: ${parse_grounding_score.output} + rag_retrieval_score: ${parse_retrieval_score.output} + use_variants: false +- name: aggregate_variants_results + type: python + source: + type: code + path: aggregate_variants_results.py + inputs: + metrics: ${inputs.metrics} + results: ${concat_scores.output} + aggregation: true + use_variants: false +- name: gpt_groundedness + type: llm + source: + type: code + path: rag_groundedness_prompt.jinja2 + inputs: + deployment_name: aoai + temperature: 0 + top_p: 1 + max_tokens: 1000 + presence_penalty: 0 + frequency_penalty: 0 + FullBody: ${inputs.documents} + answer: ${inputs.answer} + question: ${inputs.question} + connection: aoai + api: chat + activate: + when: ${validate_input.output.gpt_groundedness} + is: true + use_variants: false +- name: gpt_retrieval_score + type: llm + source: + type: code + path: rag_retrieval_prompt.jinja2 + inputs: + deployment_name: aoai + temperature: 0 + top_p: 1 + max_tokens: 1000 + presence_penalty: 0 + frequency_penalty: 0 + FullBody: ${inputs.documents} + question: ${inputs.question} + connection: aoai + api: chat + activate: + when: ${validate_input.output.gpt_retrieval_score} + is: true + use_variants: false +- name: gpt_relevance + type: llm + source: + type: code + path: rag_generation_prompt.jinja2 + inputs: + deployment_name: aoai + temperature: 0 + top_p: 1 + max_tokens: 1000 + presence_penalty: 0 + frequency_penalty: 0 + FullBody: ${inputs.documents} + answer: ${inputs.answer} + question: ${inputs.question} + connection: aoai + api: chat + activate: + when: ${validate_input.output.gpt_relevance} + is: true + use_variants: false +- name: parse_generation_score + type: python + source: + type: code + path: parse_generation_score.py + inputs: + rag_generation_score: ${gpt_relevance.output} + use_variants: false +- name: parse_retrieval_score + type: python + source: + type: code + path: parse_retrival_score.py + inputs: + retrieval_output: ${gpt_retrieval_score.output} + use_variants: false +- name: parse_grounding_score + type: python + source: + type: code + path: parse_groundedness_score.py + inputs: + rag_grounding_score: ${gpt_groundedness.output} + use_variants: false +- name: select_metrics + type: python + source: + type: code + path: select_metrics.py + inputs: + metrics: ${inputs.metrics} + use_variants: false +- name: validate_input + type: python + source: + type: code + path: validate_input.py + inputs: + answer: ${inputs.answer} + documents: ${inputs.documents} + question: ${inputs.question} + selected_metrics: ${select_metrics.output} + use_variants: false diff --git a/chat_with_pdf/flows/evaluation/parse_generation_score.py b/chat_with_pdf/flows/evaluation/parse_generation_score.py new file mode 100644 index 000000000..bbddf346c --- /dev/null +++ b/chat_with_pdf/flows/evaluation/parse_generation_score.py @@ -0,0 +1,23 @@ +from promptflow import tool +import re + + +@tool +def parse_generation_output(rag_generation_score: str) -> str: + quality_score = float('nan') + quality_reasoning = '' + for sent in rag_generation_score.split('\n'): + sent = sent.strip() + if re.match(r"\s*(<)?Quality score:", sent): + numbers_found = re.findall(r"(\d+\.*\d*)\/", sent) + if len(numbers_found) == 0: + continue + quality_score = int( + float(numbers_found[0].replace("'", ""))) + + for sent in rag_generation_score.split('\n'): + sent = sent.strip() + if re.match(r"\s*(<)?Quality score reasoning:", sent): + quality_reasoning += sent.strip() + break + return {"quality_score": quality_score, "quality_reasoning": quality_reasoning} diff --git a/chat_with_pdf/flows/evaluation/parse_groundedness_score.py b/chat_with_pdf/flows/evaluation/parse_groundedness_score.py new file mode 100644 index 000000000..bbcc4d455 --- /dev/null +++ b/chat_with_pdf/flows/evaluation/parse_groundedness_score.py @@ -0,0 +1,16 @@ +from promptflow import tool +import re + + +@tool +def parse_grounding_output(rag_grounding_score: str) -> str: + try: + numbers_found = re.findall(r"Quality score:\s*(\d+)\/\d", rag_grounding_score) + score = float(numbers_found[0]) if len(numbers_found) > 0 else 0 + except Exception: + score = float("nan") + try: + quality_reasoning, _ = rag_grounding_score.split("Quality score: ") + except Exception: + quality_reasoning = rag_grounding_score + return {"quality_score": score, "quality_reasoning": quality_reasoning} diff --git a/chat_with_pdf/flows/evaluation/parse_retrival_score.py b/chat_with_pdf/flows/evaluation/parse_retrival_score.py new file mode 100644 index 000000000..ec8f084b9 --- /dev/null +++ b/chat_with_pdf/flows/evaluation/parse_retrival_score.py @@ -0,0 +1,20 @@ +from promptflow import tool +import re + + +@tool +def parse_retrieval_output(retrieval_output: str) -> str: + score_response = [sent.strip() for sent in + retrieval_output.strip("\"").split("# Result")[-1].strip().split('.') if sent.strip()] + parsed_score_response = re.findall(r"\d+", score_response[-1]) + if len(parsed_score_response) > 0: + score = parsed_score_response[-1].strip() + if float(score) < 1.0 or float(score) > 5.0: + score = float('nan') + else: + score = float('nan') + try: + reasoning_response, _ = retrieval_output.split("# Result") + except Exception: + reasoning_response = retrieval_output + return {"quality_score": float(score), "quality_reasoning": reasoning_response} diff --git a/chat_with_pdf/flows/evaluation/rag_generation_prompt.jinja2 b/chat_with_pdf/flows/evaluation/rag_generation_prompt.jinja2 new file mode 100644 index 000000000..4c332a332 --- /dev/null +++ b/chat_with_pdf/flows/evaluation/rag_generation_prompt.jinja2 @@ -0,0 +1,41 @@ +system: +You will be provided a question, a conversation history, fetched documents related to the question and a response to the question in the domain. You task is to evaluate the quality of the provided response by following the steps below: +- Understand the context of the question based on the conversation history. +- Generate a reference answer that is only based on the conversation history, question, and fetched documents. Don't generate the reference answer based on your own knowledge. +- You need to rate the provided response according to the reference answer if it's available on a scale of 1 (poor) to 5 (excellent), based on the below criteria: + - 5 - Ideal: The provided response includes all information necessary to answer the question based on the reference answer and conversation history. Please be strict about giving a 5 score. + - 4 - Mostly Relevant: The provided response is mostly relevant, although it may be a little too narrow or too broad based on the reference answer and conversation history. + - 3 - Somewhat Relevant: The provided response may be partly helpful but might be hard to read or contain other irrelevant content based on the reference answer and conversation history. + - 2 - Barely Relevant: The provided response is barely relevant, perhaps shown as a last resort based on the reference answer and conversation history. + - 1 - Completely Irrelevant: The provided response should never be used for answering this question based on the reference answer and conversation history. +- You need to rate the provided response to be 5, if the reference answer can not be generated since no relevant documents were retrieved. +- You need to first provide a scoring reason for the evaluation according to the above criteria, and then provide a score for the quality of the provided response. +- You need to translate the provided response into English if it's in another language. +- Your final response must include both the reference answer and the evaluation result. The evaluation result should be written in English. Your response should be in the following format: +``` +[assistant](#evaluation result) + +[insert the reference answer here] + + +Quality score reasoning: [insert score reasoning here] + + +Quality score: [insert score here]/5 + +``` +- Your answer must end with <|im_end|>. + +user: +#conversation history + +#question +{{question}} +#fetched documents +{{FullBody}} +#provided response +{{answer}} + +assistant: +#evaluation result +""" \ No newline at end of file diff --git a/chat_with_pdf/flows/evaluation/rag_groundedness_prompt.jinja2 b/chat_with_pdf/flows/evaluation/rag_groundedness_prompt.jinja2 new file mode 100644 index 000000000..34ac60a92 --- /dev/null +++ b/chat_with_pdf/flows/evaluation/rag_groundedness_prompt.jinja2 @@ -0,0 +1,34 @@ +system: +You are a helpful assistant. +user: +Your task is to check and rate if factual information in chatbot's reply is all grounded to retrieved documents. +You will be given a question, chatbot's response to the question, a chat history between this chatbot and human, and a list of retrieved documents in json format. +The chatbot must base its response exclusively on factual information extracted from the retrieved documents, utilizing paraphrasing, summarization, or inference techniques. When the chatbot responds to information that is not mentioned in or cannot be inferred from the retrieved documents, we refer to it as a grounded issue. + +To rate the groundness of chat response, follow the below steps: +1. Review the chat history to understand better about the question and chat response +2. Look for all the factual information in chatbot's response +3. Compare the factual information in chatbot's response with the retrieved documents. Check if there are any facts that are not in the retrieved documents at all,or that contradict or distort the facts in the retrieved documents. If there are, write them down. If there are none, leave it blank. Note that some facts may be implied or suggested by the retrieved documents, but not explicitly stated. In that case, use your best judgment to decide if the fact is grounded or not. + For example, if the retrieved documents mention that a film was nominated for 12 awards, and chatbot's reply states the same, you can consider that fact as grounded, as it is directly taken from the retrieved documents. + However, if the retrieved documents do not mention the film won any awards at all, and chatbot reply states that the film won some awards, you should consider that fact as not grounded. +4. Rate how well grounded the chatbot response is on a Likert scale from 1 to 5 judging if chatbot response has no ungrounded facts. (higher better) + 5: agree strongly + 4: agree + 3: neither agree or disagree + 2: disagree + 1: disagree strongly + If the chatbot response used information from outside sources, or made claims that are not backed up by the retrieved documents, give it a low score. +5. Your answer should follow the format: + [insert reasoning here] + +Your answer must end with . + +# Question +{{ question }} +# Chat Response +{{ answer }} +# Chat History +# Documents +---BEGIN RETRIEVED DOCUMENTS--- +{{ FullBody }} +---END RETRIEVED DOCUMENTS--- \ No newline at end of file diff --git a/chat_with_pdf/flows/evaluation/rag_retrieval_prompt.jinja2 b/chat_with_pdf/flows/evaluation/rag_retrieval_prompt.jinja2 new file mode 100644 index 000000000..640b42e40 --- /dev/null +++ b/chat_with_pdf/flows/evaluation/rag_retrieval_prompt.jinja2 @@ -0,0 +1,22 @@ +system: +You are a helpful assistant. +user: +A chat history between user and bot is shown below +A list of documents is shown below in json format, and each document has one unique id. +These listed documents are used as contex to answer the given question. +The task is to score the relevance between the documents and the potential answer to the given question in the range of 1 to 5. +1 means none of the documents is relevant to the question at all. 5 means either one of the document or combination of a few documents is ideal for answering the given question. +Think through step by step: +- Summarize each given document first +- Determine the underlying intent of the given question, when the question is ambiguous, refer to the given chat history +- Measure how suitable each document to the given question, list the document id and the corresponding relevance score. +- Summarize the overall relevance of given list of documents to the given question after # Overall Reason, note that the answer to the question can solely from single document or a combination of multiple documents. +- Finally, output "# Result" followed by a score from 1 to 5. + +# Question +{{question}} +# Chat History +# Documents +---BEGIN RETRIEVED DOCUMENTS--- +{{FullBody}} +---END RETRIEVED DOCUMENTS--- \ No newline at end of file diff --git a/chat_with_pdf/flows/evaluation/requirements.txt b/chat_with_pdf/flows/evaluation/requirements.txt new file mode 100644 index 000000000..34d068f5f --- /dev/null +++ b/chat_with_pdf/flows/evaluation/requirements.txt @@ -0,0 +1,2 @@ +promptflow +promptflow-tools \ No newline at end of file diff --git a/chat_with_pdf/flows/evaluation/select_metrics.py b/chat_with_pdf/flows/evaluation/select_metrics.py new file mode 100644 index 000000000..297022088 --- /dev/null +++ b/chat_with_pdf/flows/evaluation/select_metrics.py @@ -0,0 +1,14 @@ +from promptflow import tool + + +@tool +def select_metrics(metrics: str) -> str: + supported_metrics = ('gpt_relevance', 'gpt_groundedness', 'gpt_retrieval_score') + user_selected_metrics = [metric.strip() for metric in metrics.split(',') if metric] + metric_selection_dict = {} + for metric in supported_metrics: + if metric in user_selected_metrics: + metric_selection_dict[metric] = True + else: + metric_selection_dict[metric] = False + return metric_selection_dict diff --git a/chat_with_pdf/flows/evaluation/validate_input.py b/chat_with_pdf/flows/evaluation/validate_input.py new file mode 100644 index 000000000..66ec5774e --- /dev/null +++ b/chat_with_pdf/flows/evaluation/validate_input.py @@ -0,0 +1,27 @@ +from promptflow import tool + + +def is_valid(input_item): + return True if input_item and input_item.strip() else False + + +@tool +def validate_input(question: str, answer: str, documents: str, selected_metrics: dict) -> dict: + input_data = {"question": is_valid(question), "answer": is_valid(answer), "documents": is_valid(documents)} + expected_input_cols = set(input_data.keys()) + dict_metric_required_fields = {"gpt_groundedness": set(["question", "answer", "documents"]), + "gpt_relevance": set(["question", "answer", "documents"]), + "gpt_retrieval_score": set(["question", "documents"])} + actual_input_cols = set() + for col in expected_input_cols: + if input_data[col]: + actual_input_cols.add(col) + data_validation = selected_metrics + for metric in selected_metrics: + if selected_metrics[metric]: + metric_required_fields = dict_metric_required_fields[metric] + if metric_required_fields <= actual_input_cols: + data_validation[metric] = True + else: + data_validation[metric] = False + return data_validation diff --git a/chat_with_pdf/flows/standard/README.md b/chat_with_pdf/flows/standard/README.md new file mode 100644 index 000000000..e19cd1081 --- /dev/null +++ b/chat_with_pdf/flows/standard/README.md @@ -0,0 +1,71 @@ +# Chat with PDF + +This is a simple flow that allow you to ask questions about the content of a PDF file and get answers. +You can run the flow with a URL to a PDF file and question as argument. +Once it's launched it will download the PDF and build an index of the content. +Then when you ask a question, it will look up the index to retrieve relevant content and post the question with the relevant content to OpenAI chat model (gpt-3.5-turbo or gpt4) to get an answer. + +Learn more on corresponding [tutorials](../../../tutorials/e2e-development/chat-with-pdf.md). + +Tools used in this flow： +- custom `python` Tool + +## Prerequisites + +Install promptflow sdk and other dependencies: +```bash +pip install -r requirements.txt +``` + +## Get started +### Create connection in this folder + +```bash +# create connection needed by flow +if pf connection list | grep open_ai_connection; then + echo "open_ai_connection already exists" +else + pf connection create --file ../../../connections/azure_openai.yml --name open_ai_connection --set api_key= api_base= +fi +``` + +### CLI Example + +#### Run flow + +**Note**: this sample uses [predownloaded PDFs](./chat_with_pdf/.pdfs/) and [prebuilt FAISS Index](./chat_with_pdf/.index/) to speed up execution time. +You can remove the folders to start a fresh run. + +```bash +# test with default input value in flow.dag.yaml +pf flow test --flow . + +# test with flow inputs +pf flow test --flow . --inputs question="What is the name of the new language representation model introduced in the document?" pdf_url="https://arxiv.org/pdf/1810.04805.pdf" + +# (Optional) create a random run name +run_name="web_classification_"$(openssl rand -hex 12) + +# run with multiline data, --name is optional +pf run create --file batch_run.yaml --name $run_name + +# visualize run output details +pf run visualize --name $run_name +``` + +#### Submit run to cloud + +Assume we already have a connection named `open_ai_connection` in workspace. + +```bash +# set default workspace +az account set -s +az configure --defaults group= workspace= +``` + +``` bash +# create run +pfazure run create --file batch_run.yaml --name $run_name +``` + +Note: Click portal_url of the run to view the final snapshot. diff --git a/chat_with_pdf/flows/standard/__init__.py b/chat_with_pdf/flows/standard/__init__.py new file mode 100644 index 000000000..9d4387029 --- /dev/null +++ b/chat_with_pdf/flows/standard/__init__.py @@ -0,0 +1,6 @@ +import sys +import os + +sys.path.append( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "chat_with_pdf") +) diff --git a/chat_with_pdf/flows/standard/batch_run.yaml b/chat_with_pdf/flows/standard/batch_run.yaml new file mode 100644 index 000000000..44f3148c3 --- /dev/null +++ b/chat_with_pdf/flows/standard/batch_run.yaml @@ -0,0 +1,17 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Run.schema.json +#name: chat_with_pdf_default_20230820_162219_559000 +flow: . +data: ./data/bert-paper-qna.jsonl +#run: +column_mapping: + chat_history: ${data.chat_history} + pdf_url: ${data.pdf_url} + question: ${data.question} + config: + EMBEDDING_MODEL_DEPLOYMENT_NAME: text-embedding-ada-002 + CHAT_MODEL_DEPLOYMENT_NAME: gpt-4 + PROMPT_TOKEN_LIMIT: 3000 + MAX_COMPLETION_TOKENS: 1024 + VERBOSE: true + CHUNK_SIZE: 1024 + CHUNK_OVERLAP: 64 \ No newline at end of file diff --git a/chat_with_pdf/flows/standard/build_index_tool.py b/chat_with_pdf/flows/standard/build_index_tool.py new file mode 100644 index 000000000..abbff62e7 --- /dev/null +++ b/chat_with_pdf/flows/standard/build_index_tool.py @@ -0,0 +1,7 @@ +from promptflow import tool +from chat_with_pdf.build_index import create_faiss_index + + +@tool +def build_index_tool(pdf_path: str) -> str: + return create_faiss_index(pdf_path) diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/.env.example b/chat_with_pdf/flows/standard/chat_with_pdf/.env.example new file mode 100644 index 000000000..09cb5b0a5 --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/.env.example @@ -0,0 +1,21 @@ +# Azure OpenAI, uncomment below section if you want to use Azure OpenAI +# Note: EMBEDDING_MODEL_DEPLOYMENT_NAME and CHAT_MODEL_DEPLOYMENT_NAME are deployment names for Azure OpenAI +OPENAI_API_TYPE=azure +OPENAI_API_BASE= +OPENAI_API_KEY= +OPENAI_API_VERSION=2023-05-15 +EMBEDDING_MODEL_DEPLOYMENT_NAME=text-embedding-ada-002 +CHAT_MODEL_DEPLOYMENT_NAME=gpt-4 + +# OpenAI, uncomment below section if you want to use OpenAI +# Note: EMBEDDING_MODEL_DEPLOYMENT_NAME and CHAT_MODEL_DEPLOYMENT_NAME are model names for OpenAI +#OPENAI_API_KEY= +#OPENAI_ORG_ID= # this is optional +#EMBEDDING_MODEL_DEPLOYMENT_NAME=text-embedding-ada-002 +#CHAT_MODEL_DEPLOYMENT_NAME=gpt-4 + +PROMPT_TOKEN_LIMIT=2000 +MAX_COMPLETION_TOKENS=1024 +CHUNK_SIZE=256 +CHUNK_OVERLAP=16 +VERBOSE=True \ No newline at end of file diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/README.md b/chat_with_pdf/flows/standard/chat_with_pdf/README.md new file mode 100644 index 000000000..e40960633 --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/README.md @@ -0,0 +1,27 @@ +# Chat with PDF +This is a simple Python application that allow you to ask questions about the content of a PDF file and get answers. +It's a console application that you start with a URL to a PDF file as argument. Once it's launched it will download the PDF and build an index of the content. Then when you ask a question, it will look up the index to retrieve relevant content and post the question with the relevant content to OpenAI chat model (gpt-3.5-turbo or gpt4) to get an answer. + +## Screenshot - ask questions about BERT paper +![screenshot-chat-with-pdf](../assets/chat_with_pdf_console.png) + +## How it works? + +## Get started +### Create .env file in this folder with below content +``` +OPENAI_API_BASE= +OPENAI_API_KEY= +EMBEDDING_MODEL_DEPLOYMENT_NAME=text-embedding-ada-002 +CHAT_MODEL_DEPLOYMENT_NAME=gpt-35-turbo +PROMPT_TOKEN_LIMIT=3000 +MAX_COMPLETION_TOKENS=256 +VERBOSE=false +CHUNK_SIZE=1024 +CHUNK_OVERLAP=64 +``` +Note: CHAT_MODEL_DEPLOYMENT_NAME should point to a chat model like gpt-3.5-turbo or gpt-4 +### Run the command line +```shell +python main.py +``` \ No newline at end of file diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/__init__.py b/chat_with_pdf/flows/standard/chat_with_pdf/__init__.py new file mode 100644 index 000000000..96a36c3a6 --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/__init__.py @@ -0,0 +1,4 @@ +import sys +import os + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/build_index.py b/chat_with_pdf/flows/standard/chat_with_pdf/build_index.py new file mode 100644 index 000000000..d96765adb --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/build_index.py @@ -0,0 +1,69 @@ +import PyPDF2 +import faiss +import os + +from pathlib import Path + +from utils.oai import OAIEmbedding +from utils.index import FAISSIndex +from utils.logging import log +from utils.lock import acquire_lock +from constants import INDEX_DIR + + +def create_faiss_index(pdf_path: str) -> str: + chunk_size = int(os.environ.get("CHUNK_SIZE")) + chunk_overlap = int(os.environ.get("CHUNK_OVERLAP")) + log(f"Chunk size: {chunk_size}, chunk overlap: {chunk_overlap}") + + file_name = Path(pdf_path).name + f".index_{chunk_size}_{chunk_overlap}" + index_persistent_path = Path(INDEX_DIR) / file_name + index_persistent_path = index_persistent_path.resolve().as_posix() + lock_path = index_persistent_path + ".lock" + log("Index path: " + os.path.abspath(index_persistent_path)) + + with acquire_lock(lock_path): + if os.path.exists(os.path.join(index_persistent_path, "index.faiss")): + log("Index already exists, bypassing index creation") + return index_persistent_path + else: + if not os.path.exists(index_persistent_path): + os.makedirs(index_persistent_path) + + log("Building index") + pdf_reader = PyPDF2.PdfReader(pdf_path) + + text = "" + for page in pdf_reader.pages: + text += page.extract_text() + + # Chunk the words into segments of X words with Y-word overlap, X=CHUNK_SIZE, Y=OVERLAP_SIZE + segments = split_text(text, chunk_size, chunk_overlap) + + log(f"Number of segments: {len(segments)}") + + index = FAISSIndex(index=faiss.IndexFlatL2(1536), embedding=OAIEmbedding()) + index.insert_batch(segments) + + index.save(index_persistent_path) + + log("Index built: " + index_persistent_path) + return index_persistent_path + + +# Split the text into chunks with CHUNK_SIZE and CHUNK_OVERLAP as character count +def split_text(text, chunk_size, chunk_overlap): + # Calculate the number of chunks + num_chunks = (len(text) - chunk_overlap) // (chunk_size - chunk_overlap) + + # Split the text into chunks + chunks = [] + for i in range(num_chunks): + start = i * (chunk_size - chunk_overlap) + end = start + chunk_size + chunks.append(text[start:end]) + + # Add the last chunk + chunks.append(text[num_chunks * (chunk_size - chunk_overlap):]) + + return chunks diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/constants.py b/chat_with_pdf/flows/standard/chat_with_pdf/constants.py new file mode 100644 index 000000000..cc937d43c --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/constants.py @@ -0,0 +1,5 @@ +import os + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +PDF_DIR = os.path.join(BASE_DIR, ".pdfs") +INDEX_DIR = os.path.join(BASE_DIR, ".index/.pdfs/") diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/download.py b/chat_with_pdf/flows/standard/chat_with_pdf/download.py new file mode 100644 index 000000000..999b935e8 --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/download.py @@ -0,0 +1,31 @@ +import requests +import os +import re + +from utils.lock import acquire_lock +from utils.logging import log +from constants import PDF_DIR + + +# Download a pdf file from a url and return the path to the file +def download(url: str) -> str: + path = os.path.join(PDF_DIR, normalize_filename(url) + ".pdf") + lock_path = path + ".lock" + + with acquire_lock(lock_path): + if os.path.exists(path): + log("Pdf already exists in " + os.path.abspath(path)) + return path + + log("Downloading pdf from " + url) + response = requests.get(url) + + with open(path, "wb") as f: + f.write(response.content) + + return path + + +def normalize_filename(filename): + # Replace any invalid characters with an underscore + return re.sub(r"[^\w\-_. ]", "_", filename) diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/find_context.py b/chat_with_pdf/flows/standard/chat_with_pdf/find_context.py new file mode 100644 index 000000000..ef0066b11 --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/find_context.py @@ -0,0 +1,31 @@ +import faiss +from jinja2 import Environment, FileSystemLoader +import os + +from utils.index import FAISSIndex +from utils.oai import OAIEmbedding, render_with_token_limit +from utils.logging import log + + +def find_context(question: str, index_path: str): + index = FAISSIndex(index=faiss.IndexFlatL2(1536), embedding=OAIEmbedding()) + index.load(path=index_path) + snippets = index.query(question, top_k=5) + + template = Environment( + loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) + ).get_template("qna_prompt.md") + token_limit = int(os.environ.get("PROMPT_TOKEN_LIMIT")) + + # Try to render the template with token limit and reduce snippet count if it fails + while True: + try: + prompt = render_with_token_limit( + template, token_limit, question=question, context=enumerate(snippets) + ) + break + except ValueError: + snippets = snippets[:-1] + log(f"Reducing snippet count to {len(snippets)} to fit token limit") + + return prompt, snippets diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/main.py b/chat_with_pdf/flows/standard/chat_with_pdf/main.py new file mode 100644 index 000000000..e50ce66de --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/main.py @@ -0,0 +1,68 @@ +import argparse +from dotenv import load_dotenv +import os + +from qna import qna +from find_context import find_context +from rewrite_question import rewrite_question +from build_index import create_faiss_index +from download import download +from utils.lock import acquire_lock +from constants import PDF_DIR, INDEX_DIR + + +def chat_with_pdf(question: str, pdf_url: str, history: list): + with acquire_lock("create_folder.lock"): + if not os.path.exists(PDF_DIR): + os.mkdir(PDF_DIR) + if not os.path.exists(INDEX_DIR): + os.makedirs(INDEX_DIR) + + pdf_path = download(pdf_url) + index_path = create_faiss_index(pdf_path) + q = rewrite_question(question, history) + prompt, context = find_context(q, index_path) + stream = qna(prompt, history) + + return stream, context + + +def print_stream_and_return_full_answer(stream): + answer = "" + for str in stream: + print(str, end="", flush=True) + answer = answer + str + "" + print(flush=True) + + return answer + + +def main_loop(url: str): + load_dotenv(os.path.join(os.path.dirname(__file__), ".env"), override=True) + + history = [] + while True: + question = input("\033[92m" + "$User (type q! to quit): " + "\033[0m") + if question == "q!": + break + + stream, context = chat_with_pdf(question, url, history) + + print("\033[92m" + "$Bot: " + "\033[0m", end=" ", flush=True) + answer = print_stream_and_return_full_answer(stream) + history = history + [ + {"role": "user", "content": question}, + {"role": "assistant", "content": answer}, + ] + + +def main(): + parser = argparse.ArgumentParser(description="Ask questions about a PDF file") + parser.add_argument("url", help="URL to the PDF file") + args = parser.parse_args() + + main_loop(args.url) + + +if __name__ == "__main__": + main() diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/qna.py b/chat_with_pdf/flows/standard/chat_with_pdf/qna.py new file mode 100644 index 000000000..fc2da7721 --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/qna.py @@ -0,0 +1,15 @@ +import os + +from utils.oai import OAIChat + + +def qna(prompt: str, history: list): + max_completion_tokens = int(os.environ.get("MAX_COMPLETION_TOKENS")) + + chat = OAIChat() + stream = chat.stream( + messages=history + [{"role": "user", "content": prompt}], + max_tokens=max_completion_tokens, + ) + + return stream diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/qna_prompt.md b/chat_with_pdf/flows/standard/chat_with_pdf/qna_prompt.md new file mode 100644 index 000000000..4535db4b2 --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/qna_prompt.md @@ -0,0 +1,15 @@ +You're a smart assistant can answer questions based on provided context and previous conversation history between you and human. + +Use the context to answer the question at the end, note that the context has order and importance - e.g. context #1 is more important than #2. + +Try as much as you can to answer based on the provided the context, if you cannot derive the answer from the context, you should say you don't know. +Answer in the same language as the question. + +# Context +{% for i, c in context %} +## Context #{{i+1}} +{{c.text}} +{% endfor %} + +# Question +{{question}} \ No newline at end of file diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/rewrite_question.py b/chat_with_pdf/flows/standard/chat_with_pdf/rewrite_question.py new file mode 100644 index 000000000..6e299a6bf --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/rewrite_question.py @@ -0,0 +1,31 @@ +from jinja2 import Environment, FileSystemLoader +import os +from utils.logging import log +from utils.oai import OAIChat, render_with_token_limit + + +def rewrite_question(question: str, history: list): + template = Environment( + loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) + ).get_template("rewrite_question_prompt.md") + token_limit = int(os.environ["PROMPT_TOKEN_LIMIT"]) + max_completion_tokens = int(os.environ["MAX_COMPLETION_TOKENS"]) + + # Try to render the prompt with token limit and reduce the history count if it fails + while True: + try: + prompt = render_with_token_limit( + template, token_limit, question=question, history=history + ) + break + except ValueError: + history = history[:-1] + log(f"Reducing chat history count to {len(history)} to fit token limit") + + chat = OAIChat() + rewritten_question = chat.generate( + messages=[{"role": "user", "content": prompt}], max_tokens=max_completion_tokens + ) + log(f"Rewritten question: {rewritten_question}") + + return rewritten_question diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/rewrite_question_prompt.md b/chat_with_pdf/flows/standard/chat_with_pdf/rewrite_question_prompt.md new file mode 100644 index 000000000..d9c0073d8 --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/rewrite_question_prompt.md @@ -0,0 +1,33 @@ +You are able to reason from previous conversation and the recent question, to come up with a rewrite of the question which is concise but with enough information that people without knowledge of previous conversation can understand the question. + +A few examples: + +# Example 1 +## Previous conversation +user: Who is Bill Clinton? +assistant: Bill Clinton is an American politician who served as the 42nd President of the United States from 1993 to 2001. +## Question +user: When was he born? +## Rewritten question +When was Bill Clinton born? + +# Example 2 +## Previous conversation +user: What is BERT? +assistant: BERT stands for "Bidirectional Encoder Representations from Transformers." It is a natural language processing (NLP) model developed by Google. +user: What data was used for its training? +assistant: The BERT (Bidirectional Encoder Representations from Transformers) model was trained on a large corpus of publicly available text from the internet. It was trained on a combination of books, articles, websites, and other sources to learn the language patterns and relationships between words. +## Question +user: What NLP tasks can it perform well? +## Rewritten question +What NLP tasks can BERT perform well? + +Now comes the actual work - please respond with the rewritten question in the same language as the question, nothing else. + +## Previous conversation +{% for item in history %} +{{item["role"]}}: {{item["content"]}} +{% endfor %} +## Question +{{question}} +## Rewritten question \ No newline at end of file diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/test.ipynb b/chat_with_pdf/flows/standard/chat_with_pdf/test.ipynb new file mode 100644 index 000000000..3d63855cf --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/test.ipynb @@ -0,0 +1,60 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from main import chat_with_pdf, print_stream_and_return_full_answer\n", + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv()\n", + "\n", + "bert_paper_url = \"https://arxiv.org/pdf/1810.04805.pdf\"\n", + "questions = [\n", + " \"what is BERT?\",\n", + " \"what NLP tasks does it perform well?\",\n", + " \"is BERT suitable for NER?\",\n", + " \"is it better than GPT\",\n", + " \"when was GPT come up?\",\n", + " \"when was BERT come up?\",\n", + " \"so about same time?\",\n", + "]\n", + "\n", + "history = []\n", + "for q in questions:\n", + " stream, context = chat_with_pdf(q, bert_paper_url, history)\n", + " print(\"User: \" + q, flush=True)\n", + " print(\"Bot: \", end=\"\", flush=True)\n", + " answer = print_stream_and_return_full_answer(stream)\n", + " history = history + [\n", + " {\"role\": \"user\", \"content\": q},\n", + " {\"role\": \"assistant\", \"content\": answer},\n", + " ]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pf", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.17" + }, + "stage": "development" + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/utils/__init__.py b/chat_with_pdf/flows/standard/chat_with_pdf/utils/__init__.py new file mode 100644 index 000000000..d55ccad1f --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/utils/__init__.py @@ -0,0 +1 @@ +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/utils/index.py b/chat_with_pdf/flows/standard/chat_with_pdf/utils/index.py new file mode 100644 index 000000000..5120b2d01 --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/utils/index.py @@ -0,0 +1,73 @@ +import os +from typing import Iterable, List, Optional +from dataclasses import dataclass +from faiss import Index +import faiss +import pickle +import numpy as np + +from .oai import OAIEmbedding as Embedding + + +@dataclass +class SearchResultEntity: + text: str = None + vector: List[float] = None + score: float = None + original_entity: dict = None + metadata: dict = None + + +INDEX_FILE_NAME = "index.faiss" +DATA_FILE_NAME = "index.pkl" + + +class FAISSIndex: + def __init__(self, index: Index, embedding: Embedding) -> None: + self.index = index + self.docs = {} # id -> doc, doc is (text, metadata) + self.embedding = embedding + + def insert_batch( + self, texts: Iterable[str], metadatas: Optional[List[dict]] = None + ) -> None: + documents = [] + vectors = [] + for i, text in enumerate(texts): + metadata = metadatas[i] if metadatas else {} + vector = self.embedding.generate(text) + documents.append((text, metadata)) + vectors.append(vector) + + self.index.add(np.array(vectors, dtype=np.float32)) + self.docs.update( + {i: doc for i, doc in enumerate(documents, start=len(self.docs))} + ) + + pass + + def query(self, text: str, top_k: int = 10) -> List[SearchResultEntity]: + vector = self.embedding.generate(text) + scores, indices = self.index.search(np.array([vector], dtype=np.float32), top_k) + docs = [] + for j, i in enumerate(indices[0]): + if i == -1: # This happens when not enough docs are returned. + continue + doc = self.docs[i] + docs.append( + SearchResultEntity(text=doc[0], metadata=doc[1], score=scores[0][j]) + ) + return docs + + def save(self, path: str) -> None: + faiss.write_index(self.index, os.path.join(path, INDEX_FILE_NAME)) + # dump docs to pickle file + with open(os.path.join(path, DATA_FILE_NAME), "wb") as f: + pickle.dump(self.docs, f) + pass + + def load(self, path: str) -> None: + self.index = faiss.read_index(os.path.join(path, INDEX_FILE_NAME)) + with open(os.path.join(path, DATA_FILE_NAME), "rb") as f: + self.docs = pickle.load(f) + pass diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/utils/lock.py b/chat_with_pdf/flows/standard/chat_with_pdf/utils/lock.py new file mode 100644 index 000000000..8b4a4b716 --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/utils/lock.py @@ -0,0 +1,27 @@ +import contextlib +import os +import sys + +if sys.platform.startswith("win"): + import msvcrt +else: + import fcntl + + +@contextlib.contextmanager +def acquire_lock(filename): + if not sys.platform.startswith("win"): + with open(filename, "a+") as f: + fcntl.flock(f, fcntl.LOCK_EX) + yield f + fcntl.flock(f, fcntl.LOCK_UN) + else: # Windows + with open(filename, "w") as f: + msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1) + yield f + msvcrt.locking(f.fileno(), msvcrt.LK_UNLCK, 1) + + try: + os.remove(filename) + except OSError: + pass # best effort to remove the lock file diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/utils/logging.py b/chat_with_pdf/flows/standard/chat_with_pdf/utils/logging.py new file mode 100644 index 000000000..9fca89ceb --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/utils/logging.py @@ -0,0 +1,7 @@ +import os + + +def log(message: str): + verbose = os.environ.get("VERBOSE", "false") + if verbose.lower() == "true": + print(message, flush=True) diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/utils/oai.py b/chat_with_pdf/flows/standard/chat_with_pdf/utils/oai.py new file mode 100644 index 000000000..99db3d098 --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/utils/oai.py @@ -0,0 +1,140 @@ +from typing import List +import openai +from openai.version import VERSION as OPENAI_VERSION +import os +import tiktoken +from jinja2 import Template + +from .retry import ( + retry_and_handle_exceptions, + retry_and_handle_exceptions_for_generator, +) +from .logging import log + + +def extract_delay_from_rate_limit_error_msg(text): + import re + + pattern = r"retry after (\d+)" + match = re.search(pattern, text) + if match: + retry_time_from_message = match.group(1) + return float(retry_time_from_message) + else: + return 5 # default retry time + + +class OAI: + def __init__(self): + if OPENAI_VERSION.startswith("0."): + raise Exception( + "Please upgrade your OpenAI package to version >= 1.0.0 or " + "using the command: pip install --upgrade openai." + ) + init_params = {} + api_type = os.environ.get("OPENAI_API_TYPE") + if os.getenv("OPENAI_API_VERSION") is not None: + init_params["api_version"] = os.environ.get("OPENAI_API_VERSION") + if os.getenv("OPENAI_ORG_ID") is not None: + init_params["organization"] = os.environ.get("OPENAI_ORG_ID") + if os.getenv("OPENAI_API_KEY") is None: + raise ValueError("OPENAI_API_KEY is not set in environment variables") + if os.getenv("OPENAI_API_BASE") is not None: + if api_type == "azure": + init_params["azure_endpoint"] = os.environ.get("OPENAI_API_BASE") + else: + init_params["base_url"] = os.environ.get("OPENAI_API_BASE") + + init_params["api_key"] = os.environ.get("OPENAI_API_KEY") + + # A few sanity checks + if api_type == "azure": + if init_params.get("azure_endpoint") is None: + raise ValueError( + "OPENAI_API_BASE is not set in environment variables, this is required when api_type==azure" + ) + if init_params.get("api_version") is None: + raise ValueError( + "OPENAI_API_VERSION is not set in environment variables, this is required when api_type==azure" + ) + if init_params["api_key"].startswith("sk-"): + raise ValueError( + "OPENAI_API_KEY should not start with sk- when api_type==azure, " + "are you using openai key by mistake?" + ) + from openai import AzureOpenAI as Client + else: + from openai import OpenAI as Client + self.client = Client(**init_params) + + +class OAIChat(OAI): + @retry_and_handle_exceptions( + exception_to_check=( + openai.RateLimitError, + openai.APIStatusError, + openai.APIConnectionError, + KeyError, + ), + max_retries=5, + extract_delay_from_error_message=extract_delay_from_rate_limit_error_msg, + ) + def generate(self, messages: list, **kwargs) -> List[float]: + # chat api may return message with no content. + message = self.client.chat.completions.create( + model=os.environ.get("CHAT_MODEL_DEPLOYMENT_NAME"), + messages=messages, + **kwargs, + ).choices[0].message + return getattr(message, "content", "") + + @retry_and_handle_exceptions_for_generator( + exception_to_check=( + openai.RateLimitError, + openai.APIStatusError, + openai.APIConnectionError, + KeyError, + ), + max_retries=5, + extract_delay_from_error_message=extract_delay_from_rate_limit_error_msg, + ) + def stream(self, messages: list, **kwargs): + response = self.client.chat.completions.create( + model=os.environ.get("CHAT_MODEL_DEPLOYMENT_NAME"), + messages=messages, + stream=False, + **kwargs, + ) + + return response.choices[0].message.content + + +class OAIEmbedding(OAI): + @retry_and_handle_exceptions( + exception_to_check=openai.RateLimitError, + max_retries=5, + extract_delay_from_error_message=extract_delay_from_rate_limit_error_msg, + ) + def generate(self, text: str) -> List[float]: + return self.client.embeddings.create( + input=text, model=os.environ.get("EMBEDDING_MODEL_DEPLOYMENT_NAME") + ).data[0].embedding + + +def count_token(text: str) -> int: + encoding = tiktoken.get_encoding("cl100k_base") + return len(encoding.encode(text)) + + +def render_with_token_limit(template: Template, token_limit: int, **kwargs) -> str: + text = template.render(**kwargs) + token_count = count_token(text) + if token_count > token_limit: + message = f"token count {token_count} exceeds limit {token_limit}" + log(message) + raise ValueError(message) + return text + + +if __name__ == "__main__": + print(count_token("hello world, this is impressive")) diff --git a/chat_with_pdf/flows/standard/chat_with_pdf/utils/retry.py b/chat_with_pdf/flows/standard/chat_with_pdf/utils/retry.py new file mode 100644 index 000000000..9b159b004 --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf/utils/retry.py @@ -0,0 +1,92 @@ +from typing import Tuple, Union, Optional, Type +import functools +import time +import random + + +def retry_and_handle_exceptions( + exception_to_check: Union[Type[Exception], Tuple[Type[Exception], ...]], + max_retries: int = 3, + initial_delay: float = 1, + exponential_base: float = 2, + jitter: bool = False, + extract_delay_from_error_message: Optional[any] = None, +): + def deco_retry(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + delay = initial_delay + for i in range(max_retries): + try: + return func(*args, **kwargs) + except exception_to_check as e: + if i == max_retries - 1: + raise Exception( + "Func execution failed after {0} retries: {1}".format( + max_retries, e + ) + ) + delay *= exponential_base * (1 + jitter * random.random()) + delay_from_error_message = None + if extract_delay_from_error_message is not None: + delay_from_error_message = extract_delay_from_error_message( + str(e) + ) + final_delay = ( + delay_from_error_message if delay_from_error_message else delay + ) + print( + "Func execution failed. Retrying in {0} seconds: {1}".format( + final_delay, e + ) + ) + time.sleep(final_delay) + + return wrapper + + return deco_retry + + +def retry_and_handle_exceptions_for_generator( + exception_to_check: Union[Type[Exception], Tuple[Type[Exception], ...]], + max_retries: int = 3, + initial_delay: float = 1, + exponential_base: float = 2, + jitter: bool = False, + extract_delay_from_error_message: Optional[any] = None, +): + def deco_retry(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + delay = initial_delay + for i in range(max_retries): + try: + for value in func(*args, **kwargs): + yield value + break + except exception_to_check as e: + if i == max_retries - 1: + raise Exception( + "Func execution failed after {0} retries: {1}".format( + max_retries, e + ) + ) + delay *= exponential_base * (1 + jitter * random.random()) + delay_from_error_message = None + if extract_delay_from_error_message is not None: + delay_from_error_message = extract_delay_from_error_message( + str(e) + ) + final_delay = ( + delay_from_error_message if delay_from_error_message else delay + ) + print( + "Func execution failed. Retrying in {0} seconds: {1}".format( + final_delay, e + ) + ) + time.sleep(final_delay) + + return wrapper + + return deco_retry diff --git a/chat_with_pdf/flows/standard/chat_with_pdf_tool.py b/chat_with_pdf/flows/standard/chat_with_pdf_tool.py new file mode 100644 index 000000000..334755322 --- /dev/null +++ b/chat_with_pdf/flows/standard/chat_with_pdf_tool.py @@ -0,0 +1,37 @@ +from promptflow import tool +from chat_with_pdf.main import chat_with_pdf + + +@tool +def chat_with_pdf_tool(question: str, pdf_url: str, history: list, ready: str): + history = convert_chat_history_to_chatml_messages(history) + + stream, context = chat_with_pdf(question, pdf_url, history) + + answer = "" + for str in stream: + answer = answer + str + "" + + return {"answer": answer, "context": context} + + +def convert_chat_history_to_chatml_messages(history): + messages = [] + for item in history: + messages.append({"role": "user", "content": item["inputs"]["question"]}) + messages.append({"role": "assistant", "content": item["outputs"]["answer"]}) + + return messages + + +def convert_chatml_messages_to_chat_history(messages): + history = [] + for i in range(0, len(messages), 2): + history.append( + { + "inputs": {"question": messages[i]["content"]}, + "outputs": {"answer": messages[i + 1]["content"]}, + } + ) + + return history diff --git a/chat_with_pdf/flows/standard/docker/dockerfile b/chat_with_pdf/flows/standard/docker/dockerfile new file mode 100644 index 000000000..942c8c97e --- /dev/null +++ b/chat_with_pdf/flows/standard/docker/dockerfile @@ -0,0 +1,3 @@ +FROM mcr.microsoft.com/azureml/promptflow/promptflow-runtime:latest +COPY ./requirements.txt . +RUN pip install -r requirements.txt \ No newline at end of file diff --git a/chat_with_pdf/flows/standard/download_tool.py b/chat_with_pdf/flows/standard/download_tool.py new file mode 100644 index 000000000..72baa90fa --- /dev/null +++ b/chat_with_pdf/flows/standard/download_tool.py @@ -0,0 +1,7 @@ +from promptflow import tool +from chat_with_pdf.download import download + + +@tool +def download_tool(url: str, env_ready_signal: str) -> str: + return download(url) diff --git a/chat_with_pdf/flows/standard/eval_run.yaml b/chat_with_pdf/flows/standard/eval_run.yaml new file mode 100644 index 000000000..526c01446 --- /dev/null +++ b/chat_with_pdf/flows/standard/eval_run.yaml @@ -0,0 +1,8 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Run.schema.json +#name: eval_groundedness_default_20230820_200152_009000 +flow: ../../evaluation/eval-groundedness +run: chat_with_pdf_default_20230820_162219_559000 +column_mapping: + question: ${run.inputs.question} + answer: ${run.outputs.answer} + context: ${run.outputs.context} \ No newline at end of file diff --git a/chat_with_pdf/flows/standard/find_context_tool.py b/chat_with_pdf/flows/standard/find_context_tool.py new file mode 100644 index 000000000..246ceea2b --- /dev/null +++ b/chat_with_pdf/flows/standard/find_context_tool.py @@ -0,0 +1,9 @@ +from promptflow import tool +from chat_with_pdf.find_context import find_context + + +@tool +def find_context_tool(question: str, index_path: str): + prompt, context = find_context(question, index_path) + + return {"prompt": prompt, "context": [c.text for c in context]} diff --git a/chat_with_pdf/flows/standard/flow.dag.yaml b/chat_with_pdf/flows/standard/flow.dag.yaml new file mode 100644 index 000000000..db6d36158 --- /dev/null +++ b/chat_with_pdf/flows/standard/flow.dag.yaml @@ -0,0 +1,81 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json +environment: + python_requirements_txt: requirements.txt +inputs: + chat_history: + type: list + default: [] + pdf_url: + type: string + default: https://arxiv.org/pdf/1810.04805.pdf + question: + type: string + is_chat_input: true + default: what is BERT? + config: + type: object + default: + EMBEDDING_MODEL_DEPLOYMENT_NAME: text-embedding-ada-002 + CHAT_MODEL_DEPLOYMENT_NAME: aoai + PROMPT_TOKEN_LIMIT: 3000 + MAX_COMPLETION_TOKENS: 1024 + VERBOSE: true + CHUNK_SIZE: 1024 + CHUNK_OVERLAP: 64 +outputs: + answer: + type: string + is_chat_output: true + reference: ${qna_tool.output.answer} + context: + type: string + reference: ${find_context_tool.output.context} +nodes: +- name: setup_env + type: python + source: + type: code + path: setup_env.py + inputs: + connection: aoai + config: ${inputs.config} +- name: download_tool + type: python + source: + type: code + path: download_tool.py + inputs: + url: ${inputs.pdf_url} + env_ready_signal: ${setup_env.output} +- name: build_index_tool + type: python + source: + type: code + path: build_index_tool.py + inputs: + pdf_path: ${download_tool.output} +- name: find_context_tool + type: python + source: + type: code + path: find_context_tool.py + inputs: + question: ${rewrite_question_tool.output} + index_path: ${build_index_tool.output} +- name: qna_tool + type: python + source: + type: code + path: qna_tool.py + inputs: + prompt: ${find_context_tool.output.prompt} + history: ${inputs.chat_history} +- name: rewrite_question_tool + type: python + source: + type: code + path: rewrite_question_tool.py + inputs: + question: ${inputs.question} + history: ${inputs.chat_history} + env_ready_signal: ${setup_env.output} diff --git a/chat_with_pdf/flows/standard/openai.yaml b/chat_with_pdf/flows/standard/openai.yaml new file mode 100644 index 000000000..64de0e2fe --- /dev/null +++ b/chat_with_pdf/flows/standard/openai.yaml @@ -0,0 +1,13 @@ +# All the values should be string type, please use "123" instead of 123 or "True" instead of True. +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/OpenAIConnection.schema.json +name: open_ai_connection +type: open_ai +api_key: "" +organization: "" + +# Note: +# The connection information will be stored in a local database with api_key encrypted for safety. +# Prompt flow will ONLY use the connection information (incl. keys) when instructed by you, e.g. manage connections, use connections to run flow etc. + + + diff --git a/chat_with_pdf/flows/standard/qna_tool.py b/chat_with_pdf/flows/standard/qna_tool.py new file mode 100644 index 000000000..98e131b75 --- /dev/null +++ b/chat_with_pdf/flows/standard/qna_tool.py @@ -0,0 +1,22 @@ +from promptflow import tool +from chat_with_pdf.qna import qna + + +@tool +def qna_tool(prompt: str, history: list): + stream = qna(prompt, convert_chat_history_to_chatml_messages(history)) + + answer = "" + for str in stream: + answer = answer + str + "" + + return {"answer": answer} + + +def convert_chat_history_to_chatml_messages(history): + messages = [] + for item in history: + messages.append({"role": "user", "content": item["inputs"]["question"]}) + messages.append({"role": "assistant", "content": item["outputs"]["answer"]}) + + return messages diff --git a/chat_with_pdf/flows/standard/requirements.txt b/chat_with_pdf/flows/standard/requirements.txt new file mode 100644 index 000000000..302388869 --- /dev/null +++ b/chat_with_pdf/flows/standard/requirements.txt @@ -0,0 +1,9 @@ +PyPDF2 +faiss-cpu +openai +jinja2 +python-dotenv +tiktoken +promptflow==1.9.0 +promptflow[azure]==1.9.0 +promptflow-tools \ No newline at end of file diff --git a/chat_with_pdf/flows/standard/rewrite_question_tool.py b/chat_with_pdf/flows/standard/rewrite_question_tool.py new file mode 100644 index 000000000..8808c4a00 --- /dev/null +++ b/chat_with_pdf/flows/standard/rewrite_question_tool.py @@ -0,0 +1,7 @@ +from promptflow import tool +from chat_with_pdf.rewrite_question import rewrite_question + + +@tool +def rewrite_question_tool(question: str, history: list, env_ready_signal: str): + return rewrite_question(question, history) diff --git a/chat_with_pdf/flows/standard/setup_env.py b/chat_with_pdf/flows/standard/setup_env.py new file mode 100644 index 000000000..6b231b878 --- /dev/null +++ b/chat_with_pdf/flows/standard/setup_env.py @@ -0,0 +1,37 @@ +import os +from typing import Union + +from promptflow import tool +from promptflow.connections import AzureOpenAIConnection, OpenAIConnection + +from chat_with_pdf.utils.lock import acquire_lock + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + "/chat_with_pdf/" + + +@tool +def setup_env(connection: Union[AzureOpenAIConnection, OpenAIConnection], config: dict): + if not connection or not config: + return + + if isinstance(connection, AzureOpenAIConnection): + os.environ["OPENAI_API_TYPE"] = "azure" + os.environ["OPENAI_API_BASE"] = connection.api_base + os.environ["OPENAI_API_KEY"] = connection.api_key + os.environ["OPENAI_API_VERSION"] = connection.api_version + + if isinstance(connection, OpenAIConnection): + os.environ["OPENAI_API_KEY"] = connection.api_key + if connection.organization is not None: + os.environ["OPENAI_ORG_ID"] = connection.organization + + for key in config: + os.environ[key] = str(config[key]) + + with acquire_lock(BASE_DIR + "create_folder.lock"): + if not os.path.exists(BASE_DIR + ".pdfs"): + os.mkdir(BASE_DIR + ".pdfs") + if not os.path.exists(BASE_DIR + ".index/.pdfs"): + os.makedirs(BASE_DIR + ".index/.pdfs") + + return "Ready" diff --git a/chat_with_pdf/llmops_config.yaml b/chat_with_pdf/llmops_config.yaml new file mode 100644 index 000000000..85994b30f --- /dev/null +++ b/chat_with_pdf/llmops_config.yaml @@ -0,0 +1,108 @@ +azure_config: + subscription_id: ${SUBSCRIPTION_ID} + resource_group_name: ${RESOURCE_GROUP_NAME} + workspace_name: ${WORKSPACE_NAME} + keyvault_name: ${KEYVAULT_NAME} + compute_target: ${COMPUTE_TARGET} + +connections: +- connection: aoai + api_type: azure + api_key: ${AZURE_OPENAI_KEY} + api_base: ${AZURE_OPENAI_ENDPOINT} + api_version: ${AZURE_OPENAI_API_VERSION} + +experiment: + name: chat_with_pdf + flow: flows/standard + + datasets: + - name: chat_with_pdf + source: data/bert-paper-qna-3-line.jsonl + description: "This dataset is for prompt experiments." + mappings: + math_question: "${data.question}" + + evaluators: + - name: chat_with_pdf_evaluation_flow + flow: flows/evaluation + + datasets: + - name: chat_with_pdf_test + reference: chat_with_pdf + source: data/data.jsonl + description: "This dataset is for evaluating flows." + mappings: + groundtruth: "${data.groundtruth}" + prediction: "${run.outputs.answer}" + +environments: + pr: + env_name: pr + + experiment: + name: standard + + datasets: + - name: chat_with_pdf_pr + source: data/bert-paper-qna-1-line.jsonl + description: "This dataset is for pr validation only." + mappings: + math_question: "${data.question}" + + evaluators: + + dev: + env_name: dev + + experiment: + + deployment_configs: + azure_managed_endpoint: + - name: azure_managed_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + ENDPOINT_NAME: + ENDPOINT_DESC: An online endpoint serving a flow for chat with pdf flow + DEPLOYMENT_DESC: prompt flow deployment + PRIOR_DEPLOYMENT_NAME: + PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION: + CURRENT_DEPLOYMENT_NAME: + CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION: "100" + DEPLOYMENT_VM_SIZE: Standard_F4s_v2 + DEPLOYMENT_INSTANCE_COUNT: 1 + ENVIRONMENT_VARIABLES: + example-name: example-value + + kubernetes_endpoint: + - name: kubernetes_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + ENDPOINT_NAME: + ENDPOINT_DESC: An kubernetes endpoint serving a flow for chat with pdf + DEPLOYMENT_DESC: prompt flow deployment + PRIOR_DEPLOYMENT_NAME: + PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION: + CURRENT_DEPLOYMENT_NAME: + CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION: 100, + COMPUTE_NAME: + DEPLOYMENT_VM_SIZE: + DEPLOYMENT_INSTANCE_COUNT: 1 + CPU_ALLOCATION: + MEMORY_ALLOCATION: + ENVIRONMENT_VARIABLES: + example-name: example-value + + webapp_endpoint: + - name: webapp_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + CONNECTION_NAMES: + - aoai + REGISTRY_NAME: + REGISTRY_RG_NAME: + APP_PLAN_NAME: + WEB_APP_NAME: + WEB_APP_RG_NAME: + WEB_APP_SKU: "B3" + USER_MANAGED_ID: \ No newline at end of file diff --git a/chat_with_pdf/sample-request.json b/chat_with_pdf/sample-request.json new file mode 100644 index 000000000..8ba444118 --- /dev/null +++ b/chat_with_pdf/sample-request.json @@ -0,0 +1 @@ +{"pdf_url":"https://arxiv.org/pdf/1810.04805.pdf", "chat_history":[], "question": "What is the name of the new language representation model introduced in the document?", "answer": "BERT", "context": "We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers."} \ No newline at end of file diff --git a/chat_with_pdf/tests/test_delete_this_file.py b/chat_with_pdf/tests/test_delete_this_file.py new file mode 100644 index 000000000..3692b2a7b --- /dev/null +++ b/chat_with_pdf/tests/test_delete_this_file.py @@ -0,0 +1,6 @@ +def test_print(): + try: + print("Hello") is None + except: + print("Test print function failed.") + assert False \ No newline at end of file diff --git a/dataops/__init__.py b/dataops/__init__.py new file mode 100644 index 000000000..cfb6b7af7 --- /dev/null +++ b/dataops/__init__.py @@ -0,0 +1,4 @@ +""" +dataops module. + +""" diff --git a/dataops/common/__init__.py b/dataops/common/__init__.py new file mode 100644 index 000000000..863836886 --- /dev/null +++ b/dataops/common/__init__.py @@ -0,0 +1,4 @@ +""" +common module. + +""" diff --git a/dataops/common/aml_data_asset.py b/dataops/common/aml_data_asset.py new file mode 100644 index 000000000..058725181 --- /dev/null +++ b/dataops/common/aml_data_asset.py @@ -0,0 +1,139 @@ +""" +This module creates the data assets. +""" +from azure.identity import DefaultAzureCredential +from azure.ai.ml.entities import Data +from azure.ai.ml import MLClient +from azure.ai.ml.constants import AssetTypes +import os +import argparse +import json + +pipeline_components = [] + +""" +This function creates and returns an Azure Machine Learning (AML) client. +The AML client is used to interact with Azure Machine Learning services. + +Args: +--subscription_id: The Azure subscription ID. +This argument is required for identifying the Azure subscription. +--resource_group_name: The name of the resource group in Azure. +This argument is required to specify the resource group in Azure. +--workspace_name: The name of the workspace in Azure Machine Learning. +This argument is required to specify the workspace in Azure Machine Learning. +""" + + +def get_aml_client( + subscription_id, + resource_group_name, + workspace_name, +): + aml_client = MLClient( + DefaultAzureCredential(), + subscription_id=subscription_id, + resource_group_name=resource_group_name, + workspace_name=workspace_name, + ) + + return aml_client + + +""" +This function registers a data asset in Azure Machine Learning. +The data asset is identified by its name and description, and is associated with a specific data store and file path. + +Args: +--name: The name of the data asset. +This argument is required to specify the name of the data asset. +--description: The description of the data asset. +This argument is required to provide a description of the data asset. +--aml_client: The Azure Machine Learning client. +This argument is required to interact with Azure Machine Learning services. +--data_store: The name of the data store in Azure. +This argument is required to specify the data store in Azure. +--file_path: The file path of the data asset in the data store. +This argument is required to specify the file path of the data asset in the data store. +""" + + +def register_data_asset( + name, + description, + aml_client, + data_store, + file_path +): + target_path = f"azureml://datastores/{data_store}/paths/{file_path}" + aml_dataset = Data( + path=target_path, + type=AssetTypes.URI_FILE, + description=description, + name=name + ) + + aml_client.data.create_or_update(aml_dataset) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--subscription_id", + type=str, + help="Azure subscription id", + required=True, + ) + parser.add_argument( + "--resource_group_name", + type=str, + help="Azure resource group", + required=True, + ) + parser.add_argument( + "--workspace_name", + type=str, + help="Azure ML workspace", + required=True, + ) + parser.add_argument( + "--config_path_root_dir", + type=str, + help="Root dir for config file", + required=True, + ) + + args = parser.parse_args() + + subscription_id = args.subscription_id + resource_group_name = args.resource_group_name + workspace_name = args.workspace_name + config_path_root_dir = args.config_path_root_dir + + config_path = os.path.join(os.getcwd(), f"{config_path_root_dir}/configs/dataops_config.json") + config = json.load(open(config_path)) + + aml_client = get_aml_client( + subscription_id, + resource_group_name, + workspace_name, + ) + + data_store = config["DATA_STORE_NAME"] + data_asset_configs = config['DATA_ASSETS'] + for data_asset_config in data_asset_configs: + data_asset_name = data_asset_config['NAME'] + data_asset_file_path = data_asset_config['PATH'] + data_asset_description = data_asset_config['DESCRIPTION'] + + register_data_asset( + name=data_asset_name, + description=data_asset_description, + aml_client=aml_client, + data_store=data_store, + file_path=data_asset_file_path + ) + + +if __name__ == "__main__": + main() diff --git a/dataops/common/aml_data_store.py b/dataops/common/aml_data_store.py new file mode 100644 index 000000000..ec7111c44 --- /dev/null +++ b/dataops/common/aml_data_store.py @@ -0,0 +1,146 @@ +""" +This module registers the data store. +""" +from azure.ai.ml import MLClient +from azure.identity import DefaultAzureCredential +from azure.ai.ml.entities import AzureBlobDatastore, AccountKeyConfiguration +import os +import argparse +import json + +pipeline_components = [] +""" +This function creates and returns an Azure Machine Learning (AML) client. +The AML client is used to interact with Azure Machine Learning services. + +Args: +--subscription_id: The Azure subscription ID. +This argument is required for identifying the Azure subscription. +--resource_group_name: The name of the resource group in Azure. +This argument is required to specify the resource group in Azure. +--workspace_name: The name of the workspace in Azure Machine Learning. +This argument is required to specify the workspace in Azure Machine Learning. +""" + + +def get_aml_client( + subscription_id, + resource_group_name, + workspace_name, +): + aml_client = MLClient( + DefaultAzureCredential(), + subscription_id=subscription_id, + resource_group_name=resource_group_name, + workspace_name=workspace_name, + ) + + return aml_client + + +""" +This function registers a data store in Azure Machine Learning. +The data store is identified by its name and description, +and is associated with a specific storage account and container. + +Args: +--name_datastore: The name of the data store. +This argument is required to specify the name of the data store. +--description: The description of the data store. +This argument is required to provide a description of the data store. +--sa_account_name: The name of the storage account in Azure. +This argument is required to specify the storage account in Azure. +--sa_container_name: The name of the container in the storage account. +This argument is required to specify the container in the storage account. +--sa_key: The key of the storage account. +This argument is required to authenticate with the storage account. +--aml_client: The Azure Machine Learning client. +This argument is required to interact with Azure Machine Learning services. +""" + + +def register_data_store( + name_datastore, + description, + sa_account_name, + sa_container_name, + sa_key, + aml_client +): + store = AzureBlobDatastore( + name=name_datastore, + description=description, + account_name=sa_account_name, + container_name=sa_container_name, + credentials=AccountKeyConfiguration(account_key=sa_key) + ) + aml_client.create_or_update(store) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--subscription_id", + type=str, + help="Azure subscription id", + required=True, + ) + parser.add_argument( + "--resource_group_name", + type=str, + help="Azure resource group", + required=True, + ) + parser.add_argument( + "--workspace_name", + type=str, + help="Azure ML workspace", + required=True, + ) + parser.add_argument( + "--sa_key", + type=str, + help="Storage account key", + required=True, + ) + parser.add_argument( + "--config_path_root_dir", + type=str, + help="Root dir for config file", + required=True, + ) + + args = parser.parse_args() + + subscription_id = args.subscription_id + resource_group_name = args.resource_group_name + workspace_name = args.workspace_name + sa_key = args.sa_key + config_path_root_dir = args.config_path_root_dir + + config_path = os.path.join(os.getcwd(), + f"{config_path_root_dir}/configs/dataops_config.json") + config = json.load(open(config_path)) + + aml_client = get_aml_client( + subscription_id, + resource_group_name, + workspace_name, + ) + + storage_config = config['STORAGE'] + storage_account = storage_config['STORAGE_ACCOUNT'] + target_container_name = storage_config['TARGET_CONTAINER'] + + register_data_store( + name_datastore=config["DATA_STORE_NAME"], + description=config["DATA_STORE_DESCRIPTION"], + sa_account_name=storage_account, + sa_container_name=target_container_name, + sa_key=sa_key, + aml_client=aml_client + ) + + +if __name__ == "__main__": + main() diff --git a/dataops/common/aml_pipeline.py b/dataops/common/aml_pipeline.py new file mode 100644 index 000000000..049bca6cb --- /dev/null +++ b/dataops/common/aml_pipeline.py @@ -0,0 +1,365 @@ +""" +This module creates a AML job and schedule it for the data pipeline. +""" +from datetime import datetime +from azure.ai.ml.dsl import pipeline +from azure.identity import DefaultAzureCredential +from azure.ai.ml import command, UserIdentityConfiguration +from azure.ai.ml import Output +from azure.ai.ml import MLClient +from azure.ai.ml.entities import ( + JobSchedule, + CronTrigger +) +import os +import argparse +import json + +pipeline_components = [] + +() + +""" +This function defines a AML pipeline for data preparation in Named Entity Recognition (NER) tasks. +The pipeline is identified by its name and description, and consists of a data preparation job. + +The data preparation job is the first component in the pipeline components list. +The output of the data preparation job is a target directory, which is returned by the pipeline. + +Decorator: +@pipeline: A decorator to declare this function as a pipeline. +It takes two arguments - name and description of the pipeline. + +Returns: +A dictionary with the target directory as the output of the data preparation job. +""" + + +@pipeline( + name="ner_data_prep_test", + description="data prep pipeline", +) +def ner_data_prep_pipeline( +): + prep_data_job = pipeline_components[0]( + ) + + return { + "target_dir": prep_data_job.outputs.target_dir + } + + +""" +This function executes a data preparation component for a data pipeline. +The data component is identified by its name, display name, +and description, and is associated with a specific environment, storage account, +source and target containers, source blob, assets, and custom compute. + +Args: +--name: The name of the data component. +This argument is required to specify the name of the data component. +--display_name: The display name of the data component. +This argument is required to specify the display name of the data component. +--description: The description of the data component. +This argument is required to provide a description of the data component. +--data_pipeline_code_dir: The directory of the data pipeline code. +This argument is required to specify the directory of the data pipeline code. +--environment: The environment for the data component. +This argument is required to specify the environment for the data component. +--storage_account: The storage account in Azure. +This argument is required to specify the storage account in Azure. +--source_container_name: The name of the source container in the storage account. +This argument is required to specify the source container in the storage account. +--target_container_name: The name of the target container in the storage account. +This argument is required to specify the target container in the storage account. +--source_blob: The name of the source blob in the source container. +This argument is required to specify the source blob in the source container. +--assets: The assets in the target container. +This argument is required to specify the assets in the target container. +--custom_compute: The custom compute for the data component. +This argument is required to specify the custom compute for the data component. +""" + + +def get_prep_data_component( + name, + display_name, + description, + data_pipeline_code_dir, + environment, + storage_account, + source_container_name, + target_container_name, + source_blob, + assets, + custom_compute +): + data_pipeline_code_dir = os.path.join(os.getcwd(), data_pipeline_code_dir) + + # Initialize an empty list to store components + prep_data_components = [] + asset_str = ":".join(map(str, assets)) + + prep_data_component = command( + name=name, + display_name=display_name, + description=description, + inputs={}, + outputs=dict( + target_dir=Output(type="uri_folder", mode="rw_mount"), + ), + code=data_pipeline_code_dir, + command=f"""python prep_data.py \ + --storage_account {storage_account} \ + --source_container_name {source_container_name} \ + --target_container_name {target_container_name} \ + --source_blob {source_blob} \ + --assets_str {asset_str} + """, + environment=environment, + compute=custom_compute, + identity=UserIdentityConfiguration() + ) + prep_data_components.append(prep_data_component) + + return prep_data_components + + +""" +This function creates and returns an Azure Machine Learning (AML) client. +The AML client is used to interact with Azure Machine Learning services. + +Args: +--subscription_id: The Azure subscription ID. +This argument is required for identifying the Azure subscription. +--resource_group_name: The name of the resource group in Azure. +This argument is required to specify the resource group in Azure. +--workspace_name: The name of the workspace in Azure Machine Learning. +This argument is required to specify the workspace in Azure Machine Learning. +""" + + +def get_aml_client( + subscription_id, + resource_group_name, + workspace_name, +): + aml_client = MLClient( + DefaultAzureCredential(), + subscription_id=subscription_id, + resource_group_name=resource_group_name, + workspace_name=workspace_name, + ) + + return aml_client + + +""" +This function creates a pipeline job with a data component. +The pipeline job is associated with a specific component name, display name, +description, data pipeline, code directory, environment, storage account +source and target containers, source blob, assets, and custom compute. + +Args: +--component_name: The name of the data component. +This argument is required to specify the name of the data component. +--component_display_name: The display name of the data component. +This argument is required to specify the display name of the data component. +--component_description: The description of the data component. +This argument is required to provide a description of the data component. +--data_pipeline_code_dir: The directory of the data pipeline code. +This argument is required to specify the directory of the data pipeline code. +--aml_env_name: The name of the Azure Machine Learning environment. +This argument is required to specify the Azure Machine Learning environment. +--storage_account: The storage account in Azure. +This argument is required to specify the storage account in Azure. +--source_container_name: The name of the source container in the storage account. +This argument is required to specify the source container in the storage account. +--target_container_name: The name of the target container in the storage account. +This argument is required to specify the target container in the storage account. +--source_blob: The name of the source blob in the source container. +This argument is required to specify the source blob in the source container. +--assets: The assets in the target container. +This argument is required to specify the assets in the target container. +--custom_compute: The custom compute for the data component. +This argument is required to specify the custom compute for the data component. +""" + + +def create_pipeline_job( + component_name, + component_display_name, + component_description, + data_pipeline_code_dir, + aml_env_name, + storage_account, + source_container_name, + target_container_name, + source_blob, + assets, + custom_compute +): + prep_data_component = get_prep_data_component( + name=component_name, + display_name=component_display_name, + description=component_description, + data_pipeline_code_dir=data_pipeline_code_dir, + environment=aml_env_name, + storage_account=storage_account, + source_container_name=source_container_name, + target_container_name=target_container_name, + source_blob=source_blob, + assets=assets, + custom_compute=custom_compute + ) + + pipeline_components.extend(prep_data_component) + + pipeline_job = ner_data_prep_pipeline() + + return pipeline_job + + +""" +This function schedules a pipeline job. +The schedule is identified by its name, cron expression, and timezone, +and is associated with a specific job and Azure Machine Learning client. + +Args: +--schedule_name: The name of the schedule. +This argument is required to specify the name of the schedule. +--schedule_cron_expression: The cron expression for the schedule. +This argument is required to specify the cron expression for the schedule. +--schedule_timezone: The timezone for the schedule. +This argument is required to specify the timezone for the schedule. +--job: The job for the schedule. +This argument is required to specify the job for the schedule. +--aml_client: The Azure Machine Learning client. +This argument is required to interact with Azure Machine Learning services. +""" + + +def schedule_pipeline_job( + schedule_name, + schedule_cron_expression, + schedule_timezone, + job, + aml_client, +): + schedule_start_time = datetime.utcnow() + cron_trigger = CronTrigger( + expression=schedule_cron_expression, + start_time=schedule_start_time, + time_zone=schedule_timezone + ) + + job_schedule = JobSchedule( + name=schedule_name, trigger=cron_trigger, create_job=job + ) + + aml_client.schedules.begin_create_or_update( + schedule=job_schedule + ).result() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--subscription_id", + type=str, + help="Azure subscription id", + required=True, + ) + parser.add_argument( + "--resource_group_name", + type=str, + help="Azure resource group", + required=True, + ) + parser.add_argument( + "--workspace_name", + type=str, + help="Azure ML workspace", + required=True, + ) + parser.add_argument( + "--aml_env_name", + type=str, + help="Azure environment name", + required=True, + ) + parser.add_argument( + "--config_path_root_dir", + type=str, + help="Root dir for config file", + required=True, + ) + + args = parser.parse_args() + + subscription_id = args.subscription_id + resource_group_name = args.resource_group_name + workspace_name = args.workspace_name + aml_env_name = args.aml_env_name + config_path_root_dir = args.config_path_root_dir + + config_path = os.path.join(os.getcwd(), f"{config_path_root_dir}/configs/dataops_config.json") + config = json.load(open(config_path)) + + component_config = config['DATA_PREP_COMPONENT'] + component_name = component_config['COMPONENT_NAME'] + component_display_name = component_config['COMPONENT_DISPLAY_NAME'] + component_description = component_config['COMPONENT_DESCRIPTION'] + + storage_config = config['STORAGE'] + storage_account = storage_config['STORAGE_ACCOUNT'] + source_container_name = storage_config['SOURCE_CONTAINER'] + source_blob = storage_config['SOURCE_BLOB'] + target_container_name = storage_config['TARGET_CONTAINER'] + + path_config = config['PATH'] + data_pipeline_code_dir = path_config['DATA_PIPELINE_CODE_DIR'] + + schedule_config = config['SCHEDULE'] + schedule_name = schedule_config['NAME'] + schedule_cron_expression = schedule_config['CRON_EXPRESSION'] + schedule_timezone = schedule_config['TIMEZONE'] + + data_asset_configs = config['DATA_ASSETS'] + assets = [] + for data_asset_config in data_asset_configs: + assets.append(data_asset_config['PATH']) + + custom_compute = config["COMPUTE_NAME"] + + aml_client = get_aml_client( + subscription_id, + resource_group_name, + workspace_name, + ) + + job = create_pipeline_job( + component_name, + component_display_name, + component_description, + data_pipeline_code_dir, + aml_env_name, + storage_account, + source_container_name, + target_container_name, + source_blob, + assets, + custom_compute + ) + + schedule_pipeline_job( + schedule_name, + schedule_cron_expression, + schedule_timezone, + job, + aml_client + ) + + +if __name__ == "__main__": + main() diff --git a/docs/Azure_devops_how_to_setup.md b/docs/Azure_devops_how_to_setup.md index bbba4ecc0..1bbd49c74 100644 --- a/docs/Azure_devops_how_to_setup.md +++ b/docs/Azure_devops_how_to_setup.md @@ -244,6 +244,7 @@ Create a new variable group `llmops_platform_dev_vg` ([follow the documentation] - **rg_name**: Name of the resource group containing the Azure ML Workspace - **ws_name**: Name of the Azure ML Workspace - **kv_name**: Name of the Key Vault associated with the Azure ML Workspace +- **COMPUTE_TARGET**: Name of the compute cluster used in the Azure ML Workspace (Note: this is only needed if you are executing the Promptflow in AML Pipeline) ![Variable group](./images/variable-group.png) @@ -326,9 +327,10 @@ As a result the code for LLMOps Prompt flow template will now be available in Az 6. Create two Azure Pipelines [[how to create a basic Azure Pipeline](https://learn.microsoft.com/en-us/azure/devops/pipelines/create-first-pipeline?view=azure-devops&tabs)] for each scenario (e.g. named_entity_recognition). Both Azure Pipelines should be created based on existing YAML files: -- The first one is based on the [named_entity_recognition_pr_dev_pipeline.yml](../named_entity_recognition/.azure-pipelines/named_entity_recognition_pr_dev_pipeline.yml), and it helps to maintain code quality for all PRs including integration tests for the Azure ML experiment. Usually, we recommend to have a toy dataset for the integration tests to make sure that the Prompt flow job can be completed fast enough - there is not a goal to check prompt quality and we just need to make sure that our job can be executed. + - The first one is based on the [named_entity_recognition_pr_dev_pipeline.yml](../named_entity_recognition/.azure-pipelines/named_entity_recognition_pr_dev_pipeline.yml), and it helps to maintain code quality for all PRs including integration tests for the Azure ML experiment. Usually, we recommend to have a toy dataset for the integration tests to make sure that the Prompt flow job can be completed fast enough - there is not a goal to check prompt quality and we just need to make sure that our job can be executed. + + - The second Azure Pipeline is based on [named_entity_recognition_ci_dev_pipeline.yml](../named_entity_recognition/.azure-pipelines/named_entity_recognition_ci_dev_pipeline.yml) is executed automatically once new PR has been merged into the *development* or *main* branch. The main idea of this pipeline is to execute bulk run, evaluation on the full dataset for all prompt variants. Both the workflow can be modified and extended based on the project's requirements. -- The second Azure Pipeline is based on [named_entity_recognition_ci_dev_pipeline.yml](../named_entity_recognition/.azure-pipelines/named_entity_recognition_ci_dev_pipeline.yml) is executed automatically once new PR has been merged into the *development* or *main* branch. The main idea of this pipeline is to execute bulk run, evaluation on the full dataset for all prompt variants. Both the workflow can be modified and extended based on the project's requirements. These following steps should be executed twice - once for PR pipeline and again for CI pipeline. @@ -372,6 +374,13 @@ From your Azure DevOps project, select `Repos -> Branches -> more options button More details about how to create a policy can be found [here](https://learn.microsoft.com/en-us/azure/devops/repos/git/branch-policies?view=azure-devops&tabs=browser). + +## Steps for executing the Promptflow in AML Pipeline + + There is another azure devops pipeline added :[web_classification_pf_in_aml_pipeline_workflow.yml](../.azure-pipelines/web_classification_pf_in_aml_pipeline_workflow.yml) + - It is used to run the promptflow in AML Pipeline as a parallel component. + - You can use this to run other use cases as well, all you need to do is change the use_case_base_path to other use cases, like math_coding, named_entity_recognition. + ## Test the pipelines From local machine, create a new git branch `featurebranch` from `development` branch. @@ -482,7 +491,12 @@ This Azure DevOps CI pipelines contains the following steps: **Run Prompts in Flow** - Upload bulk run dataset - Bulk run prompt flow based on dataset. -- Bulk run each prompt variant +- Bulk run each prompt variant + +**Run promptflow in AML Pipeline as parallel component** +- It reuses the already registered data assets for input. +- Runs the promptflow in AML Pipeline as a parallel component, where we can control the concurrency and parallelism of the promptflow execution. For more details refer [here](https://microsoft.github.io/promptflow/tutorials/pipeline.html). +- The output of the promptflow is stored in the Azure ML workspace. **Evaluate Results** - Upload ground test dataset @@ -516,4 +530,4 @@ This Azure DevOps CI pipelines contains the following steps: The example scenario can be run and deployed both for Dev environments. When you are satisfied with the performance of the prompt evaluation pipeline, Prompt flow model, and deployment in development, additional pipelines similar to `dev` pipelines can be replicated and deployed in the Production environment. -The sample Prompt flow run & evaluation and Azure DevOps pipelines can be used as a starting point to adapt your own prompt engineering code and data. \ No newline at end of file +The sample Prompt flow run & evaluation and Azure DevOps pipelines can be used as a starting point to adapt your own prompt engineering code and data. diff --git a/docs/experiment.yaml b/docs/experiment.yaml index 017b7b7b1..50d7efdc3 100644 --- a/docs/experiment.yaml +++ b/docs/experiment.yaml @@ -1,3 +1,4 @@ +# experiment_config # # Defines the experiment name which is: # 1. Used as the experiment name of Azure ML jobs. diff --git a/docs/github_workflows_how_to_setup.md b/docs/github_workflows_how_to_setup.md index 8a3dfd8ec..defb40693 100644 --- a/docs/github_workflows_how_to_setup.md +++ b/docs/github_workflows_how_to_setup.md @@ -133,6 +133,8 @@ principalId="$(echo $um_details | jq -r '.[2]')" ```bash az role assignment create --assignee $principalId --role "AzureML Data Scientist" --scope "/subscriptions/$subscriptionId/resourcegroups/$rgname/providers/Microsoft.MachineLearningServices/workspaces/$workspace_name" ``` +You need to give additional `Azure ML Operator` permissions to the user managed identity for accessing the workspace, if you are using promptflow in AML Pipeline. +Note: this will not work in serverless. You shall need a compute cluster. 8. Grant the user managed identity permission to access the workspace keyvault (get and list) @@ -243,7 +245,7 @@ From your GitHub project, select **Settings** -> **Secrets and variables**, ** ## Set up GitHub variables for each environment -There are 3 variables expected as GitHub variables: `RESOURCE_GROUP_NAME`, `WORKSPACE_NAME` and `KEY_VAULT_NAME`. These values are environment specific, so we utilize the `Environments` feature in GitHub. +There are 3 variables expected as GitHub variables: `RESOURCE_GROUP_NAME`, `WORKSPACE_NAME` and `KEY_VAULT_NAME`. These values are environment specific, so we utilize the `Environments` feature in GitHub. An additional variable name `COMPUTE_TARGET` is needed to use promptflow in AML Pipeline. From your GitHub project, select **Settings** -> **Environments**, select "New environment" and call it `dev` ![Screenshot of GitHub environments.](images/github-environments-new-env.png) @@ -274,6 +276,12 @@ The configuration for connection used while authoring the repo: ![connection details](images/connection-details.png) +## Steps for executing the Promptflow in AML Pipeline + There is another github workflow added [web_classification_pf_in_aml_pipeline_workflow.yml](../.github/workflows/web_classification_pf_in_aml_pipeline_workflow.yml) peline. + - It is used to run the promptflow in AML Pipeline as a parallel component. + - You can use this to run other use cases as well, all you need to do is change the use_case_base_path to other use cases, like math_coding, named_entity_recognition. + + ## Set up Secrets in GitHub ### Prompt flow Connection @@ -462,6 +470,11 @@ This Github CI workflow contains the following steps: - Execute the evaluation flow on the production log dataset - Generate the evaluation report +**Run promptflow in AML Pipeline as parallel component** +- It reuses the already registered data assets for input. +- Runs the promptflow in AML Pipeline as a parallel component, where we can control the concurrency and parallelism of the promptflow execution. For more details refer [here](https://microsoft.github.io/promptflow/tutorials/pipeline.html). +- The output of the promptflow is stored in the Azure ML workspace. + ### Online Endpoint 1. After the CI pipeline for an example scenario has run successfully, depending on the configuration it will either deploy to diff --git a/docs/how_to_configure_dataops.md b/docs/how_to_configure_dataops.md new file mode 100644 index 000000000..f56b13cbd --- /dev/null +++ b/docs/how_to_configure_dataops.md @@ -0,0 +1,39 @@ +# How to Configure DataOps + +Implementing the DataOps pattern will help manage and scale the data pipelines. The following sections will explain the necessary steps to integrate DataOps into the LLMOps pattern. + +## Prerequisites + +This document assumes that you have already gone through [How to Onboard new flow](./how_to_onboard_new_flows.md) and implemented the steps. Once you have all the components from the document in place, you can start setting up DataOps. + +**Data Pipeline Environment:** You will need storage account containers to store the raw and processed data used in the sample DataOps implementation. + +## The Sample Implementation + +This repository includes an implementation of DataOps for the `named_entity_recognition` sample. The sample implementation uses Azure Machine Learning to run the data pipelines. + +The data pipeline loads data from the source system, processes it, and stores it in the target location. The processed data is stored as JSONL files, which are registered as data assets + +![dataops llmops](images/dataops_llmops.png) + +The sample CI/CD pipelines manage the lifecycle of the data pipelines. They build and deploy the pipelines to the target environments. The CI/CD pipelines also register the required Datastores and Data Assets according to the processed JSONL files for Promptflow to consume. + +If you are not using data pipelines to create the data assets, the Promptflow flows will use the JSONL files inside the `data` folder to create the data assets. + +## Steps to Configure DataOps + +Follow these steps to configure DataOps for your flow: + +**New Folder for data pipelines** The data pipelines for the `named_entity_recognition` flow are inside a sub-folder named `data_pipelines`. Create a similar folder under your flow folder for the data pipelines. + +**Configure source and target location** As mentioned earlier, the data pipeline loads data from a source storage account container and stores the processed data in a target storage account container. The processed data in the target storage account gets mapped to the azure machine learning Data Asset. Create these two containers and upload the source dataset to the source container. + +**Data Pipeline Configuration:** The `dataops_config.json` file contains configurations for the data pipeline. + +You can start by copying an existing config file and modify it with relevant values. Provide valid values for all the configuration elements. + +**Updating Flow Configuration:** The configuration of the use-case is managed by the `experiment.yaml` (sets the flow paths, datasets, and evaluations). The `experiment.yaml` in the repo uses local data files. If you are using DataOps,this config file needs to point to the Data Asset path. The data asset path will look like this `azureml://datastores/[data_store_name]/paths/[file_path]` + +Update any datasets elements in the `experiment.yaml` files and make sure the source field point to the Data Asset path in format `azureml:` + +**Create Data Pipelines** The `named_entity_recognition` use case provides a sample python file for a data pipeline - [prep_data.py](../named_entity_recognition/data_pipelines/aml/prep_data.py). This is a placeholder for your data pipeline code. Replace it with your actual data transformation script. \ No newline at end of file diff --git a/docs/how_to_onboard_new_flows.md b/docs/how_to_onboard_new_flows.md index 023f55c9e..1c7851c4b 100644 --- a/docs/how_to_onboard_new_flows.md +++ b/docs/how_to_onboard_new_flows.md @@ -41,4 +41,4 @@ paths in `trigger and pr section` and the `default value` for `flow_to_execute` **Write tests in tests folder:** The `tests` folder contains unit test implementation for the flows. These are python tests that will get executed as part of PR pipelines. -**Update sample-request.json:** Create a new file 'sample-request.json' containing data needed to test a Prompt flow endpoint after deployment from within the pipelines. +**Update sample-request.json:** Create a new file 'sample-request.json' containing data needed to test a Prompt flow endpoint after deployment from within the pipelines. \ No newline at end of file diff --git a/docs/images/dataops_llmops.png b/docs/images/dataops_llmops.png new file mode 100644 index 000000000..6d1ecc07c Binary files /dev/null and b/docs/images/dataops_llmops.png differ diff --git a/docs/the_llmopsconfig_file.md b/docs/the_llmopsconfig_file.md new file mode 100644 index 000000000..e142c467a --- /dev/null +++ b/docs/the_llmopsconfig_file.md @@ -0,0 +1,13 @@ +# The llmops_config.yaml file + +The `llmops_config.yaml` is used to configure LLM Ops configuration to an use-case. At least one it configures: +- azure configuration under azure_config +- promptflow connections configuration under connections block +- base experiment configuration under experiment block +- environments configuration for usecase in environments block + - experiment overlay configuration that overrides the parameters in base experiment block + - deployment configurations in deployment_configs block + +Examples of the file are provided for the [named_entity_recognition](../named_entity_recognition/llmops_config.yaml), [math_coding](../math_coding/llmops_config.yaml) and [web classification](../web_classification/llmops_config.yaml). + + diff --git a/docs/tutorial/04-Patterns.md b/docs/tutorial/04-Patterns.md index 65facf1a6..2e30b213d 100644 --- a/docs/tutorial/04-Patterns.md +++ b/docs/tutorial/04-Patterns.md @@ -40,6 +40,6 @@ Multiple data sets can be added by following the steps mentioned below: Flows can be deployed as an endpoint in the following deployment targets: -- Kubernetes deployment: Add the deployment target named `kubernetes_endpoint` in `/configs/deployment_config.json` file for the specific use case folder. Example: for [web_classification](../../web_classification/) use case there is a deployment target defined for Kubernetes in: [deployment_config.json](../../web_classification/configs/deployment_config.json). -- AML Managed instance: Add the deployment target named `azure_managed_endpoint` in `/configs/deployment_config.json` file for the specific use case folder. Example: for [web_classification](../../web_classification/) use case there is a deployment target defined for AML Managed instance in: [deployment_config.json](../../web_classification/configs/deployment_config.json). +- Kubernetes deployment: Add the deployment target named `kubernetes_endpoint` in `llmops_config.yaml` file in deployment_configs section for the specific use case folder. Example: for [web_classification](../../web_classification/) use case there is a deployment target defined for Kubernetes in: [llmops_config.yaml](../../web_classification/llmops_config.yaml). +- AML Managed instance: Add the deployment target named `azure_managed_endpoint` in `llmops_config.yaml` file in deployment_configs section for the specific use case folder. Example: for [web_classification](../../web_classification/) use case there is a deployment target defined for Kubernetes in: [llmops_config.yaml](../../web_classification/llmops_config.yaml). - Flows can be also be exported as Docker images and can be deployed as running containers to any platform, OS and cloud. For more details refer documents in: [Prompt Flow GitHub](https://github.com/microsoft/promptflow/tree/main/docs/how-to-guides/deploy-a-flow). diff --git a/llmops/common/config_utils.py b/llmops/common/config_utils.py new file mode 100644 index 000000000..aa1c9813f --- /dev/null +++ b/llmops/common/config_utils.py @@ -0,0 +1,138 @@ +"""Configuration utils to load config from yaml""" +import os +from typing import Dict, Any +from pathlib import Path +from dotenv import load_dotenv +import yaml + + +class ExperimentConfig: + """ExperimentConfig Class.""" + _raw_config: Any + + def __init__( + self, + flow_name: str = "", + environment: str = None + ): + """Intialize raw config with yaml config data.""" + config_path = "experiment.yaml" + flow_name = flow_name.strip('./') + self._exp_config_path = Path(flow_name, config_path) + if self._exp_config_path.is_file(): + self._raw_config = self.load_yaml(self._exp_config_path) + self._environment = environment + if self._environment: + self._env_exp_config_path = Path(flow_name, f'experiment_{self._environment}.yaml') + if self._env_exp_config_path.is_file(): + self._env_exp_config = self.load_yaml(self._env_exp_config_path) + + + def load_yaml(self, config_path: str) -> Any: + """Load yaml file config""" + load_dotenv() + raw_config = None + with open(config_path, "r", encoding="utf-8") as stream: + raw_config = yaml.safe_load(os.path.expandvars(stream.read())) + return raw_config + + + def __getattr__(self, __name: str) -> Any: + """Get values for top level keys in configuration.""" + print(__name) + return self._raw_config[__name] + + + @property + def base_exp_config(self): + return self._raw_config + + @property + def azure_config(self): + """Get azure workspace config""" + if 'azure_config' in self.base_exp_config: + return self.base_exp_config['azure_config'] + else: + return None + + @property + def connections(self): + """Get connections configuration""" + if 'connections' in self.base_exp_config: + return self.base_exp_config['connections'] + + @property + def env_config(self): + """Get environment configuration.""" + return self._env_exp_config + + + @property + def base_experiment_config(self): + if 'experiment_config' in self.base_exp_config: + return self.base_exp_config['experiment_config'] + else: + return None + + @property + def overlay_experiment_config(self): + if 'experiment_config' in self._env_exp_config: + return self._env_exp_config['experiment_config'] + else: + return None + + @property + def evaluators_config(self): + """Get evaluator configuration.""" + if self.overlay_experiment_config and 'evaluators' in self.overlay_experiment_config: + return self.overlay_experiment_config['evaluators'] + else: + return None + + @property + def deployment_configs(self): + """Get deployment configuration.""" + if 'deployment_configs' in self._env_exp_config: + return self._env_exp_config['deployment_configs'] + else: + return None + + @property + def azure_managed_endpoint_config(self): + """Get azure managed endpoint deployment configuration.""" + if 'deployment_configs' in self._env_exp_config and 'azure_managed_endpoint' in self._env_exp_config['deployment_configs']: + return self._env_exp_config['deployment_configs']['azure_managed_endpoint'] + else: + return None + + @property + def kubernetes_endpoint_config(self): + """Get kubernetes endpoint deployment configuration.""" + if 'deployment_configs' in self._env_exp_config and 'kubernetes_endpoint' in self._env_exp_config['deployment_configs']: + return self._env_exp_config['deployment_configs']['kubernetes_endpoint'] + else: + return None + + @property + def webapp_endpoint_config(self): + """Get webapp endpoint deployment configuration.""" + if 'deployment_configs' in self._env_exp_config and 'webapp_endpoint' in self._env_exp_config['deployment_configs']: + return self._env_exp_config['deployment_configs']['webapp_endpoint'] + return None + + +if __name__ == "__main__": + config = ExperimentConfig(flow_name="web_classification", environment="dev") + print(config.azure_config) + print(config.connections) + print(config.env_config) + print(config.base_experiment_config) + print(config.overlay_experiment_config) + print(config.evaluators_config) + print(config.deployment_configs) + print(config.azure_managed_endpoint_config) + print(config.kubernetes_endpoint_config) + print(config.webapp_endpoint_config) + if config.webapp_endpoint_config: + for webapp_endpoint in config.webapp_endpoint_config: + print(webapp_endpoint['CONNECTION_NAMES']) \ No newline at end of file diff --git a/llmops/common/create_connections.py b/llmops/common/create_connections.py new file mode 100644 index 000000000..1339de863 --- /dev/null +++ b/llmops/common/create_connections.py @@ -0,0 +1,123 @@ + +""" +This module creates connections. + +Args: +--subscription_id: The Azure subscription ID. +This argument is required for identifying the Azure subscription. +--file: The name of the experiment file. Default is 'experiment.yaml'. +--base_path: Base path of the use case. Where flows, data, +and experiment.yaml are expected to be found. +--env_name: The environment name for execution and deployment. +This argument is required to specify the environment (dev, test, prod) +for execution or deployment. +""" + +import argparse +import hashlib +from azure.ai.ml import MLClient +from azure.ai.ml.entities import Data +from azure.ai.ml.constants import AssetTypes +from azure.identity import DefaultAzureCredential +from dotenv import load_dotenv +from typing import Optional + +from llmops.common.experiment_cloud_config import ExperimentCloudConfig +from llmops.common.config_utils import ExperimentConfig +from llmops.common.experiment import load_experiment +from llmops.common.logger import llmops_logger +from typing import Any +from azure.ai.ml import MLClient +from azure.identity import DefaultAzureCredential +from azure.ai.ml.entities import ( + WorkspaceConnection, + Workspace, + Hub, + ApiKeyConfiguration, + AzureOpenAIConnection, + AzureAIServicesConnection, + AzureAISearchConnection, + AzureContentSafetyConnection, + AzureSpeechServicesConnection, + APIKeyConnection, + OpenAIConnection, + SerpConnection, + ServerlessConnection, + AccountKeyConfiguration, +) + +logger = llmops_logger("create_connections") + +def create_connections( + base_path: str, + subscription_id: Optional[str] = None, + env_name: Optional[str] = None, +): + config = ExperimentCloudConfig(subscription_id=subscription_id, env_name=env_name) + experiment_config = ExperimentConfig(base_path, env_name) + + ml_client = MLClient( + DefaultAzureCredential(), + config.subscription_id, + config.resource_group_name, + config.workspace_name, + ) + + # Get connections configuration + connections = experiment_config.connections + + for connection in connections: + connection_type = connection["connection_type"] + connection_name = connection["connection"] + connection_config = connection["config"] + logger.info(f"Creating connection type : {connection_type} name: {connection_name}") + wps_connection = None + match connection_type: + case "AzureOpenAIConnection": + wps_connection = AzureOpenAIConnection(**connection_config) + case "AzureAISearchConnection": + wps_connection = AzureAISearchConnection(**connection_config) + case "OpenAIConnection": + wps_connection = OpenAIConnection(**connection_config) + case "Custom": + wps_connection = WorkspaceConnection(**connection_config) + case _: + logger.error("Not implemented or Unknown Connection Type") + + if wps_connection: + result = ml_client.connections.create_or_update(wps_connection) + logger.info(f"Created connection {result}") + + +def main(): + parser = argparse.ArgumentParser("create connections") + parser.add_argument( + "--subscription_id", + type=str, + help="Subscription ID, overrides the SUBSCRIPTION_ID environment variable", + default=None, + ) + parser.add_argument( + "--base_path", + type=str, + help="Base path of the use case", + required=True, + ) + parser.add_argument( + "--env_name", + type=str, + help="environment name(dev, test, prod) for execution and deployment, overrides the ENV_NAME environment variable", + default=None, + ) + + args = parser.parse_args() + + create_connections(args.base_path, args.subscription_id, args.env_name) + + +if __name__ == "__main__": + # Load variables from .env file into the environment + load_dotenv(override=True) + + main() + diff --git a/llmops/common/deployment/kubernetes_deployment.py b/llmops/common/deployment/kubernetes_deployment.py index 885c71d5a..379ebe5ba 100644 --- a/llmops/common/deployment/kubernetes_deployment.py +++ b/llmops/common/deployment/kubernetes_deployment.py @@ -40,6 +40,7 @@ from dotenv import load_dotenv +from llmops.common.config_utils import ExperimentConfig from llmops.common.logger import llmops_logger from llmops.common.experiment_cloud_config import ExperimentCloudConfig from llmops.common.experiment import load_experiment @@ -56,14 +57,16 @@ def create_kubernetes_deployment( subscription_id: Optional[str] = None, ): config = ExperimentCloudConfig(subscription_id=subscription_id, env_name=env_name) + llmops_config = ExperimentConfig(flow_name=base_path, environment=env_name) experiment = load_experiment( - filename=exp_filename, base_path=base_path, env=config.environment_name + base_path=base_path, + base_experiment_config=llmops_config.base_experiment_config, + overlay_experiment_config=llmops_config.overlay_experiment_config, + env=config.environment_name ) experiment_name = experiment.name model_name = f"{experiment_name}_{env_name}" - real_config = f"{base_path}/configs/deployment_config.json" - logger.info(f"Model name: {model_name}") ml_client = MLClient( @@ -75,9 +78,8 @@ def create_kubernetes_deployment( model = ml_client.models.get(model_name, model_version) - config_file = open(real_config) - endpoint_config = json.load(config_file) - for elem in endpoint_config["kubernetes_endpoint"]: + kubernetes_endpoints = llmops_config.deployment_configs['kubernetes_endpoint'] + for elem in kubernetes_endpoints: if "ENDPOINT_NAME" in elem and "ENV_NAME" in elem: if env_name == elem["ENV_NAME"]: endpoint_name = elem["ENDPOINT_NAME"] diff --git a/llmops/common/deployment/kubernetes_endpoint.py b/llmops/common/deployment/kubernetes_endpoint.py index 010b24d5d..b0b88d990 100644 --- a/llmops/common/deployment/kubernetes_endpoint.py +++ b/llmops/common/deployment/kubernetes_endpoint.py @@ -26,6 +26,7 @@ from llmops.common.logger import llmops_logger from llmops.common.experiment_cloud_config import ExperimentCloudConfig +from llmops.common.config_utils import ExperimentConfig logger = llmops_logger("provision_endpoint") @@ -38,8 +39,7 @@ def create_kubernetes_endpoint( output_file: Optional[str] = None, ): config = ExperimentCloudConfig(subscription_id=subscription_id, env_name=env_name) - - real_config = f"{base_path}/configs/deployment_config.json" + experiment_config = ExperimentConfig(flow_name=base_path, environment=env_name) ml_client = MLClient( DefaultAzureCredential(), @@ -48,10 +48,9 @@ def create_kubernetes_endpoint( config.workspace_name, ) - config_file = open(real_config) - endpoint_config = json.load(config_file) + kubernetes_endpoints = experiment_config.deployment_configs['kubernetes_endpoint'] - for elem in endpoint_config["kubernetes_endpoint"]: + for elem in kubernetes_endpoints: if "ENDPOINT_NAME" in elem and "ENV_NAME" in elem: if env_name == elem["ENV_NAME"]: endpoint_name = elem["ENDPOINT_NAME"] diff --git a/llmops/common/deployment/provision_deployment.py b/llmops/common/deployment/provision_deployment.py index 90698e779..bbbef17fd 100644 --- a/llmops/common/deployment/provision_deployment.py +++ b/llmops/common/deployment/provision_deployment.py @@ -37,6 +37,7 @@ from llmops.common.logger import llmops_logger from llmops.common.experiment_cloud_config import ExperimentCloudConfig +from llmops.common.config_utils import ExperimentConfig from llmops.common.experiment import load_experiment logger = llmops_logger("provision_deployment") @@ -51,14 +52,16 @@ def create_deployment( subscription_id: Optional[str] = None, ): config = ExperimentCloudConfig(subscription_id=subscription_id, env_name=env_name) + experiment_config = ExperimentConfig(flow_name=base_path, environment=env_name) experiment = load_experiment( - filename=exp_filename, base_path=base_path, env=config.environment_name + base_path=base_path, + base_experiment_config=experiment_config.base_experiment_config, + overlay_experiment_config=experiment_config.overlay_experiment_config, + env=config.environment_name ) experiment_name = experiment.name model_name = f"{experiment_name}_{env_name}" - real_config = f"{base_path}/configs/deployment_config.json" - logger.info(f"Model name: {model_name}") ml_client = MLClient( @@ -70,9 +73,8 @@ def create_deployment( model = ml_client.models.get(model_name, model_version) - config_file = open(real_config) - endpoint_config = json.load(config_file) - for elem in endpoint_config["azure_managed_endpoint"]: + azure_managed_endpoints = experiment_config.deployment_configs['azure_managed_endpount'] + for elem in azure_managed_endpoints: if "ENDPOINT_NAME" in elem and "ENV_NAME" in elem: if env_name == elem["ENV_NAME"]: endpoint_name = elem["ENDPOINT_NAME"] diff --git a/llmops/common/deployment/provision_endpoint.py b/llmops/common/deployment/provision_endpoint.py index 733357ac5..cb1903808 100644 --- a/llmops/common/deployment/provision_endpoint.py +++ b/llmops/common/deployment/provision_endpoint.py @@ -23,6 +23,7 @@ from azure.ai.ml import MLClient from azure.ai.ml.entities import ManagedOnlineEndpoint from azure.identity import DefaultAzureCredential +from llmops.common.config_utils import ExperimentConfig from llmops.common.logger import llmops_logger from llmops.common.experiment_cloud_config import ExperimentCloudConfig @@ -38,8 +39,11 @@ def create_endpoint( output_file: Optional[str] = None, ): config = ExperimentCloudConfig(subscription_id=subscription_id, env_name=env_name) + experiment_config = ExperimentConfig(flow_name=base_path, environment=env_name) - real_config = f"{base_path}/configs/deployment_config.json" + + real_config = experiment_config.deployment_configs + azure_managed_endpoints = real_config['azure_managed_endpoint'] ml_client = MLClient( DefaultAzureCredential(), @@ -48,10 +52,9 @@ def create_endpoint( config.workspace_name, ) - config_file = open(real_config) - endpoint_config = json.load(config_file) - for elem in endpoint_config["azure_managed_endpoint"]: + + for elem in azure_managed_endpoints: if "ENDPOINT_NAME" in elem and "ENV_NAME" in elem: if env_name == elem["ENV_NAME"]: endpoint_name = elem["ENDPOINT_NAME"] diff --git a/llmops/common/deployment/register_model.py b/llmops/common/deployment/register_model.py index 3db41a457..4f4cd0359 100644 --- a/llmops/common/deployment/register_model.py +++ b/llmops/common/deployment/register_model.py @@ -26,6 +26,7 @@ from llmops.common.logger import llmops_logger from llmops.common.experiment_cloud_config import ExperimentCloudConfig +from llmops.common.config_utils import ExperimentConfig from llmops.common.experiment import load_experiment logger = llmops_logger("register_flow") @@ -59,8 +60,12 @@ def register_model( output_file: Optional[str] = None, ): config = ExperimentCloudConfig(subscription_id=subscription_id, env_name=env_name) + experiment_config = ExperimentConfig(base_path, environment=env_name) experiment = load_experiment( - filename=exp_filename, base_path=base_path, env=config.environment_name + base_path=base_path, + base_experiment_config=experiment_config.base_experiment_config, + overlay_experiment_config=experiment_config.overlay_experiment_config, + env=config.environment_name ) experiment_name = experiment.name model_name = f"{experiment_name}_{env_name}" diff --git a/llmops/common/deployment/test_local_flow.py b/llmops/common/deployment/test_local_flow.py index 5a5a37c1d..6673ee14b 100644 --- a/llmops/common/deployment/test_local_flow.py +++ b/llmops/common/deployment/test_local_flow.py @@ -10,6 +10,7 @@ import json import requests import time + from llmops.common.logger import llmops_logger logger = llmops_logger("test local container endpoint") diff --git a/llmops/common/deployment/test_model_on_aml.py b/llmops/common/deployment/test_model_on_aml.py index 9faf37ad1..dc5e70682 100644 --- a/llmops/common/deployment/test_model_on_aml.py +++ b/llmops/common/deployment/test_model_on_aml.py @@ -18,7 +18,7 @@ from azure.ai.ml import MLClient from azure.identity import DefaultAzureCredential - +from llmops.common.config_utils import ExperimentConfig from llmops.common.logger import llmops_logger from llmops.common.experiment_cloud_config import ExperimentCloudConfig @@ -31,7 +31,7 @@ def test_aml_model( subscription_id: Optional[str], ): config = ExperimentCloudConfig(subscription_id=subscription_id, env_name=env_name) - real_config = f"{base_path}/configs/deployment_config.json" + experiment_config = ExperimentConfig(flow_name=base_path, environment=env_name) ml_client = MLClient( DefaultAzureCredential(), @@ -40,9 +40,8 @@ def test_aml_model( config.workspace_name, ) - config_file = open(real_config) - endpoint_config = json.load(config_file) - for elem in endpoint_config["azure_managed_endpoint"]: + azure_managed_endpoints = experiment_config.deployment_configs['azure_managed_endpoint'] + for elem in azure_managed_endpoints: if "ENDPOINT_NAME" in elem and "ENV_NAME" in elem: if env_name == elem["ENV_NAME"]: endpoint_name = elem["ENDPOINT_NAME"] diff --git a/llmops/common/deployment/test_model_on_kubernetes.py b/llmops/common/deployment/test_model_on_kubernetes.py index 4fcb57903..b46ac4dd1 100644 --- a/llmops/common/deployment/test_model_on_kubernetes.py +++ b/llmops/common/deployment/test_model_on_kubernetes.py @@ -18,7 +18,7 @@ from azure.ai.ml import MLClient from azure.identity import DefaultAzureCredential - +from llmops.common.config_utils import ExperimentConfig from llmops.common.logger import llmops_logger from llmops.common.experiment_cloud_config import ExperimentCloudConfig @@ -31,7 +31,7 @@ def test_aml_model( subscription_id: Optional[str], ): config = ExperimentCloudConfig(subscription_id=subscription_id, env_name=env_name) - real_config = f"{base_path}/configs/deployment_config.json" + experiment_config = ExperimentConfig(flow_name=base_path, environment=env_name) ml_client = MLClient( DefaultAzureCredential(), @@ -40,9 +40,8 @@ def test_aml_model( config.workspace_name, ) - config_file = open(real_config) - endpoint_config = json.load(config_file) - for elem in endpoint_config["kubernetes_endpoint"]: + kubernetes_endpoints = experiment_config.deployment_configs['kubernetes_endpoint'] + for elem in kubernetes_endpoints: if "ENDPOINT_NAME" in elem and "ENV_NAME" in elem: if env_name == elem["ENV_NAME"]: endpoint_name = elem["ENDPOINT_NAME"] diff --git a/llmops/common/experiment.py b/llmops/common/experiment.py index 3445c6c69..88c4d7c9c 100644 --- a/llmops/common/experiment.py +++ b/llmops/common/experiment.py @@ -431,11 +431,8 @@ def _resolve_flow_dir(base_path: Optional[str], flow: str) -> str: return os.path.join(safe_base_path, _DEFAULT_FLOWS_DIR, flow) -def _load_base_experiment(exp_file_path: str, base_path: Optional[str]) -> Experiment: - exp_config: dict - with open(exp_file_path, "r") as yaml_file: - exp_config = yaml.safe_load(yaml_file) - +def _load_base_experiment(exp_config: dict, base_path: Optional[str]) -> Experiment: + # Read base raw datasets and create base datasets and mappings raw_datasets: list[dict] = exp_config.get("datasets") if not raw_datasets: @@ -462,15 +459,9 @@ def _load_base_experiment(exp_file_path: str, base_path: Optional[str]) -> Exper def _apply_overlay( - experiment: Experiment, overlay_file_path: str, base_path: Optional[str] + experiment: Experiment, overlay_config: dict, base_path: Optional[str] ): - overlay_config: dict - with open(overlay_file_path, "r") as yaml_file: - overlay_config = yaml.safe_load(yaml_file) - - if not overlay_config: - return - + experiment_dataset_map: dict[str, Dataset] = { ds.dataset.name: ds.dataset for ds in experiment.datasets } @@ -503,8 +494,9 @@ def _apply_overlay( def load_experiment( - filename: Optional[str] = None, - base_path: Optional[str] = None, + base_path: str, + base_experiment_config: dict, + overlay_experiment_config: dict, env: Optional[str] = None, ) -> Experiment: """ @@ -519,24 +511,11 @@ def load_experiment( :type env: Optional[str] """ - safe_base_path = base_path or "" - experiment_file_name = filename or "experiment.yaml" - - # Validate the experiment file name - file_parts = os.path.splitext(experiment_file_name) - if len(file_parts) != 2: # noqa: PLR2004 - raise ValueError(f"Invalid experiment filename '{experiment_file_name}'") - env_experiment_file_name = f"{file_parts[0]}.{env}{file_parts[1]}" - # Create base experiment - exp_file_path = os.path.join(safe_base_path, experiment_file_name) - if not os.path.exists(exp_file_path): - raise ValueError(f"Could not open experiment file in path {exp_file_path}") - experiment = _load_base_experiment(exp_file_path, safe_base_path) + experiment = _load_base_experiment(base_experiment_config, base_path) # Apply environment overlay - env_exp_file_path = os.path.join(safe_base_path, env_experiment_file_name) - if os.path.exists(env_exp_file_path): - _apply_overlay(experiment, env_exp_file_path, base_path) + if overlay_experiment_config: + _apply_overlay(experiment, overlay_experiment_config, base_path) return experiment diff --git a/llmops/common/experiment_cloud_config.py b/llmops/common/experiment_cloud_config.py index 2b0c37a35..ef830d816 100644 --- a/llmops/common/experiment_cloud_config.py +++ b/llmops/common/experiment_cloud_config.py @@ -53,6 +53,7 @@ def __init__( resource_group_name: Optional[str] = None, workspace_name: Optional[str] = None, env_name: Optional[str] = None, + compute_target: Optional[str] = None, ): self.subscription_id = subscription_id or _try_get_env_var("SUBSCRIPTION_ID") self.resource_group_name = resource_group_name or _try_get_env_var( @@ -60,3 +61,4 @@ def __init__( ) self.workspace_name = workspace_name or _try_get_env_var("WORKSPACE_NAME") self.environment_name = env_name or _get_optional_env_var("ENV_NAME") + self.compute_target = compute_target or _get_optional_env_var("COMPUTE_TARGET") diff --git a/llmops/common/prompt_eval.py b/llmops/common/prompt_eval.py index 945788406..427c88d67 100644 --- a/llmops/common/prompt_eval.py +++ b/llmops/common/prompt_eval.py @@ -27,6 +27,7 @@ from promptflow.azure import PFClient from typing import Optional +from llmops.common.config_utils import ExperimentConfig from llmops.common.common import resolve_run_ids, wait_job_finish from llmops.common.experiment_cloud_config import ExperimentCloudConfig from llmops.common.experiment import load_experiment @@ -37,7 +38,7 @@ def prepare_and_execute( run_id: str, - exp_filename: Optional[str] = None, + flow_name: Optional[str] = None, base_path: Optional[str] = None, subscription_id: Optional[str] = None, build_id: Optional[str] = None, @@ -56,8 +57,13 @@ def prepare_and_execute( None """ config = ExperimentCloudConfig(subscription_id=subscription_id, env_name=env_name) + experiment_config = ExperimentConfig(flow_name=base_path, environment=env_name) + experiment = load_experiment( - filename=exp_filename, base_path=base_path, env=config.environment_name + base_path=base_path, + base_experiment_config=experiment_config.base_experiment_config, + overlay_experiment_config=experiment_config.overlay_experiment_config, + env=config.environment_name ) experiment_name = experiment.name @@ -259,9 +265,9 @@ def main(): parser.add_argument( "--file", type=str, - help="The experiment file. Default is 'experiment.yaml'", + help="The experiment flow name", required=False, - default="experiment.yaml", + default="", ) parser.add_argument( "--subscription_id", @@ -316,5 +322,4 @@ def main(): if __name__ == "__main__": # Load variables from .env file into the environment load_dotenv(override=True) - main() diff --git a/llmops/common/prompt_local_connections.py b/llmops/common/prompt_local_connections.py index cacb3de53..6eff76a06 100644 --- a/llmops/common/prompt_local_connections.py +++ b/llmops/common/prompt_local_connections.py @@ -15,7 +15,9 @@ from promptflow.entities import AzureOpenAIConnection from promptflow import PFClient +from llmops.common.config_utils import ExperimentConfig from llmops.common.logger import llmops_logger +from llmops.common.config_utils import ExperimentConfig logger = llmops_logger("prompt_aoai_connection") @@ -40,13 +42,12 @@ def prepare_and_execute( secret_config = json.loads(connection_details) - dep_config = f"{base_path}/configs/deployment_config.json" - config_file = open(dep_config) + config = ExperimentConfig(base_path, env_name) pf = PFClient() - connection_config = json.load(config_file) - for elem in connection_config["webapp_endpoint"]: + webapp_endpoint_configs = config.webapp_endpoint_config + for elem in webapp_endpoint_configs: if "CONNECTION_NAMES" in elem and "ENV_NAME" in elem: if env_name == elem["ENV_NAME"]: con_to_create = list(elem["CONNECTION_NAMES"]) @@ -103,7 +104,4 @@ def main(): if __name__ == "__main__": - # Load variables from .env file into the environment - load_dotenv(override=True) - main() diff --git a/llmops/common/prompt_pipeline.py b/llmops/common/prompt_pipeline.py index 802ec4942..6089962fa 100644 --- a/llmops/common/prompt_pipeline.py +++ b/llmops/common/prompt_pipeline.py @@ -47,15 +47,18 @@ from azure.identity import DefaultAzureCredential from promptflow.entities import Run from promptflow.azure import PFClient +from promptflow.tracing import start_trace, stop_trace from dotenv import load_dotenv from enum import Enum from typing import Optional +from llmops.common.config_utils import ExperimentConfig from llmops.common.common import wait_job_finish from llmops.common.experiment_cloud_config import ExperimentCloudConfig from llmops.common.experiment import load_experiment from llmops.common.logger import llmops_logger + logger = llmops_logger("prompt_pipeline") @@ -116,7 +119,7 @@ def from_args(cls, variants: str): def prepare_and_execute( variants_selector: VariantsSelector, - exp_filename: Optional[str] = None, + flow_name: Optional[str] = None, base_path: Optional[str] = None, subscription_id: Optional[str] = None, report_dir: Optional[str] = None, @@ -140,10 +143,15 @@ def prepare_and_execute( None """ config = ExperimentCloudConfig(subscription_id=subscription_id, env_name=env_name) + exp_config = ExperimentConfig(base_path, env_name) + experiment = load_experiment( - filename=exp_filename, base_path=base_path, env=config.environment_name + base_path=base_path, + base_experiment_config=exp_config.base_experiment_config, + overlay_experiment_config=exp_config.overlay_experiment_config, + env=config.environment_name ) - + start_trace() pf = PFClient( DefaultAzureCredential(), config.subscription_id, @@ -345,6 +353,7 @@ def prepare_and_execute( f_metrics.write(html_table_metrics) logger.info(f"Saved the metrics in files in {report_dir} folder") + stop_trace() def main(): diff --git a/llmops/common/register_data_asset.py b/llmops/common/register_data_asset.py index dbcac065e..f36b00962 100644 --- a/llmops/common/register_data_asset.py +++ b/llmops/common/register_data_asset.py @@ -22,6 +22,7 @@ from typing import Optional from llmops.common.experiment_cloud_config import ExperimentCloudConfig +from llmops.common.config_utils import ExperimentConfig from llmops.common.experiment import load_experiment from llmops.common.logger import llmops_logger @@ -52,8 +53,13 @@ def register_data_asset( env_name: Optional[str] = None, ): config = ExperimentCloudConfig(subscription_id=subscription_id, env_name=env_name) + experiment_config = ExperimentConfig(base_path, env_name) + experiment = load_experiment( - filename=exp_filename, base_path=base_path, env=config.environment_name + base_path=base_path, + base_experiment_config=experiment_config.base_experiment_config, + overlay_experiment_config=experiment_config.overlay_experiment_config, + env=config.environment_name ) ml_client = MLClient( DefaultAzureCredential(), diff --git a/llmops/common/scripts/az_webapp_deploy.sh b/llmops/common/scripts/az_webapp_deploy.sh index 03653ef9a..19eaafa2a 100755 --- a/llmops/common/scripts/az_webapp_deploy.sh +++ b/llmops/common/scripts/az_webapp_deploy.sh @@ -40,17 +40,18 @@ set -e # fail on error # read values from deployment_config.json related to `webapp_endpoint` env_name=$deploy_environment -deploy_config="./$use_case_base_path/configs/deployment_config.json" -con_object=$(jq ".webapp_endpoint[] | select(.ENV_NAME == \"$env_name\")" "$deploy_config") -REGISTRY_NAME=$(echo "$con_object" | jq -r '.REGISTRY_NAME') -rgname=$(echo "$con_object" | jq -r '.WEB_APP_RG_NAME') -udmid=$(echo "$con_object" | jq -r '.USER_MANAGED_ID') -appserviceplan=$(echo "$con_object" | jq -r '.APP_PLAN_NAME') -appserviceweb=$(echo "$con_object" | jq -r '.WEB_APP_NAME') -acr_rg=$(echo "$con_object" | jq -r '.REGISTRY_RG_NAME') -websku=$(echo "$con_object" | jq -r '.WEB_APP_SKU') - -read -r -a connection_names <<< "$(echo "$con_object" | jq -r '.CONNECTION_NAMES | join(" ")')" +config="./$use_case_base_path/experiment_$env_name.yaml" +con_object=$(yq ".deployment_configs.webapp_endpoint[] | select(.ENV_NAME == \"$env_name\")" "$config") +# con_object=$(jq ".webapp_endpoint[] | select(.ENV_NAME == \"$env_name\")" "$deploy_config") +REGISTRY_NAME=$(echo "$con_object" | yq -r '.REGISTRY_NAME') +rgname=$(echo "$con_object" | yq -r '.WEB_APP_RG_NAME') +udmid=$(echo "$con_object" | yq -r '.USER_MANAGED_ID') +appserviceplan=$(echo "$con_object" | yq -r '.APP_PLAN_NAME') +appserviceweb=$(echo "$con_object" | yq -r '.WEB_APP_NAME') +acr_rg=$(echo "$con_object" | yq -r '.REGISTRY_RG_NAME') +websku=$(echo "$con_object" | yq -r '.WEB_APP_SKU') + +read -r -a connection_names <<< "$(echo "$con_object" | yq -r '.CONNECTION_NAMES | join(" ")')" echo $connection_names # create a resource group @@ -79,7 +80,7 @@ az webapp config appsettings set --resource-group $rgname --name $appserviceweb --settings WEBSITES_PORT=8080 for name in "${connection_names[@]}"; do - api_key=$(echo ${CONNECTION_DETAILS} | jq -r --arg name "$name" '.[] | select(.name == $name) | .api_key') + api_key=$(echo ${CONNECTION_DETAILS} | yq -r --arg name "$name" '.[] | select(.name == $name) | .api_key') uppercase_name=$(echo "$name" | tr '[:lower:]' '[:upper:]') modified_name="${uppercase_name}_API_KEY" diff --git a/llmops/common/scripts/gen_docker_image.sh b/llmops/common/scripts/gen_docker_image.sh index 9ec396c15..ae861398a 100755 --- a/llmops/common/scripts/gen_docker_image.sh +++ b/llmops/common/scripts/gen_docker_image.sh @@ -44,6 +44,8 @@ config_path="./$use_case_base_path/experiment.yaml" if [[ -e "$config_path" ]]; then STANDARD_FLOW=$(yq eval '.flow // .name' "$config_path") + + pip install -r ./$use_case_base_path/$STANDARD_FLOW/requirements.txt pf flow build --source "./$use_case_base_path/$STANDARD_FLOW" --output "./$use_case_base_path/docker" --format docker cp "./$use_case_base_path/environment/Dockerfile" "./$use_case_base_path/docker/Dockerfile" @@ -53,8 +55,8 @@ if [[ -e "$config_path" ]]; then docker images - deploy_config="./$use_case_base_path/configs/deployment_config.json" - con_object=$(jq ".webapp_endpoint[] | select(.ENV_NAME == \"$env_name\")" "$deploy_config") + deploy_config="./$use_case_base_path/experiment_$deploy_environment.yaml" + con_object=$(yq ".deployment_configs.webapp_endpoint[] | select(.ENV_NAME == \"$deploy_environment\")" "$deploy_config") read -r -a connection_names <<< "$(echo "$con_object" | jq -r '.CONNECTION_NAMES | join(" ")')" result_string="" @@ -65,7 +67,7 @@ if [[ -e "$config_path" ]]; then modified_name="${uppercase_name}_API_KEY" result_string+=" -e $modified_name=$api_key" done - + docker_args=$result_string docker_args+=" -m 512m --memory-reservation=256m --cpus=2 -dp 8080:8080 localpf:latest" docker run $(echo "$docker_args") @@ -93,4 +95,4 @@ if [[ -e "$config_path" ]]; then else echo $config_path "not found" -fi +fi \ No newline at end of file diff --git a/local_execution/math_coding_local_experiment.py b/local_execution/math_coding_local_experiment.py index dcf4d5c64..7d445f271 100644 --- a/local_execution/math_coding_local_experiment.py +++ b/local_execution/math_coding_local_experiment.py @@ -1,22 +1,25 @@ +from pathlib import Path +from llmops.common.config_utils import ExperimentConfig +from local_execution.prompt_experimentation.run_local import LocalFlowExecution -from dotenv import load_dotenv -load_dotenv() - -from prompt_experimentation.run_local import LocalFlowExecution def main(): + config = ExperimentConfig("math_coding") + connections_config = config.connections data = "math_coding/data/test_data.jsonl" flow = "math_coding/flows/math_standard_flow" eval_flow = "math_coding/flows/math_evaluation_flow" - math_coding_flow = LocalFlowExecution(flow, eval_flow, data, {"math_question": "${data.question}"}) + math_coding_flow = LocalFlowExecution(flow, eval_flow, data, {"math_question": "${data.question}"}, connections_config) + math_coding_flow.process_local_flow() math_coding_flow.create_local_connections() run_ids = math_coding_flow.execute_experiment() - math_coding_flow.execute_evaluation(run_ids,data,{ + math_coding_flow.execute_evaluation(run_ids,data, { "groundtruth": "${data.groundtruth}", "prediction": "${run.outputs.answer}" }) + if __name__ == "__main__": main() \ No newline at end of file diff --git a/local_execution/named_entity_local_experiment.py b/local_execution/named_entity_local_experiment.py index 3bc06a9e0..4461eb5f0 100644 --- a/local_execution/named_entity_local_experiment.py +++ b/local_execution/named_entity_local_experiment.py @@ -1,13 +1,16 @@ from dotenv import load_dotenv load_dotenv() -from prompt_experimentation.run_local import LocalFlowExecution +from local_execution.prompt_experimentation.run_local import LocalFlowExecution +from llmops.common.config_utils import ExperimentConfig def main(): + config = ExperimentConfig("named_entity_recognition") + connections_config = config.connections data = "named_entity_recognition/data/data.jsonl" flow = "named_entity_recognition/flows/standard" eval_flow = "named_entity_recognition/flows/evaluation" - named_entity_flow = LocalFlowExecution(flow, eval_flow, data, {"text": "${data.text}", "entity_type": "${data.entity_type}"}) + named_entity_flow = LocalFlowExecution(flow, eval_flow, data, {"text": "${data.text}", "entity_type": "${data.entity_type}"}, connections_config) named_entity_flow.process_local_flow() named_entity_flow.create_local_connections() run_ids = named_entity_flow.execute_experiment() diff --git a/local_execution/prompt_experimentation/run_local.py b/local_execution/prompt_experimentation/run_local.py index ed11e17b9..47f105061 100644 --- a/local_execution/prompt_experimentation/run_local.py +++ b/local_execution/prompt_experimentation/run_local.py @@ -7,7 +7,7 @@ from promptflow.entities import Run from promptflow.entities import AzureOpenAIConnection -from promptflow import PFClient +from promptflow import PFClient def are_dictionaries_similar(dict1, old_runs): for old_run in old_runs: @@ -18,6 +18,11 @@ def are_dictionaries_similar(dict1, old_runs): return False +def find_dictionary_by_value(key, value, list_of_dictionaries): + for element in list_of_dictionaries: + if element[key] == value: + return element + def column_widths(column): max_length = max(column.astype(str).apply(len)) return f'width: {max_length}em;' @@ -27,13 +32,15 @@ def __init__(self, exp_flow_path, eval_flow_path, data_path, - column_mapping + column_mapping, + connections_config ): self.exp_flow_path = exp_flow_path self.eval_flow_path = eval_flow_path - self.data_path = data_path + self.data_path = data_path self.column_mapping = column_mapping + self.connections_config = connections_config self.local_pf_client = PFClient() @@ -43,7 +50,7 @@ def process_local_flow(self): all_llm_nodes = set() all_connection_nodes = [] default_variants = {} - llm_connections = [] + llm_connections = [] connections = {} flow_file = f"{self.exp_flow_path}/flow.dag.yaml" @@ -89,13 +96,15 @@ def process_local_flow(self): "deployment_name": llm_con["deployment_name"] } print(all_connection_nodes) - llm_connection ={} + llm_connection = {} + connection_config = find_dictionary_by_value('connection', llm_con["connection_name"], self.connections_config) + print(connection_config) llm_connection["name"] = llm_con["connection_name"] llm_connection["provider"] = llm_con["provider"] - llm_connection["api_key"] = json.loads(os.getenv(llm_con["connection_name"]))["api_key"] - llm_connection["api_base"] = json.loads(os.getenv(llm_con["connection_name"]))["api_base"] - llm_connection["api_type"] = json.loads(os.getenv(llm_con["connection_name"]))["api_type"] - llm_connection["api_version"] = json.loads(os.getenv(llm_con["connection_name"]))["api_version"] + llm_connection["api_key"] = connection_config["api_key"] + llm_connection["api_base"] = connection_config["api_base"] + llm_connection["api_type"] = connection_config["api_type"] + llm_connection["api_version"] = connection_config["api_version"] llm_connections.append(llm_connection) self.all_variants = all_variants diff --git a/local_execution/web_classification_local_experiment.py b/local_execution/web_classification_local_experiment.py index 710d1d1c1..64ecfe086 100644 --- a/local_execution/web_classification_local_experiment.py +++ b/local_execution/web_classification_local_experiment.py @@ -2,13 +2,16 @@ from dotenv import load_dotenv load_dotenv() -from prompt_experimentation.run_local import LocalFlowExecution +from local_execution.prompt_experimentation.run_local import LocalFlowExecution +from llmops.common.config_utils import ExperimentConfig def main(): + config = ExperimentConfig("web_classification") + connections_config = config.connections data = "web_classification/data/data.jsonl" flow = "web_classification/flows/experiment" eval_flow = "web_classification/flows/evaluation" - web_classification_flow = LocalFlowExecution(flow, eval_flow, data, {"url": "${data.url}"}) + web_classification_flow = LocalFlowExecution(flow, eval_flow, data, {"url": "${data.url}"}, connections_config) web_classification_flow.process_local_flow() web_classification_flow.create_local_connections() run_ids = web_classification_flow.execute_experiment() diff --git a/math_coding/configs/deployment_config.json b/math_coding/configs/deployment_config.json deleted file mode 100644 index d1a3d49f0..000000000 --- a/math_coding/configs/deployment_config.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "azure_managed_endpoint":[ - { - "ENV_NAME": "dev", - "TEST_FILE_PATH": "sample-request.json", - "ENDPOINT_NAME": "", - "ENDPOINT_DESC": "An online endpoint serving a flow for math coding flow", - "DEPLOYMENT_DESC": "prompt flow deployment", - "PRIOR_DEPLOYMENT_NAME": "", - "PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION": "", - "CURRENT_DEPLOYMENT_NAME": "", - "CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION": "100", - "DEPLOYMENT_VM_SIZE": "Standard_F4s_v2", - "DEPLOYMENT_INSTANCE_COUNT": 1, - "ENVIRONMENT_VARIABLES": { - "example-name": "example-value" - } - } - ], - "kubernetes_endpoint":[ - { - "ENV_NAME": "dev", - "TEST_FILE_PATH": "sample-request.json", - "ENDPOINT_NAME": "", - "ENDPOINT_DESC": "An kubernetes endpoint serving a flow for math coding", - "DEPLOYMENT_DESC": "prompt flow deployment", - "PRIOR_DEPLOYMENT_NAME": "", - "PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION": "", - "CURRENT_DEPLOYMENT_NAME": "", - "CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION": 100, - "COMPUTE_NAME": "", - "DEPLOYMENT_VM_SIZE": "promptinstancetype", - "DEPLOYMENT_INSTANCE_COUNT": 1, - "CPU_ALLOCATION": "", - "MEMORY_ALLOCATION": "", - "ENVIRONMENT_VARIABLES": { - "example-name": "example-value" - } - } - ], - "webapp_endpoint":[ - { - "ENV_NAME": "dev", - "TEST_FILE_PATH": "sample-request.json", - "CONNECTION_NAMES": ["aoai"], - "REGISTRY_NAME": "", - "REGISTRY_RG_NAME": "", - "APP_PLAN_NAME": "", - "WEB_APP_NAME": "", - "WEB_APP_RG_NAME": "", - "WEB_APP_SKU": "B3", - "USER_MANAGED_ID": "" - - } - ] -} \ No newline at end of file diff --git a/math_coding/experiment.dev.yaml b/math_coding/experiment.dev.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/math_coding/experiment.pr.yaml b/math_coding/experiment.pr.yaml deleted file mode 100644 index 57f985c72..000000000 --- a/math_coding/experiment.pr.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: math_standard_flow - -datasets: -- name: math_coding_pr - source: data/math_data.jsonl - description: "This dataset is for pr validation only." - mappings: - math_question: "${data.question}" - -evaluators: \ No newline at end of file diff --git a/math_coding/experiment.yaml b/math_coding/experiment.yaml index 1f67df919..28b95004a 100644 --- a/math_coding/experiment.yaml +++ b/math_coding/experiment.yaml @@ -1,21 +1,37 @@ -name: math_coding -flow: flows/math_standard_flow - -datasets: -- name: math_coding_training - source: data/math_data.jsonl - description: "This dataset is for prompt experiments." - mappings: - math_question: "${data.question}" - -evaluators: -- name: math_evaluation_flow - flow: flows/math_evaluation_flow - datasets: - - name: math_coding_test - reference: math_coding_training - source: data/test_data.jsonl - description: "This dataset is for evaluating flows." - mappings: - groundtruth: "${data.groundtruth}" - prediction: "${run.outputs.answer}" + +azure_config: + subscription_id: ${SUBSCRIPTION_ID} + resource_group_name: ${RESOURCE_GROUP_NAME} + workspace_name: ${WORKSPACE_NAME} + keyvault_name: ${KEYVAULT_NAME} + compute_target: ${COMPUTE_TARGET} + +connections: +- connection: aoai + api_type: azure + api_key: ${AZURE_OPENAI_KEY} + api_base: ${AZURE_OPENAI_ENDPOINT} + api_version: ${AZURE_OPENAI_API_VERSION} + +experiment: + name: math_coding + flow: flows/math_standard_flow + + datasets: + - name: math_coding_training + source: data/math_data.jsonl + description: "This dataset is for prompt experiments." + mappings: + math_question: "${data.question}" + + evaluators: + - name: math_evaluation_flow + flow: flows/math_evaluation_flow + datasets: + - name: math_coding_test + reference: math_coding_training + source: data/test_data.jsonl + description: "This dataset is for evaluating flows." + mappings: + groundtruth: "${data.groundtruth}" + prediction: "${run.outputs.answer}" \ No newline at end of file diff --git a/math_coding/experiment_dev.yaml b/math_coding/experiment_dev.yaml new file mode 100644 index 000000000..62c31e96c --- /dev/null +++ b/math_coding/experiment_dev.yaml @@ -0,0 +1,53 @@ +env_name: dev + +experiment_config: + +deployment_configs: + azure_managed_endpoint: + - name: azure_managed_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + ENDPOINT_NAME: + ENDPOINT_DESC: An online endpoint serving a flow for math coding flow + DEPLOYMENT_DESC: prompt flow deployment + PRIOR_DEPLOYMENT_NAME: + PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION: + CURRENT_DEPLOYMENT_NAME: + CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION: "100" + DEPLOYMENT_VM_SIZE: Standard_F4s_v2 + DEPLOYMENT_INSTANCE_COUNT: 1 + ENVIRONMENT_VARIABLES: + example-name: example-value + + kubernetes_endpoint: + - name: kubernetes_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + ENDPOINT_NAME: + ENDPOINT_DESC: An kubernetes endpoint serving a flow for math coding + DEPLOYMENT_DESC: prompt flow deployment + PRIOR_DEPLOYMENT_NAME: + PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION: + CURRENT_DEPLOYMENT_NAME: + CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION: 100 + COMPUTE_NAME: + DEPLOYMENT_VM_SIZE: promptinstancetype + DEPLOYMENT_INSTANCE_COUNT: 1 + CPU_ALLOCATION: + MEMORY_ALLOCATION: + ENVIRONMENT_VARIABLES: + example-name: example-value + + webapp_endpoint: + - name: webapp_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + CONNECTION_NAMES: + - aoai + REGISTRY_NAME: + REGISTRY_RG_NAME: + APP_PLAN_NAME: + WEB_APP_NAME: + WEB_APP_RG_NAME: + WEB_APP_SKU: "B3" + USER_MANAGED_ID: \ No newline at end of file diff --git a/math_coding/experiment_pr.yaml b/math_coding/experiment_pr.yaml new file mode 100644 index 000000000..d43358c72 --- /dev/null +++ b/math_coding/experiment_pr.yaml @@ -0,0 +1,12 @@ +env_name: pr +experiment_config: + name: math_standard_flow + + datasets: + - name: math_coding_pr + source: data/data.jsonl + description: "This dataset is for pr validation only." + mappings: + url: "${data.url}" + + evaluators: \ No newline at end of file diff --git a/named_entity_recognition/configs/dataops_config.json b/named_entity_recognition/configs/dataops_config.json new file mode 100644 index 000000000..8519a1f9b --- /dev/null +++ b/named_entity_recognition/configs/dataops_config.json @@ -0,0 +1,40 @@ +{ + "DATA_STORE_NAME": "ner_data_store", + "COMPUTE_NAME": "ner_compute", + "DATA_STORE_DESCRIPTION": "pipeline data store description for evaluation", + "DATA_PREP_COMPONENT": + { + "COMPONENT_NAME": "prep_data_component", + "COMPONENT_DISPLAY_NAME": "Prepare data component", + "COMPONENT_DESCRIPTION": "Loading and processing data for prompt engineering" + }, + "STORAGE": + { + "STORAGE_ACCOUNT": "saner", + "SOURCE_CONTAINER": "source", + "SOURCE_BLOB": "ner_source.csv", + "TARGET_CONTAINER": "data" + }, + "PATH": + { + "DATA_PIPELINE_CODE_DIR": "named_entity_recognition/data_pipelines/aml" + }, + "SCHEDULE": + { + "NAME": "ner_data_pipeline_schedule", + "CRON_EXPRESSION": "10 14 * * 1", + "TIMEZONE": "Eastern Standard Time" + }, + "DATA_ASSETS":[ + { + "NAME": "ner_eval", + "PATH": "eval.jsonl", + "DESCRIPTION": "NER eval data asset" + }, + { + "NAME": "ner_exp", + "PATH": "exp.jsonl", + "DESCRIPTION": "NER experiment data asset" + } + ] +} \ No newline at end of file diff --git a/named_entity_recognition/configs/deployment_config.json b/named_entity_recognition/configs/deployment_config.json deleted file mode 100644 index 7925f9365..000000000 --- a/named_entity_recognition/configs/deployment_config.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "azure_managed_endpoint":[ - { - "ENV_NAME": "dev", - "TEST_FILE_PATH": "sample-request.json", - "ENDPOINT_NAME": "", - "ENDPOINT_DESC": "An online endpoint serving a flow for named entity flow", - "DEPLOYMENT_DESC": "prompt flow deployment", - "PRIOR_DEPLOYMENT_NAME": "", - "PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION": "", - "CURRENT_DEPLOYMENT_NAME": "", - "CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION": "100", - "DEPLOYMENT_VM_SIZE": "Standard_F4s_v2", - "DEPLOYMENT_INSTANCE_COUNT": 1, - "ENVIRONMENT_VARIABLES": { - "example-name": "example-value" - } - } - ], - "kubernetes_endpoint":[ - { - "ENV_NAME": "dev", - "TEST_FILE_PATH": "sample-request.json", - "ENDPOINT_NAME": "", - "ENDPOINT_DESC": "An kubernetes endpoint serving a flow for named entity", - "DEPLOYMENT_DESC": "prompt flow deployment", - "PRIOR_DEPLOYMENT_NAME": "", - "PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION": "", - "CURRENT_DEPLOYMENT_NAME": "", - "CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION": 100, - "COMPUTE_NAME": "", - "DEPLOYMENT_VM_SIZE": "", - "DEPLOYMENT_INSTANCE_COUNT": 1, - "CPU_ALLOCATION": "", - "MEMORY_ALLOCATION": "", - "ENVIRONMENT_VARIABLES": { - "example-name": "example-value" - } - } - ], - "webapp_endpoint":[ - { - "ENV_NAME": "dev", - "TEST_FILE_PATH": "sample-request.json", - "CONNECTION_NAMES": ["aoai"], - "REGISTRY_NAME": "", - "REGISTRY_RG_NAME": "", - "APP_PLAN_NAME": "", - "WEB_APP_NAME": "", - "WEB_APP_RG_NAME": "", - "WEB_APP_SKU": "B3", - "USER_MANAGED_ID": "" - - } - ] -} \ No newline at end of file diff --git a/named_entity_recognition/data/source.txt b/named_entity_recognition/data/source.txt new file mode 100644 index 000000000..ee199586c --- /dev/null +++ b/named_entity_recognition/data/source.txt @@ -0,0 +1,16 @@ +text entity_type results +The software engineer is working on a new update for the application. job title software engineer +The project manager and the data analyst are collaborating to interpret the project data. job title "project manager, data analyst" +The marketing manager is coordinating with the graphic designer to create a new advertisement campaign. job title "marketing manager, graphic designer" +The CEO and CFO are discussing the financial forecast for the next quarter. job title "CEO, CFO" +The web developer and UX designer are working together to improve the website's user interface. job title "web developer, UX designer" +John finally decided to change his phone number after receiving too many spam calls. phone number None +"If you have any questions about our products, please call our customer service at (123) 456-7890." phone number (123) 456-7890 +"My new phone number is (098) 765-4321, please update your contact list." phone number (098) 765-4321 +The phone number (321) 654-0987 is no longer in service. phone number (321) 654-0987 +Please dial the following phone number: (555) 123-4567 to reach our technical support. phone number (555) 123-4567 +John Doe has been appointed as the new CEO of the company. people's full name John Doe +The novel 'The Great Gatsby' was written by F. Scott Fitzgerald. people's full name F. Scott Fitzgerald +Mary Jane Watson and Peter Parker are characters in the Spider-Man series. people's full name "Mary Jane Watson, Peter Parker" +"The famous physicists, Albert Einstein and Isaac Newton, made significant contributions to the field of physics." people's full name "Isaac Newton, Albert Einstein" +The Eiffel Tower is an iconic landmark in Paris. people's full name None diff --git a/named_entity_recognition/data_pipelines/aml/prep_data.py b/named_entity_recognition/data_pipelines/aml/prep_data.py new file mode 100644 index 000000000..5d684ecc1 --- /dev/null +++ b/named_entity_recognition/data_pipelines/aml/prep_data.py @@ -0,0 +1,100 @@ +import argparse +import json + +import pandas as pd +from azure.identity import DefaultAzureCredential +from azure.storage.blob import BlobServiceClient +import io + +""" +This function prepares data for processing. +It reads a CSV file from a source blob storage, +converts the CSV data to JSONL (JSON Lines) format, +and then uploads the JSONL data to a target blob storage. + +Args: +--blob_service_client: The Azure blob service client. +This argument is required for interacting with Azure blob storage. +--source_container_name: The name of the source container in blob storage. +This argument is required to specify the source container from where the CSV data is read. +--target_container_name: The name of the target container in blob storage. +This argument is required to specify the target container to where the JSONL data is uploaded. +--source_blob: The name of the source blob in the source container. +This argument is required to specify the source blob from where the CSV data is read. +--target_data_assets: The target data assets in the target container. +This argument is required to specify the target data assets to where the JSONL data is uploaded. +""" + + +def prepare_data(blob_service_client, + source_container_name, + target_container_name, + source_blob, + target_data_assets): + print('Data processing component') + + source_blob_client = blob_service_client.get_blob_client(container=source_container_name, + blob=source_blob) + source_blob_content = source_blob_client.download_blob().readall() + + assets = [item.strip() for item in target_data_assets.split(":")] + + df = pd.read_csv(io.StringIO(source_blob_content.decode('utf-8'))) + + jsonl_list = [] + for _, row in df.iterrows(): + jsonl_list.append(json.dumps(row.to_dict())) + + # Upload JSONL data to the target container + for asset in assets: + target_blob_client = blob_service_client.get_blob_client(container=target_container_name, + blob=asset) + target_blob_client.upload_blob('\n'.join(jsonl_list), overwrite=True) + print(f"CSV data converted to JSONL and uploaded successfully!: {asset}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--storage_account", + type=str, + help="storage account", + ) + parser.add_argument( + "--source_container_name", + type=str, + help="source container name", + ) + parser.add_argument( + "--target_container_name", + type=str, + help="target container name", + ) + parser.add_argument( + "--source_blob", + type=str, + help="source blob file (csv)", + ) + parser.add_argument( + "--assets_str", + type=str, + help="target assets to be created as a string" + ) + + args = parser.parse_args() + storage_account = args.storage_account + source_container_name = args.source_container_name + target_container_name = args.target_container_name + source_blob = args.source_blob + target_data_assets = args.assets_str + + storage_account_url = f"https://{storage_account}.blob.core.windows.net" + + blob_service_client = BlobServiceClient(storage_account_url, + credential=DefaultAzureCredential()) + + prepare_data(blob_service_client, + source_container_name, + target_container_name, + source_blob, + target_data_assets) diff --git a/named_entity_recognition/environment/conda.yml b/named_entity_recognition/environment/conda.yml new file mode 100644 index 000000000..6aed6c93c --- /dev/null +++ b/named_entity_recognition/environment/conda.yml @@ -0,0 +1,9 @@ +name: named-entity-env +channels: + - conda-forge +dependencies: + - python=3.9 + - pip + - pip: + - jinja2 + diff --git a/named_entity_recognition/experiment-postprod.yaml b/named_entity_recognition/experiment-postprod.yaml deleted file mode 100644 index d48ac172d..000000000 --- a/named_entity_recognition/experiment-postprod.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: named_entity_recognition -flow: flows/post-production-evaluation - -datasets: -- name: named_entity_recognition_pp_eval - source: data/production_log.jsonl - description: "This dataset is for post production evaluation." - mappings: - text: "${data.text}" - entity_type: "${data.entity_type}" - -evaluators: \ No newline at end of file diff --git a/named_entity_recognition/experiment.dev.yaml b/named_entity_recognition/experiment.dev.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/named_entity_recognition/experiment.pr.yaml b/named_entity_recognition/experiment.pr.yaml deleted file mode 100644 index ef2d464f0..000000000 --- a/named_entity_recognition/experiment.pr.yaml +++ /dev/null @@ -1,11 +0,0 @@ -name: named_entity_recognition - -datasets: -- name: named_entity_recognition_pr - source: data/data.jsonl - description: "This dataset is for pr validation only." - mappings: - text: "${data.text}" - entity_type: "${data.entity_type}" - -evaluators: \ No newline at end of file diff --git a/named_entity_recognition/experiment.yaml b/named_entity_recognition/experiment.yaml index 74341304a..36e380227 100644 --- a/named_entity_recognition/experiment.yaml +++ b/named_entity_recognition/experiment.yaml @@ -1,22 +1,37 @@ -name: named_entity_recognition -flow: flows/standard - -datasets: -- name: named_entity_recognition_train - source: data/data.jsonl - description: "This dataset is for prompt experiments." - mappings: - text: "${data.text}" - entity_type: "${data.entity_type}" - -evaluators: -- name: matcher - flow: flows/evaluation - datasets: - - name: named_entity_recognition_test - reference: named_entity_recognition_train - source: data/eval_data.jsonl - description: "This dataset is for evaluating flows." - mappings: - ground_truth: "${data.results}" - entities: "${run.outputs.entities}" +azure_config: + subscription_id: ${SUBSCRIPTION_ID} + resource_group_name: ${RESOURCE_GROUP_NAME} + workspace_name: ${WORKSPACE_NAME} + keyvault_name: ${KEYVAULT_NAME} + compute_target: ${COMPUTE_TARGET} + +connections: +- connection: aoai + api_type: azure + api_key: ${AZURE_OPENAI_KEY} + api_base: ${AZURE_OPENAI_ENDPOINT} + api_version: ${AZURE_OPENAI_API_VERSION} + +experiment_config: + name: named_entity_recognition + flow: flows/standard + + datasets: + - name: named_entity_recognition_train + source: data/data.jsonl + description: "This dataset is for prompt experiments." + mappings: + text: "${data.text}" + entity_type: "${data.entity_type}" + + evaluators: + - name: matcher + flow: flows/evaluation + datasets: + - name: named_entity_recognition_test + reference: named_entity_recognition_train + source: data/eval_data.jsonl + description: "This dataset is for evaluating flows." + mappings: + ground_truth: "${data.results}" + entities: "${run.outputs.entities}" \ No newline at end of file diff --git a/named_entity_recognition/experiment_dev.yaml b/named_entity_recognition/experiment_dev.yaml new file mode 100644 index 000000000..1ceef5adf --- /dev/null +++ b/named_entity_recognition/experiment_dev.yaml @@ -0,0 +1,53 @@ +env_name: dev + +experiment_config: + +deployment_configs: + azure_managed_endpoint: + - name: azure_managed_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + ENDPOINT_NAME: + ENDPOINT_DESC: An online endpoint serving a flow for named entity flow + DEPLOYMENT_DESC: prompt flow deployment + PRIOR_DEPLOYMENT_NAME: + PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION: + CURRENT_DEPLOYMENT_NAME: + CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION: "100" + DEPLOYMENT_VM_SIZE: Standard_F4s_v2 + DEPLOYMENT_INSTANCE_COUNT: 1 + ENVIRONMENT_VARIABLES: + example-name: example-value + + kubernetes_endpoint: + - name: kubernetes_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + ENDPOINT_NAME: + ENDPOINT_DESC: An kubernetes endpoint serving a flow for named entity + DEPLOYMENT_DESC: prompt flow deployment + PRIOR_DEPLOYMENT_NAME: + PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION: + CURRENT_DEPLOYMENT_NAME: + CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION: 100, + COMPUTE_NAME: + DEPLOYMENT_VM_SIZE: + DEPLOYMENT_INSTANCE_COUNT: 1 + CPU_ALLOCATION: + MEMORY_ALLOCATION: + ENVIRONMENT_VARIABLES: + example-name: example-value + + webapp_endpoint: + - name: webapp_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + CONNECTION_NAMES: + - aoai + REGISTRY_NAME: + REGISTRY_RG_NAME: + APP_PLAN_NAME: + WEB_APP_NAME: + WEB_APP_RG_NAME: + WEB_APP_SKU: "B3" + USER_MANAGED_ID: diff --git a/named_entity_recognition/experiment_pr.yaml b/named_entity_recognition/experiment_pr.yaml new file mode 100644 index 000000000..beb5f1792 --- /dev/null +++ b/named_entity_recognition/experiment_pr.yaml @@ -0,0 +1,15 @@ +env_name: pr + +experiment_config: + name: named_entity_recognition + + datasets: + - name: named_entity_recognition_pr + source: data/data.jsonl + description: "This dataset is for pr validation only." + + mappings: + text: "${data.text}" + entity_type: "${data.entity_type}" + + evaluators: \ No newline at end of file diff --git a/named_entity_recognition/experiment_preprod b/named_entity_recognition/experiment_preprod new file mode 100644 index 000000000..9f0c3d4ba --- /dev/null +++ b/named_entity_recognition/experiment_preprod @@ -0,0 +1,15 @@ +env_name: postprodeval + +experiment_config: + name: named_entity_recognition + flow: flows/post-production-evaluation + + datasets: + - name: named_entity_recognition_pp_eval + source: data/production_log.jsonl + description: "This dataset is for post production evaluation." + mappings: + text: "${data.text}" + entity_type: "${data.entity_type}" + + evaluators: \ No newline at end of file diff --git a/pf_aml_pipeline/components/postprocess.py b/pf_aml_pipeline/components/postprocess.py new file mode 100644 index 000000000..44ea46d89 --- /dev/null +++ b/pf_aml_pipeline/components/postprocess.py @@ -0,0 +1,38 @@ +import argparse + +import pandas as pd +from pathlib import Path + +PF_OUTPUT_FILE_NAME = "parallel_run_step.jsonl" +def parse_args(): + """ + Parses the user arguments. + + Returns: + argparse.Namespace: The parsed user arguments. + """ + parser = argparse.ArgumentParser( + allow_abbrev=False, description="parse user arguments" + ) + parser.add_argument("--input_data_path", type=str) + + args, _ = parser.parse_known_args() + return args + + +def main(): + """ + The main function that orchestrates the data preparation process. + """ + args = parse_args() + + # Read promptflow output file and do some postprocessing + input_data_path = args.input_data_path + '/' + PF_OUTPUT_FILE_NAME + with open((Path(input_data_path)), 'r') as file: + promptflow_output = pd.read_json(file, lines=True) + print(promptflow_output.head()) + + return + +if __name__ == "__main__": + main() diff --git a/pf_aml_pipeline/components/preprocess.py b/pf_aml_pipeline/components/preprocess.py new file mode 100644 index 000000000..5285fdfdf --- /dev/null +++ b/pf_aml_pipeline/components/preprocess.py @@ -0,0 +1,47 @@ +import argparse + +import pandas as pd + + +def parse_args(): + """ + Parses the user arguments. + + Returns: + argparse.Namespace: The parsed user arguments. + """ + parser = argparse.ArgumentParser( + allow_abbrev=False, description="parse user arguments" + ) + parser.add_argument("--max_records", type=int, default=1) + parser.add_argument("--input_data_path", type=str) + parser.add_argument("--output_data_path", type=str) + + args, _ = parser.parse_known_args() + return args + + +def main(): + """ + The main function that orchestrates the data preparation process. + """ + args = parse_args() + print("Maximum records to keep", args.max_records) + + input_data_path = args.input_data_path + input_data_df = pd.read_json(input_data_path, lines=True) + + # take only max_records from input_data_df + input_data_df = input_data_df.head(args.max_records) + + # Write input_data_df to a jsonl file + input_data_df.to_json( + args.output_data_path, orient="records", lines=True + ) + print("Successfully written filtered data") + + return + + +if __name__ == "__main__": + main() diff --git a/pf_aml_pipeline/promptflow_in_aml_pipeline.py b/pf_aml_pipeline/promptflow_in_aml_pipeline.py new file mode 100644 index 000000000..943ce37bf --- /dev/null +++ b/pf_aml_pipeline/promptflow_in_aml_pipeline.py @@ -0,0 +1,234 @@ +import argparse +import datetime +from typing import Optional +from dotenv import load_dotenv + + +from azure.ai.ml import Input, MLClient, Output, command, dsl, load_component +from azure.ai.ml.constants import AssetTypes, InputOutputModes +from azure.identity import DefaultAzureCredential +from llmops.common.experiment_cloud_config import ExperimentCloudConfig +from llmops.common.config_utils import ExperimentConfig +from llmops.common.experiment import load_experiment +from llmops.common.logger import llmops_logger + +logger = llmops_logger("promptflow_in_aml_pipeline") + +pipeline_components = [] + +def create_dynamic_evaluation_pipeline( + pipeline_name, + input_data_path, +): + """ + Construct evaluation pipeline definition dynamically for a specific app and evaluator. + + Args: + pipeline_name (str): Name of the pipeline. + """ + + @dsl.pipeline( + name=pipeline_name, + input_data_path=input_data_path, + ) + def evaluation_pipeline(name: str, input_data_path: str): + + preprocess_input_path = Input( + path=input_data_path, + type=AssetTypes.URI_FILE, + mode=InputOutputModes.RO_MOUNT, + ) + + preprocess = pipeline_components[0]( + input_data_path=preprocess_input_path, max_records=2 + ) + + experiment = pipeline_components[1]( + data=preprocess.outputs.output_data_path, + url="${data.url}", + ) + + postprocess = pipeline_components[2]( + input_data_path=experiment.outputs.flow_outputs, + ) + + return evaluation_pipeline + + +def build_pipeline(pipeline_name: str, flow_path: str, input_data_path: str): + """ + Constructs an Azure Machine Learning pipeline. It encapsulates the process of defining pipeline inputs, + loading pipeline components from YAMLs, configuring component environments settings, configuring pipeline settings etc. + + Args: + pipeline_name (str): Name of the pipeline. + + Returns: + PipelineJob: Azure Machine Learning pipeline job. + """ + preprocess_component = command( + name="./components/preprocess", + display_name="Data preparation for Promptflow in a pipeline experiment", + description="Reads the input data and prepares it for the Promptflow experiment", + inputs={ + "input_data_path": Input(path="string", type="uri_file", mode="ro_mount"), + "max_records": Input(type="number"), + }, + outputs={ + "output_data_path": Output(type="uri_file", mode="rw_mount"), + }, + # The source folder of the component + code="./pf_aml_pipeline/components/", + command="""python preprocess.py \ + --input_data_path "${{inputs.input_data_path}}" \ + --max_records "${{inputs.max_records}}" \ + --output_data_path "${{outputs.output_data_path}}" \ + """, + environment="azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:1", # TODO FIXME + ) + # This step loads the promptflow in the pipeline as a component + evaluation_promptflow_component = load_component( + flow_path, + ) + postprocess_component = command( + name="postprocess", + display_name="Post processing for Promptflow in a pipeline experiment", + description="Reads the output of the Promptflow experiment and does some post processing.", + inputs={ + "input_data_path": Input(type="uri_folder", mode="rw_mount"), + }, + # The source folder of the component + code="./pf_aml_pipeline/components/", + command="""python postprocess.py \ + --input_data_path "${{inputs.input_data_path}}" \ + """, + environment="azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:1", + ) + pipeline_components.append(preprocess_component) + pipeline_components.append(evaluation_promptflow_component) + pipeline_components.append(postprocess_component) + + pipeline_definition = create_dynamic_evaluation_pipeline( + pipeline_name=pipeline_name, + input_data_path=input_data_path, + ) + + return pipeline_definition + + +def prepare_and_execute( + exp_filename: Optional[str] = None, + base_path: Optional[str] = None, + subscription_id: Optional[str] = None, + env_name: Optional[str] = None, +): + """ + Run the experimentation loop by executing standard flows. + + reads latest experiment data assets. + identifies all variants across all nodes. + executes the flow creating a new job using + unique variant combination across nodes. + saves the results in both csv and html format. + saves the job ids in text file for later use. + + Returns: + None + """ + config = ExperimentCloudConfig(subscription_id=subscription_id, env_name=env_name) + llmops_config = ExperimentConfig(flow_name=base_path, environment=env_name) + + experiment = load_experiment( + base_path=base_path, + base_experiment_config=llmops_config.base_experiment_config, + overlay_experiment_config=llmops_config.overlay_experiment_config, + env=config.environment_name + ) + + flow_detail = experiment.get_flow_detail() + + logger.info(f"Running experiment {experiment.name}") + for mapped_dataset in experiment.datasets: + logger.info(f"Using dataset {mapped_dataset.dataset.source}") + dataset = mapped_dataset.dataset + + ml_client = MLClient( + DefaultAzureCredential(), + config.subscription_id, + config.resource_group_name, + config.workspace_name + ) + + experiment_name = f"{experiment.name}_{env_name}" + input_data_uri_file = ml_client.data.get(name=dataset.name, label="latest") + + flow_path = f"{flow_detail.flow_path}/flow.dag.yaml" + build_pipeline("mypipeline", flow_path, input_data_uri_file) + + pipeline_definition = build_pipeline( + pipeline_name="mypipeline", + flow_path=flow_path, + input_data_path=input_data_uri_file, + ) + + pipeline_job = pipeline_definition(name="mypipeline", input_data_path=input_data_uri_file) + pipeline_job.settings.default_compute = config.compute_target + # Execute the ML Pipeline + job = ml_client.jobs.create_or_update( + pipeline_job, + experiment_name=experiment_name, + ) + + ml_client.jobs.stream(name=job.name) + + +def main(): + """ + main() function to run experiment or evaluations. + + Returns: + None + """ + parser = argparse.ArgumentParser("prompt_bulk_run") + parser.add_argument( + "--file", + type=str, + help="The experiment file. Default is 'experiment.yaml'", + required=False, + default="experiment.yaml", + ) + + parser.add_argument( + "--subscription_id", + type=str, + help="Subscription ID, overrides the SUBSCRIPTION_ID environment variable", + default=None, + ) + parser.add_argument( + "--base_path", + type=str, + help="Base path of the use case", + required=True, + ) + parser.add_argument( + "--env_name", + type=str, + help="environment name(dev, test, prod) for execution and deployment, overrides the ENV_NAME environment variable", + default=None, + ) + + args = parser.parse_args() + + prepare_and_execute( + args.file, + args.base_path, + args.subscription_id, + args.env_name, + ) + + +if __name__ == "__main__": + # Load variables from .env file into the environment + load_dotenv(override=True) + + main() \ No newline at end of file diff --git a/plan_and_execute/.azure-pipelines/plan_and_execute_ci_dev_pipeline.yml b/plan_and_execute/.azure-pipelines/plan_and_execute_ci_dev_pipeline.yml new file mode 100644 index 000000000..1d9340dc5 --- /dev/null +++ b/plan_and_execute/.azure-pipelines/plan_and_execute_ci_dev_pipeline.yml @@ -0,0 +1,41 @@ +pr: none +trigger: + branches: + include: + - main + - development + paths: + include: + - .azure-pipelines/* + - llmops/* + - plan_and_execute/* + + +pool: + vmImage: ubuntu-latest + +variables: +- group: llmops_platform_dev_vg + +parameters: + - name: env_name + displayName: "Execution Environment" + default: "dev" + - name: flow_to_execute + displayName: "flow to execute" + default: "plan_and_execute" + - name: deployment_type + displayName: "Determine type of deployment - aml, aks, docker, webapp" + default: "aml" + +#===================================== +# Execute platform_ci_dev_pipeline pipeline for experiment, evaluation and deployment of flows +#===================================== +stages: + - template: ../../.azure-pipelines/platform_ci_dev_pipeline.yml + parameters: + exec_environment: ${{ parameters.env_name }} + flow_to_execute: ${{ parameters.flow_to_execute }} + deployment_type: ${{ lower(parameters.deployment_type) }} + connection_details: '$(COMMON_DEV_CONNECTIONS)' + registry_details: '$(DOCKER_IMAGE_REGISTRY)' \ No newline at end of file diff --git a/plan_and_execute/.azure-pipelines/plan_and_execute_pr_dev_pipeline.yml b/plan_and_execute/.azure-pipelines/plan_and_execute_pr_dev_pipeline.yml new file mode 100644 index 000000000..817e3bb79 --- /dev/null +++ b/plan_and_execute/.azure-pipelines/plan_and_execute_pr_dev_pipeline.yml @@ -0,0 +1,34 @@ +trigger: none +pr: + branches: + include: + - main + - development + paths: + include: + - .azure-pipelines/* + - llmops/* + - plan_and_execute/* + +pool: + vmImage: ubuntu-latest + +variables: +- group: llmops_platform_dev_vg + +parameters: + - name: env_name + displayName: "Execution Environment" + default: "pr" + - name: flow_to_execute + displayName: "flow to execute" + default: "plan_and_execute" + +#===================================== +# Execute platform_pr_dev_pipeline pipeline for experiment, evaluation and deployment of flows +#===================================== +stages: + - template: ../../.azure-pipelines/platform_pr_dev_pipeline.yml + parameters: + exec_environment: ${{ parameters.env_name }} + flow_to_execute: ${{ parameters.flow_to_execute }} \ No newline at end of file diff --git a/plan_and_execute/README.md b/plan_and_execute/README.md new file mode 100644 index 000000000..5e7150014 --- /dev/null +++ b/plan_and_execute/README.md @@ -0,0 +1,48 @@ +# Plan and Execute with LLM Agents + +This is an example implementation of an agentic flow, capable of planning the steps needed to execute a user's request, then efficiently executing the plan through external function calling, and assembling a final response. + +It implements the core ideas from these two papers: +- [ReWOO: Decoupling Reasoning from Observations for Efficient Augmented Language Models](https://arxiv.org/abs/2305.18323) +- [An LLM Compiler for Parallel Function Calling](https://arxiv.org/abs/2312.04511) + +The idea is to optimize the traditional loop of reasoning and acting for planning and executing tasks with LLM-based agents, usually implemented by the [ReAct pattern](https://arxiv.org/abs/2210.03629), where the planning and acting steps are interleaved in a sequential manner. + +By decoupling the planning from the acting, we make several potential optimizations possible: +- by having a separate LLM agent concerned with the planning only, we open up the possibility of fine-tuning a specialized model, which could lead to more efficiency and reduced costs, depending on the scenario. +- by having a separate component for orchestrating external tools calling for the execution of the plan steps, we can optimize for latency by executing functions in parallel, when they are not dependent from each other. + +This implementation also uses components of the [Microsoft's AutoGen framework](https://github.com/microsoft/autogen), to facilitate the interaction with LLMs in all modules and execute external functions, as explained in the Architecture Overview below. + +## Architecture Overview +Plan and Execute - Architecture Overview + +The main components of this implementation are depicted in the architecture diagram above. Planner, Executor, and Solver are implemented as Prompt flow Python nodes. Tools are implemented as standard Python functions. + +### Planner +The Planner is implemented as an [AutoGen AssistantAgent](https://microsoft.github.io/autogen/docs/reference/agentchat/assistant_agent). Its system message with few shot examples is implemented as a Prompt flow prompt. Planner is aware of the available tools capabilities and how to use them. It takes as input a user's request and is instructed to generate a step-by-step plan to solve it. The plan is specified to be generated as a valid JSON object, with a list of descriptions for each plan step, and a list of functions to be called to solve each step. Dependencies between those functions are specified as variable assignments using a specific notation. + +### Executor +The Executor is implemented as a combination of custom Python code and an [AutoGen UserProxyAgent](https://microsoft.github.io/autogen/docs/reference/agentchat/user_proxy_agent/). It takes the generated plan as input. The custom Python code takes care of fetching function calls from the plan, solving function dependencies, dispatching functions for execution, and collecting results. The AutoGen UserProxyAgent facilitates the actual execution of Python functions, including parallel execution, as it already has these functionalities implemented. The output of Executor is a list with the results from all plan steps. + +### Tools +Tools are implemented as standard Python functions, but strongly typed. In this way, they can seamlessly be registered within Autogen AssistantAgent and UserProxyAgent, without the need of maintaining a separate function definitions dictionary. + +### Solver +The Solver is also implemented as an AutoGen AssistantAgent. Its system message is implemented as a Prompt flow prompt. It takes as input the user's request and the plan steps results and is instructed to use the information from the plan step results to answer the user's request. + +## How to setup and deploy this flow +Notice: the current implementation support deployment with Azure DevOps only. Support for GitHub workflows will be added in the future. + +This example supports Azure DevOps pipelines as a platform for Flow operationalization. Please follow the instructions in [How to setup the repo with Azure DevOps](https://github.com/microsoft/llmops-promptflow-template/blob/main/docs/Azure_devops_how_to_setup.md). + +### Prerequisites +Follow the prerequisites in [How to setup the repo with Azure DevOps](https://github.com/microsoft/llmops-promptflow-template/blob/main/docs/Azure_devops_how_to_setup.md). After that, you will also need the following: + +- a Bing Web Search API key. You create one in your Azure subscription following the instructions [here](https://aka.ms/bingapisignup). +- a `gpt-35-turbo` and a `gpt-4` model deployment on your Azure Open AI service. Both should be under the same service (same base URL). Please see [here](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/working-with-models) for more details. +- a Prompt flow custom connection. Please see below. + +Create a Prompt flow connection of type Custom and name it `plan_execute_agent_connection`. To do so, go to your Azure Machine Learning workspace portal, click `Prompt flow` -> `Connections` -> `Create` -> `Custom`. Fill in the key-value pairs according to the figure below: + +Custom Connection Information diff --git a/plan_and_execute/configs/data_config.json b/plan_and_execute/configs/data_config.json new file mode 100644 index 000000000..4cb60cd7b --- /dev/null +++ b/plan_and_execute/configs/data_config.json @@ -0,0 +1,29 @@ +{ + "datasets":[ + { + "PROMPT_FLOW_CONFIG_NAME": "plan_and_execute", + "ENV_NAME": "pr", + "DATA_PURPOSE": "pr_data", + "DATA_PATH":"data/data.jsonl", + "DATASET_NAME":"temp_dataset", + "DATASET_DESC":"this dataset is for pr validation only" + }, + { + "PROMPT_FLOW_CONFIG_NAME": "plan_and_execute", + "ENV_NAME": "dev", + "DATA_PURPOSE": "test_data", + "DATA_PATH":"data/eval_data.jsonl", + "DATASET_NAME":"plan_and_execute_test", + "RELATED_EXP_DATASET": "plan_and_execute_train", + "DATASET_DESC":"this dataset is for evaluating flows" + }, + { + "PROMPT_FLOW_CONFIG_NAME": "plan_and_execute", + "ENV_NAME": "dev", + "DATA_PURPOSE": "training_data", + "DATA_PATH":"data/data.jsonl", + "DATASET_NAME":"plan_and_execute_train", + "DATASET_DESC":"this dataset is for prompt experiments" + } + ] +} diff --git a/plan_and_execute/configs/mapping_config.json b/plan_and_execute/configs/mapping_config.json new file mode 100644 index 000000000..495f45a65 --- /dev/null +++ b/plan_and_execute/configs/mapping_config.json @@ -0,0 +1,13 @@ +{ + "experiment": { + "question": "${data.question}" + }, + "evaluation": { + "evaluation": { + "question": "${data.question}", + "generated_answer": "${run.outputs.answer}", + "ground_truth_answer": "${data.answer}" + } + } + +} \ No newline at end of file diff --git a/plan_and_execute/data/data.jsonl b/plan_and_execute/data/data.jsonl new file mode 100644 index 000000000..1f21c746f --- /dev/null +++ b/plan_and_execute/data/data.jsonl @@ -0,0 +1 @@ +{"question": "What was the total box office performance of 'Inception' and 'Interstellar' together?", "answer": "The total box office performance of 'Inception' and 'Interstellar' together was $1,509,329,092."} \ No newline at end of file diff --git a/plan_and_execute/data/eval_data.jsonl b/plan_and_execute/data/eval_data.jsonl new file mode 100644 index 000000000..237b9322e --- /dev/null +++ b/plan_and_execute/data/eval_data.jsonl @@ -0,0 +1,8 @@ +{"question": "What was the total box office performance of 'Inception' and 'Interstellar' together?", "answer": "The total box office performance of 'Inception' and 'Interstellar' together was $1,509,329,092."} +{"question": "What is the change rate of the U.S. inflation between 2022 and 2023? Was there an increase or decrease in the inflation?", "answer": "The change rate of the U.S. inflation between 2022 and 2023 was approximately -48.75%. There was a decrease in the inflation rate."} +{"question": "What is the percentage breakdown of the number of native speakers of the top 3 languages in the world?", "answer": "The percentage breakdown of the number of native speakers of the top 3 languages in the world is as follows:\n\n- Chinese (Mandarin): approximately 52.06%\n- Spanish: approximately 26.89%\n- English: approximately 21.05%"} +{"question": "What was the percentage change of the Tokyo population from 2010 to 2020? Did it increase or decrease?", "answer": "The population of Tokyo increased by approximately 7.80% from 2010 to 2020."} +{"question": "Which Nobel Prize category has awarded the most prizes to women, and who was the latest female recipient?", "answer": "The Nobel Prize category that has awarded the most prizes to women is the Nobel Peace Prize, and the latest female recipient is Narges Mohammadi in 2023."} +{"question": "Calculate the total number of goals scored by Lionel Messi, Cristiano Ronaldo, and Neymar in international matches.", "answer": "Lionel Messi, Cristiano Ronaldo, and Neymar have scored a total of 296 international goals combined."} +{"question": "Who invented the first programmable computer, and what is the inventor's name and place of birth?", "answer": "Charles Babbage invented the first programmable computer. He was born at 44 Crosby Row, Walworth Road, London, England."} +{"question": "How does the GDP per capita of the wealthiest country compare to that of the poorest country, in order of magnitude?", "answer": "The GDP per capita of the wealthiest country, Monaco, is approximately three orders of magnitude higher than that of the poorest country, Burundi."} \ No newline at end of file diff --git a/plan_and_execute/environment/Dockerfile b/plan_and_execute/environment/Dockerfile new file mode 100644 index 000000000..625e9aead --- /dev/null +++ b/plan_and_execute/environment/Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker/dockerfile:1 +FROM docker.io/continuumio/miniconda3:latest + +WORKDIR / + +COPY ./flow/requirements.txt /flow/requirements.txt + +RUN apt-get update && apt-get install -y runit gcc + +# create conda environment +RUN conda create -n promptflow-serve python=3.9.16 pip=23.0.1 -q -y && \ + conda run -n promptflow-serve \ + pip install -r /flow/requirements.txt && \ + conda run -n promptflow-serve pip install keyrings.alt && \ + conda run -n promptflow-serve pip install gunicorn==20.1.0 && \ + conda run -n promptflow-serve pip cache purge && \ + conda clean -a -y + +COPY ./flow /flow + +EXPOSE 8080 + +COPY ./connections/* /connections/ + +# reset runsvdir +RUN rm -rf /var/runit +COPY ./runit /var/runit +# grant permission +RUN chmod -R +x /var/runit + +COPY ./start.sh / +CMD ["bash", "./start.sh"] \ No newline at end of file diff --git a/plan_and_execute/experiment.yaml b/plan_and_execute/experiment.yaml new file mode 100644 index 000000000..0ed9e65ea --- /dev/null +++ b/plan_and_execute/experiment.yaml @@ -0,0 +1,37 @@ +azure_config: + subscription_id: ${SUBSCRIPTION_ID} + resource_group_name: ${RESOURCE_GROUP_NAME} + workspace_name: ${WORKSPACE_NAME} + keyvault_name: ${KEYVAULT_NAME} + compute_target: ${COMPUTE_TARGET} + +connections: +- connection: aoai + api_type: azure + api_key: ${AZURE_OPENAI_KEY} + api_base: ${AZURE_OPENAI_ENDPOINT} + api_version: ${AZURE_OPENAI_API_VERSION} + +experiment_config: + name: plan_and_execute + flow: flows/standard + + datasets: + - name: plan_and_execute_train + source: data/data.jsonl + description: "This dataset is for prompt experiments." + mappings: + question: "${data.question}" + + evaluators: + - name: evaluator + flow: flows/evaluation + datasets: + - name: plan_and_execute_test + reference: plan_and_execute_train + source: data/eval_data.jsonl + description: "This dataset is for evaluating flows." + mappings: + question: "${data.question}" + generated_answer: "${run.outputs.answer}" + ground_truth_answer: "${data.answer}" \ No newline at end of file diff --git a/plan_and_execute/experiment_dev.yaml b/plan_and_execute/experiment_dev.yaml new file mode 100644 index 000000000..86279bab0 --- /dev/null +++ b/plan_and_execute/experiment_dev.yaml @@ -0,0 +1,53 @@ +env_name: dev + +experiment_config: + +deployment_configs: + azure_managed_endpoint: + - name: azure_managed_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + ENDPOINT_NAME: + ENDPOINT_DESC: An online endpoint serving a flow for plan and execute + DEPLOYMENT_DESC: prompt flow deployment + PRIOR_DEPLOYMENT_NAME: + PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION: + CURRENT_DEPLOYMENT_NAME: + CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION: "100" + DEPLOYMENT_VM_SIZE: Standard_F4s_v2 + DEPLOYMENT_INSTANCE_COUNT: 1 + ENVIRONMENT_VARIABLES: + example-name: example-value + + kubernetes_endpoint: + - name: kubernetes_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + ENDPOINT_NAME: + ENDPOINT_DESC: An kubernetes endpoint serving a flow for plant and execute + DEPLOYMENT_DESC: prompt flow deployment + PRIOR_DEPLOYMENT_NAME: + PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION: + CURRENT_DEPLOYMENT_NAME: + CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION: 100, + COMPUTE_NAME: + DEPLOYMENT_VM_SIZE: + DEPLOYMENT_INSTANCE_COUNT: 1 + CPU_ALLOCATION: + MEMORY_ALLOCATION: + ENVIRONMENT_VARIABLES: + example-name: example-value + + webapp_endpoint: + - name: webapp_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + CONNECTION_NAMES: + - aoai + REGISTRY_NAME: + REGISTRY_RG_NAME: + APP_PLAN_NAME: + WEB_APP_NAME: + WEB_APP_RG_NAME: + WEB_APP_SKU: "B3" + USER_MANAGED_ID: \ No newline at end of file diff --git a/plan_and_execute/experiment_pr.yaml b/plan_and_execute/experiment_pr.yaml new file mode 100644 index 000000000..974221e90 --- /dev/null +++ b/plan_and_execute/experiment_pr.yaml @@ -0,0 +1,14 @@ +env_name: pr +experiment_config: + + name: plan_and_execute + + datasets: + - name: plan_and_execute_pr + source: data/data.jsonl + description: "This dataset is for pr validation only." + + mappings: + question: "${data.question}" + + evaluators: \ No newline at end of file diff --git a/plan_and_execute/figs/architecture.svg b/plan_and_execute/figs/architecture.svg new file mode 100644 index 000000000..0e184f521 --- /dev/null +++ b/plan_and_execute/figs/architecture.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/plan_and_execute/figs/connection.svg b/plan_and_execute/figs/connection.svg new file mode 100644 index 000000000..494a3eb36 --- /dev/null +++ b/plan_and_execute/figs/connection.svg @@ -0,0 +1 @@ +<your AOAI API key><your Bing Web Search API key><your GPT-4 deployment name><your GPT-35 deployment name><your AOAI base URL>2023-12-01-preview<Bing Web Search API endpoint> \ No newline at end of file diff --git a/plan_and_execute/flows/evaluation/evaluator.jinja2 b/plan_and_execute/flows/evaluation/evaluator.jinja2 new file mode 100644 index 000000000..fd7710d6f --- /dev/null +++ b/plan_and_execute/flows/evaluation/evaluator.jinja2 @@ -0,0 +1,30 @@ +system: +You are an assistant specialized in evaluate generated answers to questions, given a ground truth answer. +Given a question, a generated answer and a ground truth answer, you have to compare the generated answer with the ground truth answer, and evaluate it againts the question. +You should respond with "correct", "incorrect" or "unsure" depending on the quality of the generated answer. + +Use "correct" if: +- The generated answer and the ground truth answer have the same meaning. If there are quantitiess involved, the difference between the quantities should be less than 10%. +and +- The generated answer answers all aspects of the question in the same way as the ground truth answer. + +Use "incorrect" if: +- The generated answer and the ground truth answer have different meanings. +or +- The generated answer does not answer all aspects of the question. + +Use "unsure" if: +- You are not able to decide if the answer is correct or incorrect. In this case, you must provide your reasoning. + +Begin! + +Question: +{{question}} + +Generated answer: +{{generated_answer}} + +Ground truth answer: +{{ground_truth_answer}} + +Your evaluation: diff --git a/plan_and_execute/flows/evaluation/flow.dag.yaml b/plan_and_execute/flows/evaluation/flow.dag.yaml new file mode 100644 index 000000000..bc46019c4 --- /dev/null +++ b/plan_and_execute/flows/evaluation/flow.dag.yaml @@ -0,0 +1,28 @@ +inputs: + question: + type: string + default: What was the total box office performance of 'Inception' and 'Interstellar' together? + generated_answer: + type: string + default: The total box office performance of 'Inception' and 'Interstellar' together was $1,509,329,092. + ground_truth_answer: + type: string + default: The total box office performance of 'Inception' and 'Interstellar' together was $1,509,329,092. +outputs: + evaluation: + type: string + reference: ${evaluator.output} +nodes: +- name: evaluator + type: llm + source: + type: code + path: evaluator.jinja2 + inputs: + deployment_name: gpt-4 + temperature: 0.5 + question: ${inputs.question} + generated_answer: ${inputs.generated_answer} + ground_truth_answer: ${inputs.ground_truth_answer} + connection: aoai + api: chat diff --git a/plan_and_execute/flows/standard/connection_utils.py b/plan_and_execute/flows/standard/connection_utils.py new file mode 100644 index 000000000..66b2713ba --- /dev/null +++ b/plan_and_execute/flows/standard/connection_utils.py @@ -0,0 +1,41 @@ +"""This are helper classes to provide custom connection information in promptflow.""" +from promptflow.connections import CustomStrongTypeConnection +from promptflow.contracts.types import Secret + + +class CustomConnection(CustomStrongTypeConnection): + """Define the custom connection keys and values. + + :param aoai_api_key: The api key for Azure Open AI. + :type aoai_api_key: Secret + :param bing_api_key: The api key for the Bing Search. + :type bing_api_key: Secret + :param aoai_model_gpt4: The deployment name for the GPT-4 model. + :type aoai_model_gpt4: String + :param aoai_model_gpt35: The deployment name for the GPT-3.5 model. + :type aoai_model_gpt35: String + :param aoai_base_url: The base url for the Azure Open AI. + :type aoai_base_url: String + :param aoai_api_version: The api version for the Azure Open AI. + :type aoai_api_version: String + :param bing_endpoint: The endpoint for the Bing Search. + :type bing_endpoint: String + """ + + aoai_api_key: Secret + bing_api_key: Secret + aoai_model_gpt4: str + aoai_model_gpt35: str + aoai_base_url: str + aoai_api_version: str + bing_endpoint: str + + +class ConnectionInfo(object): + """Singleton class to store connection information.""" + + def __new__(cls): + """Store connection information.""" + if not hasattr(cls, 'instance'): + cls.instance = super(ConnectionInfo, cls).__new__(cls) + return cls.instance diff --git a/plan_and_execute/flows/standard/docker/dockerfile b/plan_and_execute/flows/standard/docker/dockerfile new file mode 100644 index 000000000..942c8c97e --- /dev/null +++ b/plan_and_execute/flows/standard/docker/dockerfile @@ -0,0 +1,3 @@ +FROM mcr.microsoft.com/azureml/promptflow/promptflow-runtime:latest +COPY ./requirements.txt . +RUN pip install -r requirements.txt \ No newline at end of file diff --git a/plan_and_execute/flows/standard/executor.py b/plan_and_execute/flows/standard/executor.py new file mode 100644 index 000000000..8a1095216 --- /dev/null +++ b/plan_and_execute/flows/standard/executor.py @@ -0,0 +1,235 @@ +"""Executor node of the plan_and_execute flow.""" + +import concurrent.futures +import json +from promptflow.core import tool +from autogen import UserProxyAgent, AssistantAgent +from connection_utils import CustomConnection, ConnectionInfo +from tools import register_tools + + +def prepare_connection_info(connection): + """Prepare the connection info for the agents.""" + return { + "aoai_model_gpt35": connection.configs["aoai_model_gpt35"], + "aoai_model_gpt4": connection.configs["aoai_model_gpt4"], + "aoai_api_key": connection.secrets["aoai_api_key"], + "aoai_base_url": connection.configs["aoai_base_url"], + "aoai_api_version": connection.configs["aoai_api_version"], + "bing_api_key": connection.secrets["bing_api_key"], + "bing_endpoint": connection.configs["bing_endpoint"] + } + + +def prepare_executor(connection_info): + """Prepare the executor agent.""" + config_list_gpt35 = [ + { + "model": connection_info["aoai_model_gpt35"], + "api_key": connection_info["aoai_api_key"], + "base_url": connection_info["aoai_base_url"], + "api_type": "azure", + "api_version": connection_info["aoai_api_version"] + } + ] + executor = UserProxyAgent( + name="EXECUTOR", + description=( + "An agent that acts as a proxy for the user and executes the " + "suggested function calls from PLANNER." + ), + code_execution_config=False, + llm_config={ + "config_list": config_list_gpt35, + "timeout": 60, + "cache_seed": None + }, + human_input_mode="NEVER" + ) + return executor, config_list_gpt35 + + +def llm_tool(request, context, config_list_gpt35): + """Define the LLM agent.""" + llm_assistant = AssistantAgent( + name="LLM_ASSISTANT", + description=( + "An agent expert in answering requests by analyzing and " + "extracting information from the given context." + ), + system_message=( + "Given a request and optionally some context with potentially " + "relevant information to answer it, analyze the context and " + "extract the information needed to answer the request. Then, " + "create a sentence that answers the request. You must strictly " + "limit your response to only what was asked in the request." + ), + code_execution_config=False, + llm_config={ + "config_list": config_list_gpt35, + "timeout": 60, + "temperature": 0.3, + "cache_seed": None + } + ) + + llm_assistant.clear_history() + + message = f""" + Request: + {request} + + Context: + {context} + """ + try: + reply = llm_assistant.generate_reply( + messages=[{"content": message, "role": "user"}] + ) + return reply + except Exception as e: + return f"Error: {str(e)}" + + +def substitute_dependency(id, original_argument_value, dependency_value, config_list_gpt35): + """Substitute dependencies in the execution plan.""" + instruction = ( + "Extract the entity name or fact from the dependency value in a way " + "that makes sense to use it to substitute the variable #E in the " + "original argument value. Do not include any other text in your " + "response, other than the entity name or fact extracted." + ) + + context = f""" + original argument value: + {original_argument_value} + + dependency value: + {dependency_value} + + extracted fact or entity: + + """ + + return llm_tool(instruction, context, config_list_gpt35) + + +def has_unresolved_dependencies(item, resolved_ids, plan_ids): + """Check for unresolved dependencies in a plan step.""" + try: + args = json.loads(item['function']['arguments']) + except json.JSONDecodeError: + return False + + for arg in args.values(): + if isinstance(arg, str) and any( + ref_id for ref_id in plan_ids + if ref_id not in resolved_ids and ref_id in arg + ): + return True + return False + + +def submit_task(item_id, item, thread_executor, executor_agent, futures): + """Submit a task for execution.""" + arguments = item['function']['arguments'] + future = thread_executor.submit( + executor_agent.execute_function, + {'name': item['function']['name'], 'arguments': arguments} + ) + futures[item_id] = future + + +def process_done_future( + future, futures, results, resolved_ids, plan_ids, thread_executor, executor_agent, config_list_gpt35 +): + """Process a completed future and trigger the submission of ready tasks.""" + item_id = next((id for id, f in futures.items() if f == future), None) + if item_id: + _, result = future.result() + results[item_id] = result + resolved_ids.add(item_id) + del futures[item_id] + submit_ready_tasks(plan_ids, resolved_ids, futures, results, + thread_executor, executor_agent, config_list_gpt35) + + +def submit_ready_tasks( + plan_ids, resolved_ids, futures, results, thread_executor, executor_agent, config_list_gpt35 +): + """Submit plan tasks that have all dependencies resolved and are ready to be executed.""" + for next_item_id, next_item in plan_ids.items(): + if ( + next_item_id not in resolved_ids + and next_item_id not in futures + and not has_unresolved_dependencies(next_item, resolved_ids, plan_ids) + ): + update_and_submit_task(next_item_id, next_item, thread_executor, + executor_agent, futures, results, config_list_gpt35) + + +def update_and_submit_task( + item_id, item, thread_executor, executor_agent, futures, results, config_list_gpt35 +): + """Update the arguments of a task with dependency results and submit it for execution.""" + updated_arguments = json.loads(item['function']['arguments']) + for arg_key, arg_value in updated_arguments.items(): + if isinstance(arg_value, str): + for res_id, res in results.items(): + if arg_key == "context": + arg_value = arg_value.replace(res_id, res['content']) + else: + arg_value = arg_value.replace( + res_id, + substitute_dependency(res_id, arg_value, res['content'], config_list_gpt35) + ) + updated_arguments[arg_key] = arg_value + future = thread_executor.submit( + executor_agent.execute_function, + { + 'name': item['function']['name'], + 'arguments': json.dumps(updated_arguments) + } + ) + futures[item_id] = future + + +def execute_plan_parallel(plan, executor_agent, config_list_gpt35): + """Execute the plan in parallel.""" + plan_ids = {item['id']: item for item in plan} + results = {} + resolved_ids = set() + futures = {} + + with concurrent.futures.ThreadPoolExecutor() as thread_executor: + for item_id, item in plan_ids.items(): + if not has_unresolved_dependencies(item, resolved_ids, plan_ids): + submit_task(item_id, item, thread_executor, executor_agent, futures) + + while futures: + done, _ = concurrent.futures.wait( + futures.values(), return_when=concurrent.futures.FIRST_COMPLETED + ) + for future in done: + process_done_future(future, futures, results, resolved_ids, plan_ids, + thread_executor, executor_agent, config_list_gpt35) + + result_str = '\n'.join( + [f"{key} = {value['content']}" for key, value in results.items()] + ) + return result_str + + +@tool +def worker_tool(connection: CustomConnection, plan: str) -> str: + """Execute the plan generated by the planner node.""" + connection_info = prepare_connection_info(connection) + ConnectionInfo().connection_info = connection_info + + executor, config_list_gpt35 = prepare_executor(connection_info) + register_tools(executor) + + plan = json.loads(plan) + executor_reply = execute_plan_parallel(plan['Functions'], executor, config_list_gpt35) + + return executor_reply diff --git a/plan_and_execute/flows/standard/flow.dag.yaml b/plan_and_execute/flows/standard/flow.dag.yaml new file mode 100644 index 000000000..2496617c6 --- /dev/null +++ b/plan_and_execute/flows/standard/flow.dag.yaml @@ -0,0 +1,48 @@ +inputs: + question: + type: string + default: What was the total box office performance of 'Inception' and 'Interstellar' together? +outputs: + answer: + type: string + reference: ${solver.output} +nodes: +- name: planner_system_prompt + type: prompt + source: + type: code + path: planner_system_prompt.jinja2 + inputs: {} +- name: planner + type: python + source: + type: code + path: planner.py + inputs: + connection: plan_execute_agent_connection + system_message: ${planner_system_prompt.output} + question: ${inputs.question} +- name: executor + type: python + source: + type: code + path: executor.py + inputs: + connection: plan_execute_agent_connection + plan: ${planner.output} +- name: solver_system_prompt + type: prompt + source: + type: code + path: solver_system_prompt.jinja2 + inputs: {} +- name: solver + type: python + source: + type: code + path: solver.py + inputs: + connection: plan_execute_agent_connection + system_message: ${solver_system_prompt.output} + question: ${inputs.question} + results: ${executor.output} diff --git a/plan_and_execute/flows/standard/planner.py b/plan_and_execute/flows/standard/planner.py new file mode 100644 index 000000000..e230ff8ce --- /dev/null +++ b/plan_and_execute/flows/standard/planner.py @@ -0,0 +1,42 @@ +"""Planner node for the plan_and_execute flow.""" +from promptflow.core import tool +from autogen import AssistantAgent +from connection_utils import CustomConnection +from tools import register_tools + + +# The inputs section will change based on the arguments of the tool function, after you save the code +# Adding type to arguments and return value will help the system show the types properly +# Please update the function name/signature per need +@tool +def planner_tool(connection: CustomConnection, system_message: str, question: str) -> str: + """Generate a step-by-step execution plan to solve the user's request.""" + config_list_gpt4 = [{ + "model": connection.configs["aoai_model_gpt4"], + "api_key": connection.secrets["aoai_api_key"], + "base_url": connection.configs["aoai_base_url"], + "api_type": "azure", + "api_version": connection.configs["aoai_api_version"] + }] + + planner = AssistantAgent( + name="PLANNER", + description=""" + An agent expert in creating a step-by-step execution plan to solve the user's request. + """, + system_message=system_message, + code_execution_config=False, + llm_config={ + "config_list": config_list_gpt4, + "temperature": 0, + "timeout": 120, + "cache_seed": None + } + ) + + register_tools(planner) + + planner_reply = planner.generate_reply(messages=[{"content": question, "role": "user"}]) + planner_reply = planner_reply.replace("```json", "").replace("```", "").strip() + + return planner_reply diff --git a/plan_and_execute/flows/standard/planner_system_prompt.jinja2 b/plan_and_execute/flows/standard/planner_system_prompt.jinja2 new file mode 100644 index 000000000..5e9347adc --- /dev/null +++ b/plan_and_execute/flows/standard/planner_system_prompt.jinja2 @@ -0,0 +1,119 @@ +For the given question, you make a plan that can solve the problem step by step. For each plan step, +indicate which external function should be used to retrieve evidence, together with the function arguments. +For each function call, you must use '#En' as the id for the function call, where n is the plan step number. +When you need to reference the output of a function call as an argument or part of an argument to another function call, +you must reference the corresponding '#En' in the argument, as a comma separated string, such as: "#E1, #E2, ...". +You must follow the JSON schema provided in the examples below. + +Examples: + +Question: Were Pavel Urysohn and Leonid Levin known for the same type of work? +{"Plan": ["Search Wikipedia for Pavel Urysohn.", + "Search Wikipedia for Leonid Levin.", + "Use LLM to compare the two and determine if they were known for the same type of work."], + "Functions": [ + {"id": "#E1", + "function": { + "arguments": "{\"query\": \"Pavel Urysohn\"}", + "name": "wikipedia_tool"}, + "type": "function"}, + {"id": "#E2", + "function": { + "arguments": "{\"query\": \"Leonid Levin\"}", + "name": "wikipedia_tool"}, + "type": "function"}, + {"id": "#E3", + "function": { + "arguments": "{\"request\": \"Were Pavel Urysohn and Leonid Levin known for the same type of work\", \"context\": \"#E1, #E2\"}", + "name": "llm_tool"}, + "type": "function"}]} + +Question: What is the hometown of the 2024 australian open winner? +{"Plan": ["Search the Web for the name of the 2024 Australian Open winner.", + "Search Wikipedia for more information about the 2024 Australian Open winner.", + "Use LLM to find the hometown of the 2024 Australian Open winner."], + "Functions": [ + {"id": "#E1", + "function": { + "arguments": "{\"query\": \"2024 Australian Open winner\"}", + "name": "web_tool"}, + "type": "function"}, + {"id": "#E2", + "function": { + "arguments": "{\"query\": \"#E1\"}", + "name": "wikipedia_tool"}, + "type": "function"}, + {"id": "#E3", + "function": { + "arguments": "{\"request\": \"Find the hometown of the 2024 Australian Open Winner\", \"context\": \"#E1, #E2\"}", + "name": "llm_tool"}, + "type": "function"}]} + +Question: What is the combined age of the latest 2 former Unites States presidents when they left office? +{"Plan": ["Search the Web for the age of the most recent former President of the United States when they left office.", + "Search the Web for the age of the second most recent former President of the United States when they left office.", + "Use Math to add the ages of the two former Presidents when they left office."], + "Functions": [ + {"id": "#E1", + "function": { + "arguments": "{\"query\": \"Age of most recent former President of United States when left office\"}", + "name": "web_tool"}, + "type": "function"}, + {"id": "#E2", + "function": { + "arguments": "{\"query\": \"Age of second most recent former President of United States when left office\"}", + "name": "web_tool"}, + "type": "function"}, + {"id": "#E3", + "function": { + "arguments": "{\"problem_description\": \"Add the ages of the two former Presidents when they left office\", \"context\": \"#E1, #E2\"}", + "name": "math_tool"}, + "type": "function"}]} + +Question: What is the sum of the GDPs of the top 3 countries with the highest GDPs? +{"Plan": ["Search the Web for the GDPs of the top 3 countries with the highest GDPs.", + "Use LLM to extract the GDP of the first country with the highest GDP.", + "Use LLM to extract the GDP of the second country with the highest GDP.", + "Use LLM to extract the GDP of the third country with the highest GDP.", + "Use Math to add the GDPs of the top 3 countries with the highest GDPs."], +"Functions": [ +{"id": "#E1", +"function": { + "arguments": "{\"query\": \"GDPs of the top 3 countries with the highest GDPs\"}", + "name": "web_tool"}, +"type": "function"}, +{"id": "#E2", +"function": { + "arguments": "{\"request\": \"GDP of the first country with the highest GDP\", \"context\": \"#E1\"}", + "name": "llm_tool"}, +"type": "function"}, +{"id": "#E3", +"function": { + "arguments": "{\"request\": \"GDP of the second country with the highest GDP\", \"context\": \"#E1\"}", + "name": "llm_tool"}, +"type": "function"}, +{"id": "#E4", +"function": { + "arguments": "{\"request\": \"GDP of the third country with the highest GDP\", \"context\": \"#E1\"}", + "name": "llm_tool"}, +"type": "function"}, +{"id": "#E5", +"function": { + "arguments": "{\"problem_description\": \"Add the GDPs of the top 3 countries\", \"context\": \"#E2, #E3, #E4\"}", + "name": "math_tool"}, +"type": "function"}]} + +You should describe your plans with rich details. Each plan step should correspond to only one function call. +Make sure you don't include redundant or irrelevant plan steps and make the plan as efficient as possible. +When uisng web_tool, make your query as specific as possible to get the most relevant information. +When using wikipedia_tool, make sure your query specifies a single person name, place, entity or concept only, or a single '#En'. + +Do not respond with actual function calls. Follow the response format in the examples above instead. +Do not use any prefix for the function names. +You must respond with a valid JSON string only. Do not include any other text in your response. +Remember that ou must follow the JSON schema provided in the examples above. +Very important: the value for the "arguments" key has to be a string that represents a JSON object. + +Begin! + +Question: diff --git a/plan_and_execute/flows/standard/requirements.txt b/plan_and_execute/flows/standard/requirements.txt new file mode 100644 index 000000000..f81307193 --- /dev/null +++ b/plan_and_execute/flows/standard/requirements.txt @@ -0,0 +1,5 @@ +promptflow +pyautogen +bs4 +wikipedia +numexpr \ No newline at end of file diff --git a/plan_and_execute/flows/standard/solver.py b/plan_and_execute/flows/standard/solver.py new file mode 100644 index 000000000..6d155fce9 --- /dev/null +++ b/plan_and_execute/flows/standard/solver.py @@ -0,0 +1,44 @@ + +"""Solver node for the plan_and_execute flow.""" +from promptflow.core import tool +from autogen import AssistantAgent +from connection_utils import CustomConnection + + +# The inputs section will change based on the arguments of the tool function, after you save the code +# Adding type to arguments and return value will help the system show the types properly +# Please update the function name/signature per need +@tool +def solver_tool(connection: CustomConnection, system_message: str, question: str, results: str) -> str: + """Create a final response to the user's request.""" + config_list_gpt4 = [{ + "model": connection.configs["aoai_model_gpt4"], + "api_key": connection.secrets["aoai_api_key"], + "base_url": connection.configs["aoai_base_url"], + "api_type": "azure", + "api_version": connection.configs["aoai_api_version"] + }] + + solver = AssistantAgent( + name="SOLVER", + description=""" + An agent expert in creating a final response to the user's request. + """, + system_message=system_message, + code_execution_config=False, + llm_config={ + "config_list": config_list_gpt4, + "timeout": 60, + "cache_seed": None + } + ) + + solver_message = f""" + Question: + {question} + + Step results: + {results} + """ + + return solver.generate_reply(messages=[{"content": solver_message, "role": "user"}]) diff --git a/plan_and_execute/flows/standard/solver_system_prompt.jinja2 b/plan_and_execute/flows/standard/solver_system_prompt.jinja2 new file mode 100644 index 000000000..e033f15c8 --- /dev/null +++ b/plan_and_execute/flows/standard/solver_system_prompt.jinja2 @@ -0,0 +1,6 @@ +For the given question and plan steps results, you synthesize a final response to the user. +You should use analyze the steps results to find the necessary information to synthesize the final response. +Do not include any explanation regarding the plan steps results in the final response. +If there is not enough information in the plan steps results to synthesize the final response, you should try to construct a response using your own knowledge and indicate that in the response. +You must limit your response to only what was asked in the question, without writing any further explanations. + diff --git a/plan_and_execute/flows/standard/tools.py b/plan_and_execute/flows/standard/tools.py new file mode 100644 index 000000000..ec585fe0d --- /dev/null +++ b/plan_and_execute/flows/standard/tools.py @@ -0,0 +1,270 @@ +"""Tools definitions for AutoGen.""" +from autogen import AssistantAgent, UserProxyAgent +from autogen.agentchat import register_function +from connection_utils import ConnectionInfo +from typing_extensions import Annotated, Optional + +tool_descriptions = { + "web_tool": { + "function": ( + "Worker that searches results from the internet. Useful when you need to find short and succinct " + "answers about a specific topic." + ), + "query": "The search query string.", + "number_of_results": "The number of search results to return." + }, + "wikipedia_tool": { + "function": ( + "Worker that search for page contents from Wikipedia. Useful when you need to get holistic " + "knowledge about people, places, companies, historical events, or other subjects. You use it when you " + "already have identified the entity name, usually after searching for the entity name using web_tool." + ), + "query": "The single person name, entity, or concept to be searched.", + "number_of_results": "The number of search results to return." + }, + "llm_tool": { + "function": ( + "An agent expert in solving problems by analyzing and extracting information from the given " + "context. It should never be used to do calculations." + ), + "request": "The request to be answered.", + "context": "Context with the relevant information to answer the request." + }, + "math_tool": { + "function": ( + "A tool that can solve math problems by computing arithmetic expressions. It must be used " + "whenever you need to do calculations or solve math problems. " + "You can use it to solve simple or complex math problems." + ), + "problem_description": "The problem to be solved.", + "context": "Context with the relevant information to solve the problem." + } +} + + +def register_tools(agent): + """Register tools for the agent.""" + for tool in tool_descriptions.keys(): + register_function( + globals()[tool], caller=agent, executor=agent, + description=tool_descriptions[tool]["function"] + ) + + +def llm_tool( + request: Annotated[str, tool_descriptions["llm_tool"]["request"]], + context: Optional[Annotated[str, tool_descriptions["llm_tool"]["context"]]] = None +) -> str: + """Use an LLM to analyze and extract information from the given context to answer the request.""" + connection_info = ConnectionInfo().connection_info + + try: + llm_assistant = AssistantAgent( + name="LLM_ASSISTANT", + description=( + "An agent expert in answering requests by analyzing and extracting information from the given context." + ), + system_message=( + "Given a request and optionally some context with potentially relevant information to answer it, " + "analyze the context and extract the information needed to answer the request. " + "Then, create a sentence that answers the request. " + "You must strictly limit your response to only what was asked in the request." + ), + code_execution_config=False, + llm_config={ + "config_list": [ + { + "model": connection_info["aoai_model_gpt35"], + "api_key": connection_info["aoai_api_key"], + "base_url": connection_info["aoai_base_url"], + "api_type": "azure", + "api_version": connection_info["aoai_api_version"] + } + ], + "timeout": 60, + "temperature": 0.3, + "cache_seed": None + } + ) + except Exception as e: + print("LLM_ASSISTANT error:", e) + return "" + + llm_assistant.clear_history() + + message = f""" + Request: + {request} + + Context: + {context} + """ + try: + reply = llm_assistant.generate_reply(messages=[{"content": message, "role": "user"}]) + return reply + except Exception as e: + return f"Error: {str(e)}" + + +def web_tool( + query: Annotated[str, tool_descriptions["web_tool"]["query"]], + number_of_results: Optional[Annotated[int, tool_descriptions["web_tool"]["number_of_results"]]] = 3 +) -> list: + """Search results from the internet.""" + import requests + from bs4 import BeautifulSoup + + connection_info = ConnectionInfo().connection_info + + headers = {"Ocp-Apim-Subscription-Key": connection_info["bing_api_key"]} + params = { + "q": query, "count": number_of_results, "offset": 0, "mkt": "en-US", "safesearch": "Strict", + "textDecorations": False, "textFormat": "HTML" + } + response = requests.get(connection_info["bing_endpoint"], headers=headers, params=params) + response.raise_for_status() + results = response.json() + + search_results = [] + for i in range(number_of_results): + title = results["webPages"]["value"][i]["name"] + url = results["webPages"]["value"][i]["url"] + snippet = results["webPages"]["value"][i]["snippet"] + + try: + response = requests.get(url) + if response.status_code == 200: + soup = BeautifulSoup(response.content, 'html.parser') + text = soup.get_text(separator=' ', strip=True) + text = text[:5000] + else: + text = f"Failed to fetch content, status code: {response.status_code}" + except Exception as e: + text = f"Error fetching the page: {str(e)}" + + search_results.append({"title": title, "url": url, "snippet": snippet, "content": text}) + + return llm_tool(query, search_results) + + +def wikipedia_tool( + query: Annotated[str, tool_descriptions["wikipedia_tool"]["query"]], + number_of_results: Optional[Annotated[int, tool_descriptions["wikipedia_tool"]["number_of_results"]]] = 3 +) -> list: + """Search for page contents from Wikipedia.""" + import wikipedia + + wikipedia.set_lang("en") + results = wikipedia.search(query, results=number_of_results) + + search_results = [] + + for title in results: + try: + page = wikipedia.page(title) + search_results.append({"title": page.title, "url": page.url, "content": page.content[:5000]}) + except wikipedia.exceptions.DisambiguationError: + continue + except wikipedia.exceptions.PageError: + continue + except Exception as e: + search_results.append(f"Error fetching the page: {str(e)}") + + return search_results + + +def math_tool( + problem_description: Annotated[str, tool_descriptions["math_tool"]["problem_description"]], + context: Optional[Annotated[str, tool_descriptions["math_tool"]["context"]]] = None +) -> str: + """Solve math problems by computing arithmetic expressions.""" + connection_info = ConnectionInfo().connection_info + + def is_termination_msg(content): + have_content = content.get("content", None) is not None + if have_content and "TERMINATE" in content["content"]: + return True + return False + + math_assistant = AssistantAgent( + name="MATH_ASSISTANT", + description="An agent expert in solving math problems and math expressions.", + system_message=( + "Given a math problem and optionally some context with relevant information to solve the problem, " + "translate the math problem into an expression that can be executed using Python's numexpr library. " + "Then, use the available tool (evaluate_math_expression) to solve the expression and return the result. " + "Reply 'TERMINATE' in the end when everything is done." + ), + code_execution_config=False, + is_termination_msg=is_termination_msg, + llm_config={ + "config_list": [ + { + "model": connection_info["aoai_model_gpt4"], + "api_key": connection_info["aoai_api_key"], + "base_url": connection_info["aoai_base_url"], + "api_type": "azure", + "api_version": connection_info["aoai_api_version"] + } + ], + "timeout": 60, + "cache_seed": None + } + ) + + math_executor = UserProxyAgent( + name="TOOL_EXECUTOR", + description=( + "An agent that acts as a proxy for the user and executes " + "the suggested function calls from MATH_ASSISTANT." + ), + code_execution_config=False, + is_termination_msg=is_termination_msg, + human_input_mode="NEVER" + ) + + tool_descriptions = { + "evaluate_math_expression": { + "function": "Function to evaluate math expressions using Python's numexpr library.", + "expression": "The expression to be evaluated. It should be a valid numerical expression." + } + } + + @math_executor.register_for_execution() + @math_assistant.register_for_llm(description=tool_descriptions["evaluate_math_expression"]["function"]) + def evaluate_math_expression( + expression: Annotated[str, tool_descriptions["evaluate_math_expression"]["expression"]] + ) -> str: + import math + import numexpr + import re + try: + local_dict = {"pi": math.pi, "e": math.e} + output = str( + numexpr.evaluate( + expression.strip(), + global_dict={}, # restrict access to globals + local_dict=local_dict, # add common mathematical functions + ) + ) + except Exception as e: + raise ValueError( + f'Failed to evaluate "{expression}". Raised error: {repr(e)}. ' + "Please try again with a valid numerical expression." + ) + + return re.sub(r"^\[|\]$", "", output) + + message = f""" + Problem: + {problem_description} + + Context: + {context} + """ + math_assistant.clear_history() + math_executor.clear_history() + + math_executor.initiate_chat(message=message, recipient=math_assistant, silent=True, clear_history=True) + result = math_executor.last_message()['content'].split("TERMINATE")[0].strip() + return result diff --git a/plan_and_execute/sample-request.json b/plan_and_execute/sample-request.json new file mode 100644 index 000000000..bb902951f --- /dev/null +++ b/plan_and_execute/sample-request.json @@ -0,0 +1,3 @@ +{ + "question": "What was the total box office performance of 'Inception' and 'Interstellar' together?" +} \ No newline at end of file diff --git a/plan_and_execute/tests/test_delete_this_file.py b/plan_and_execute/tests/test_delete_this_file.py new file mode 100644 index 000000000..3692b2a7b --- /dev/null +++ b/plan_and_execute/tests/test_delete_this_file.py @@ -0,0 +1,6 @@ +def test_print(): + try: + print("Hello") is None + except: + print("Test print function failed.") + assert False \ No newline at end of file diff --git a/web_classification/configs/deployment_config.json b/web_classification/configs/deployment_config.json deleted file mode 100644 index 7ea59719b..000000000 --- a/web_classification/configs/deployment_config.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "azure_managed_endpoint":[ - { - "ENV_NAME": "dev", - "TEST_FILE_PATH": "sample-request.json", - "ENDPOINT_NAME": "", - "ENDPOINT_DESC": "An AML compute based endpoint serving a flow for web classification", - "DEPLOYMENT_DESC": "prompt flow deployment", - "PRIOR_DEPLOYMENT_NAME": "", - "PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION": "0", - "CURRENT_DEPLOYMENT_NAME": "", - "CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION": "100", - "DEPLOYMENT_VM_SIZE": "Standard_E16s_v3", - "DEPLOYMENT_INSTANCE_COUNT": 1, - "ENVIRONMENT_VARIABLES": { - "example-name": "example-value" - } - } - ], - "kubernetes_endpoint":[ - { - "ENV_NAME": "dev", - "TEST_FILE_PATH": "sample-request.json", - "ENDPOINT_NAME": "", - "ENDPOINT_DESC": "An kubernetes endpoint serving a flow for web classification", - "DEPLOYMENT_DESC": "prompt flow deployment", - "PRIOR_DEPLOYMENT_NAME": "", - "PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION": "", - "CURRENT_DEPLOYMENT_NAME": "", - "CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION": 100, - "COMPUTE_NAME": "", - "DEPLOYMENT_VM_SIZE": "", - "DEPLOYMENT_INSTANCE_COUNT": 1, - "CPU_ALLOCATION": "", - "MEMORY_ALLOCATION": "", - "ENVIRONMENT_VARIABLES": { - "example-name": "example-value" - } - } - ], - "webapp_endpoint":[ - { - "ENV_NAME": "dev", - "TEST_FILE_PATH": "sample-request.json", - "CONNECTION_NAMES": ["aoai"], - "REGISTRY_NAME": "", - "REGISTRY_RG_NAME": "", - "APP_PLAN_NAME": "", - "WEB_APP_NAME": "", - "WEB_APP_RG_NAME": "", - "WEB_APP_SKU": "B3", - "USER_MANAGED_ID": "" - - } - ] -} \ No newline at end of file diff --git a/web_classification/experiment.dev.yaml b/web_classification/experiment.dev.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/web_classification/experiment.pr.yaml b/web_classification/experiment.pr.yaml deleted file mode 100644 index 91f6feb7c..000000000 --- a/web_classification/experiment.pr.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: web_classification - -datasets: -- name: web_classification_pr - source: data/data.jsonl - description: "This dataset is for pr validation only." - mappings: - url: "${data.url}" - -evaluators: \ No newline at end of file diff --git a/web_classification/experiment.yaml b/web_classification/experiment.yaml index ce4b2d90d..2ff3fa06b 100644 --- a/web_classification/experiment.yaml +++ b/web_classification/experiment.yaml @@ -1,36 +1,62 @@ -name: web_classification -flow: flows/experiment - -datasets: -- name: web_classification_train - source: data/data.jsonl - description: "This dataset is for prompt experiments." - mappings: - url: "${data.url}" -- name: web_classification_train_new - source: data/data_new.jsonl - description: "This dataset is for prompt experiments." - mappings: - url: "${data.url}" - -evaluators: -- name: web_classification_flow - flow: flows/evaluation - datasets: - - name: web_classification_train_eval - reference: web_classification_train - source: data/eval_data.jsonl - description: "This dataset is for evaluating flows." - mappings: - groundtruth: "${data.answer}" - prediction: "${run.outputs.category}" -- name: web_classification_flow_adv - flow: flows/evaluation_adv - datasets: - - name: web_classification_train_new_eval - reference: web_classification_train_new - source: data/eval_data_new.jsonl - description: "This dataset is for evaluating flows." - mappings: - groundtruth: "${data.answer}" - prediction: "${run.outputs.category}" + +azure_config: + subscription_id: ${SUBSCRIPTION_ID} + resource_group_name: ${RESOURCE_GROUP_NAME} + workspace_name: ${WORKSPACE_NAME} + keyvault_name: ${KEYVAULT_NAME} + compute_target: ${COMPUTE_TARGET} + +connections: +- connection: aoai + connection_type: AzureOpenAIConnection + config: + name: aoai + api_type: Azure + api_key: ${AZURE_OPENAI_KEY} + azure_endpoint: ${AZURE_OPENAI_ENDPOINT} + api_version: ${AZURE_OPENAI_API_VERSION} +- connection : ai_search + connection_type : AzureAISearchConnection + config: + name: ai_search + endpoint: ${AZURE_AI_SEARCH_ENDPOINT} + api_key: ${AZURE_AI_SEARCH_APIKEY} + +experiment_config: + name: web_classification + flow: flows/experiment + + datasets: + - name: web_classification_train + source: data/data.jsonl + description: "This dataset is for prompt experiments." + mappings: + url: "${data.url}" + - name: web_classification_train_new + source: data/data_new.jsonl + description: "This dataset is for prompt experiments." + mappings: + url: "${data.url}" + + evaluators: + - name: web_classification_flow + flow: flows/evaluation + datasets: + - name: web_classification_train_eval + reference: web_classification_train + source: data/eval_data.jsonl + description: "This dataset is for evaluating flows." + mappings: + groundtruth: "${data.answer}" + prediction: "${run.outputs.category}" + + - name: web_classification_flow_adv + flow: flows/evaluation_adv + datasets: + - name: web_classification_train_new_eval + reference: web_classification_train_new + source: data/eval_data_new.jsonl + description: "This dataset is for evaluating flows." + mappings: + groundtruth: "${data.answer}" + prediction: "${run.outputs.category}" diff --git a/web_classification/experiment_dev.yaml b/web_classification/experiment_dev.yaml new file mode 100644 index 000000000..23c3ed808 --- /dev/null +++ b/web_classification/experiment_dev.yaml @@ -0,0 +1,53 @@ +env_name: dev + +experiment_config: + +deployment_configs: + azure_managed_endpoint: + - name: azure_managed_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + ENDPOINT_NAME: + ENDPOINT_DESC: An AML compute based endpoint serving a flow for web classification + DEPLOYMENT_DESC: prompt flow deployment + PRIOR_DEPLOYMENT_NAME: + PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION: "0" + CURRENT_DEPLOYMENT_NAME: + CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION: "100" + DEPLOYMENT_VM_SIZE: Standard_E16s_v3 + DEPLOYMENT_INSTANCE_COUNT: 1 + ENVIRONMENT_VARIABLES: + example-name: example-value + + kubernetes_endpoint: + - name: kubernetes_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + ENDPOINT_NAME: + ENDPOINT_DESC: An kubernetes endpoint serving a flow for web classification + DEPLOYMENT_DESC: prompt flow deployment + PRIOR_DEPLOYMENT_NAME: + PRIOR_DEPLOYMENT_TRAFFIC_ALLOCATION: + CURRENT_DEPLOYMENT_NAME: + CURRENT_DEPLOYMENT_TRAFFIC_ALLOCATION: 100, + COMPUTE_NAME: + DEPLOYMENT_VM_SIZE: + DEPLOYMENT_INSTANCE_COUNT: 1 + CPU_ALLOCATION: + MEMORY_ALLOCATION: + ENVIRONMENT_VARIABLES: + example-name: example-value + + webapp_endpoint: + - name: webapp_endpoint_1 + ENV_NAME: dev + TEST_FILE_PATH: sample-request.json + CONNECTION_NAMES: + - aoai + REGISTRY_NAME: + REGISTRY_RG_NAME: + APP_PLAN_NAME: + WEB_APP_NAME: + WEB_APP_RG_NAME: + WEB_APP_SKU: "B3" + USER_MANAGED_ID: \ No newline at end of file diff --git a/web_classification/experiment_pr.yaml b/web_classification/experiment_pr.yaml new file mode 100644 index 000000000..1313dc6a2 --- /dev/null +++ b/web_classification/experiment_pr.yaml @@ -0,0 +1,12 @@ +env_name: pr +experiment_config: + name: web_classification + + datasets: + - name: web_classification_pr + source: data/data.jsonl + description: "This dataset is for pr validation only." + mappings: + url: "${data.url}" + + evaluators: \ No newline at end of file