diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4787202 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +demo/ diff --git a/.env b/.env deleted file mode 100644 index e3d2dcc..0000000 --- a/.env +++ /dev/null @@ -1,10 +0,0 @@ -TAXONOMY_NAME=dspipelines -PROFILER_NAME=dspipelines - -LANGFUSE_SECRET_KEY= -LANGFUSE_PUBLIC_KEY= -LANGFUSE_HOST="http://langfuse-web:3000" - -LLM_MODEL_ID=Qwen/Qwen3-8B-AWQ -LLM_REASONING_PARSER=qwen3 -LLM_QUANTIZATION=awq \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..00ddec0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,51 @@ +name: CI + +on: + push: + branches: ["**"] + pull_request: + branches: ["**"] + +jobs: + lint-and-type-check: + name: Pre-commit hooks (lint, sort, type-check, docker) + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies + run: uv sync --all-extras + + - name: Run pre-commit on all files + uses: pre-commit/action@v3.0.1 + with: + extra_args: --all-files # TODO ymu : maybe run only on changed files + + test: + name: Tests + runs-on: ubuntu-latest + needs: lint-and-type-check + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies + run: uv sync --all-extras diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9b084d4 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,50 @@ +name: Release + +on: + push: + tags: ["v*"] + +env: + REGISTRY: ghcr.io + IMAGE_PREFIX: ghcr.io/${{ github.repository_owner }} + +jobs: + build-and-push: + name: Build and push image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate tags + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_PREFIX }}/mlprofiler + tags: | + type=raw,value=latest + type=sha,prefix=,format=short + type=semver,pattern={{version}} + + - name: Build and push mlprofiler + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile.prod + push: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index ab9c4f1..a291228 100644 --- a/.gitignore +++ b/.gitignore @@ -182,9 +182,9 @@ cython_debug/ .abstra/ # Visual Studio Code -# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore -# and can be added to the global gitignore or merged into this file. However, if you prefer, +# and can be added to the global gitignore or merged into this file. However, if you prefer, # you could uncomment the following to ignore the entire vscode folder # .vscode/ @@ -209,3 +209,5 @@ __marimo__/ # Others out/ data/ +.idea/ +docker/.env diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..dd29232 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.4 + hooks: + - id: ruff + args: [--fix, --select, "I"] + #- id: ruff-format + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.10.0 + hooks: + - id: mypy + additional_dependencies: + - fastapi + - pydantic + - types-ujson + +- repo: https://github.com/hadolint/hadolint + rev: v2.12.0 + hooks: + - id: hadolint-docker + args: [--failure-threshold, warning] + +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-toml + - id: check-merge-conflict + - id: check-added-large-files + args: [--maxkb=1000] + - id: debug-statements diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/CHANGELOG.md b/CHANGELOG.md index b06c798..7257c28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Parser metadata in JSON ML profile. +- "dspipelines" parser +- External taxonomies support +- Distinction between context and code to classify +- Removed the CLI + ## [0.3] - 2025-07-16 ### Added @@ -19,7 +27,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Taxonomy metadata in JSON ML profile. -- "dspipelines" taxonomy (from *The Art and Practice of Data Science Pipelines*, 2022 ICSE) with some improvements (Library Loading, Others) +- "dspipelines" taxonomy (from *The Art and Practice of Data Science Pipelines*, 2022 ICSE) with some improvements ( + Library Loading, Others) ## [0.1] - 2025-06-01 diff --git a/Dockerfile.prod b/Dockerfile.prod deleted file mode 100644 index 6f55498..0000000 --- a/Dockerfile.prod +++ /dev/null @@ -1,30 +0,0 @@ -FROM ghcr.io/astral-sh/uv:python3.12-alpine - -# TODO: split builder/runner to reduce image size - -LABEL version="0.1" -LABEL description="This is the image used to build the LLM API." - -WORKDIR /mlprofiler-builder - -COPY pyproject.toml uv.lock ./ - -# Install dependencies -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,source=uv.lock,target=uv.lock \ - --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - uv sync --locked --no-install-project - -RUN adduser -D standarduser - -RUN chown -R standarduser:standarduser /mlprofiler-builder - -USER standarduser - -COPY ./templates/ /mlprofiler-builder/templates/ - -COPY ./resources/ /mlprofiler-builder/resources/ - -COPY ./app/ /mlprofiler-builder/app/ - -CMD [".venv/bin/fastapi", "run", "app/main.py", "--host", "0.0.0.0", "--port", "8081"] diff --git a/README.md b/README.md index 17a9df3..4ace21c 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,98 @@ -# mlprofiler - The ML pipelines profiler +# ML Profiler + +Profiles ML pipelines by analyzing source code and classifying pipeline steps. ## Requirements -* Docker (tested on version 28.5.1, build e180ab8) -* Docker Compose (tested on version v2.40.0) +- Docker ≥ 28.5.1 +- Docker Compose ≥ 2.40.0 +- _(Advanced)_ [Docker Model Runner](https://docs.docker.com/ai/model-runner/get-started/) ≥ 1.1.37 required for the LLM profiler function + +## Quick start + +```bash +cd docker +cp .env.sample .env # edit .env if needed, the defaults work for basic setup +docker compose --env-file .env up --build +``` + +## Advanced setup (LLM profiler) + +The LLM profiler function requires Docker Model Runner (DMR). Setup differs by platform. -## Deployment +### Mac DMR -The following docker compose commands deploy the mlprofiler API and vLLM instance: +No extra installation needed. DMR uses llama.cpp natively via Metal (Apple Silicon). ```bash -$ docker compose --env-file .env build -$ docker compose --env-file .env up +docker compose -f docker-compose.yml -f docker-compose.mac.yml --env-file .env up --build ``` -It is also possible to deploy the LLM monitoring platform (Langfuse) as follows: +### Linux DMR + +[Set up the vLLM inference backend](https://docs.docker.com/ai/model-runner/inference-engines/#setting-up-vllm): ```bash -$ docker compose --file docker-compose-monitor-local.yml +docker model install-runner --backend vllm --gpu cuda ``` -## Troubleshooting +[Configure the model](https://docs.docker.com/ai/model-runner/inference-engines/#vllm-configuration): -### Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock +```bash +docker model configure --hf_overrides '{ + "max_model_len": 8192, + "max_num_seqs": 10, + "gpu_memory_utilization": 0.8, + "enforce_eager": true +}' hf.co/Qwen/Qwen2.5-Coder-7B-Instruct-AWQ +``` -Allow your user to use the socket: ```bash -sudo usermod -a -G docker $USER -newgrp docker -reboot +docker compose -f docker-compose.yml -f docker-compose.linux.dmr.yml --env-file .env up --build ``` -### Failed to solve: process "/bin/sh -c uv sync --locked --no-install-project" did not complete successfully: exit code: 1 +### Linux vLLM (standalone) + +Set `VLLM_FORCE=1` in `.env`, then: -Regenerate the lock file by executing: ```bash -uv lock +docker compose -f docker-compose.yml -f docker-compose.linux.dmr.yml -f docker-compose.linux.vllm.yml --env-file .env up --build +``` + +## Monitoring (optional) + +| Stack | Command | +|---|---| +| Langfuse ([docs](https://langfuse.com/docs)) | `docker compose -f docker-compose-monitor-langfuse.yml up` | +| Grafana/Prometheus ([docs](https://github.com/vllm-project/vllm/tree/main/examples/online_serving/prometheus_grafana)) | `docker compose -f docker-compose-monitor-grafana.yml up` | + +## Development + +Install pre-commit hooks: + +```bash +uv run --with pre-commit pre-commit install +``` + +## Troubleshooting + +**`permission denied` connecting to Docker socket** +```bash +sudo usermod -a -G docker $USER && newgrp docker ``` +Then reboot. -### Network colombus-dev_network declared as external, but could not be found +--- -Create the missing network: +**`uv sync` fails during build** + +Regenerate the lockfile: ```bash -docker network create "colombus-dev_network" +uv lock ``` -### Error response from daemon: could not select device driver "nvidia" with capabilities: [[gpu]] +--- + +**`could not select device driver "nvidia"`** -Install the nvidia toolkit and restart docker using this link: -https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html +Install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html#with-apt-ubuntu-debian) and [configure Docker to use the NVIDIA runtime](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html#configuring-docker). diff --git a/app/cli.py b/app/cli.py deleted file mode 100644 index dd68517..0000000 --- a/app/cli.py +++ /dev/null @@ -1,232 +0,0 @@ -import httpx -import json -import os - -import typer - -from pathlib import Path -from tqdm import tqdm -from typing import Annotated, Any, Generator, Literal, cast - -from custom_types import ( - ParserSubgraph, - SupportedProfilerFunction, - SupportedTaxonomiesFunction, -) - -# TODO: fix this import to avoid duplicating the ApiTokenHttpxAuth declaration -# and hardcoded API Token - -# sys.path.append("..") - -# from common.auth import ApiTokenHttpxAuth - -TAXONOMY_NAME: SupportedTaxonomiesFunction = cast( - SupportedTaxonomiesFunction, os.getenv("TAXONOMY_NAME", "headergen") -) -PROFILER_NAME: SupportedProfilerFunction = cast( - SupportedProfilerFunction, os.getenv("PROFILER_NAME", "llm") -) - - -class ApiTokenHttpxAuth(httpx.Auth): - """API Token Authentifier for httpx client.""" - - def auth_flow(self, request: httpx.Request) -> Generator[httpx.Request, Any, None]: - """Update the request headers to add the X-API-Token required by - the API. - - Parameters - ---------- - request : httpx.Request - the request to update the headers - - Yields - ------ - Iterator[httpx.Request] - the updated request - """ - request.headers["X-API-Token"] = "profil-platform-token" - yield request - - -app = typer.Typer() - - -# Out directory - -out_dir = Path("./out") -out_dir.mkdir(exist_ok=True, parents=True) - - -def save_result( - json_profile: dict, original_file: Path, profile_out_dir: Path | None -) -> Path: - """Save the json profile result to the specified output directory. - - Parameters - ---------- - json_profile : dict - the JSON profile to save - original_file : Path - the original file to retrieve the name (to store the profile with a similar name) - profile_out_dir : Path | None - the output directory. If None, then out_dir will be used - """ - current_out_dir = (profile_out_dir or out_dir) / original_file.parent - current_out_dir.mkdir(exist_ok=True, parents=True) - result_profile_path = current_out_dir / original_file.with_suffix(".json").name - with open(result_profile_path, "w") as f: - json.dump(json_profile, f) - return result_profile_path - - -def _cell_source_as_list(source: list[str] | str) -> list[str]: - """Return the source cell content as a list. - - Parameters - ---------- - source : list[str] | str - the source cell content - - Returns - ------- - list[str] - the source cell content - """ - return source if isinstance(source, list) else [source] - - -def read_notebook_content(notebook_path: Path) -> str: - with open(notebook_path) as f: - notebook_content = json.load(f) - - all_python_code: list[str] = [ - block - for cell in notebook_content["cells"] - if cell["cell_type"] == "code" - for block in _cell_source_as_list(cell["source"]) - ] - return "\n".join(all_python_code) - - -def read_famix_subgraphes(famix_subgraphs_path: Path) -> list[ParserSubgraph]: - with open(famix_subgraphs_path) as f: - raw_famix_subgraphs_content = sorted( - json.load(f)["elements"], key=lambda sg: sg["line_start"] - ) - return [ - ParserSubgraph( - id=e["id"], - library=e["library"], - function=e["function"], - value=e["value"], - source=e["source"], - step_name=e["step_name"], - ) - for e in raw_famix_subgraphs_content - ] - - -@app.command() -def profile_notebooks( - notebook_file_or_directory: Path, - famix_subgraphs_file_or_directory: Path, - data_type: Annotated[str, typer.Argument()] = "ipynb", # Literal["ipynb", "py"] - output_directory: Annotated[Path, typer.Argument()] = None, - verbose_mode: Annotated[bool, typer.Argument()] = False, -): - """Profile the given notebook(s) (file or directory). - When providing a directory, the files contained in the famix subgraphs directory - should match *_subgraph.json to be used. - - Parameters - ---------- - notebook_file_or_directory : Path - the notebook file or directory to profile - famix_subgraphs_file_or_directory : Path - the subgraphs file or directory to use for profiling - output_directory : Annotated[Path, typer.Argument], optional - the optional output directory to save the profile, by default None - verbose_mode : Annotated[bool, typer.Argument], optional - should log additional information or not, by default False - """ - log = print if verbose_mode else lambda *x: None - - if notebook_file_or_directory.is_dir(): - errors = {} - all_notebooks = list(notebook_file_or_directory.rglob(f"*.{data_type}")) - all_subgraphs = list(famix_subgraphs_file_or_directory.rglob("*_subgraph.json")) - all_save_paths: list[Path] = [] - for notebook_file, subgraph_file in ( - pbar := tqdm(zip(all_notebooks, all_subgraphs)) - ): - pbar.set_description( - f"Generating profile for {str(notebook_file)[:50].ljust(50)}" - ) - log("notebook_file=", notebook_file) - try: - with httpx.Client(auth=ApiTokenHttpxAuth()) as client: - posted_notebook_response = client.post( - "http://localhost:8081/profile", - json={ - "notebook_file_stem": notebook_file.stem, - "python_content": ( - read_notebook_content(notebook_file) - if data_type == "ipynb" - else notebook_file.read_text() - ), - "parser_elements": [ - s.model_dump() - for s in read_famix_subgraphes(subgraph_file) - ], - "taxonomy_name": TAXONOMY_NAME, - "profiler_name": PROFILER_NAME, - }, - timeout=None, - ) - posted_notebook_response.raise_for_status() - all_save_paths.append( - save_result( - posted_notebook_response.json(), notebook_file, output_directory - ) - ) - except Exception as e: - errors[notebook_file.name] = str(e) - for save_path in all_save_paths: - print(f"Saved: {save_path}") - if errors: - print("ERRORS: ", errors) - else: - with httpx.Client(auth=ApiTokenHttpxAuth()) as client: - posted_notebook_response = client.post( - f"http://localhost:8081/profile", - json={ - "notebook_file_stem": notebook_file_or_directory.stem, - "python_content": ( - read_notebook_content(notebook_file_or_directory) - if data_type == "ipynb" - else notebook_file_or_directory.read_text() - ), - "parser_elements": [ - s.model_dump() - for s in read_famix_subgraphes( - famix_subgraphs_file_or_directory - ) - ], - "taxonomy_name": TAXONOMY_NAME, - "profiler_name": PROFILER_NAME, - }, - timeout=None, - ) - posted_notebook_response.raise_for_status() - save_path = save_result( - posted_notebook_response.json(), - notebook_file_or_directory, - output_directory, - ) - print(f"Saved: {save_path}") - - -if __name__ == "__main__": - app() diff --git a/app/constants.py b/app/constants.py new file mode 100644 index 0000000..2e4949b --- /dev/null +++ b/app/constants.py @@ -0,0 +1,13 @@ +import os + +APP_VERSION = "0.3.0" + +# TODO ymu: Replace with Pydantic config +VLLM_FORCE = int(os.environ["VLLM_FORCE"]) +if VLLM_FORCE: + INFERENCE_API_URL_PREFIX = os.environ["vLLM_INFERENCE_API_URL_PREFIX"] + print(f'found VLLM_FORCE={VLLM_FORCE} using VLLM url={INFERENCE_API_URL_PREFIX}') +else: + INFERENCE_API_URL_PREFIX = os.environ["LLM_INFERENCE_API_URL"] + print(f'found VLLM_FORCE={VLLM_FORCE} using DMR url={INFERENCE_API_URL_PREFIX}') +PARSER_API_URL_PREFIX = os.environ["VESPUCCI_PARSER_API_URL_PREFIX"] diff --git a/app/custom_types.py b/app/custom_types.py deleted file mode 100644 index bebe4fc..0000000 --- a/app/custom_types.py +++ /dev/null @@ -1,19 +0,0 @@ -from typing import Any, Literal - -from pydantic import BaseModel, TypeAdapter - - -class ParserSubgraph(BaseModel): - id: str - library: str - function: str - value: dict[str, Any] - source: str - step_name: str - - -ParserSubgraphListAdapter = TypeAdapter(list[ParserSubgraph]) - - -SupportedTaxonomiesFunction = Literal["headergen", "dspipelines", "daswow"] -SupportedProfilerFunction = Literal["llm", "dspipelines", "headergen"] diff --git a/app/main.py b/app/main.py index 8aed54b..ab59ceb 100644 --- a/app/main.py +++ b/app/main.py @@ -1,120 +1,12 @@ -import datetime - -from typing import Any - from fastapi import FastAPI -from pydantic import BaseModel - -from app.custom_types import ( - ParserSubgraph, - SupportedProfilerFunction, - SupportedTaxonomiesFunction, -) -from app.profiling_functions._factory import get_profiler - -app = FastAPI() - -# TODO: adapt core_api to /profile API changes - -APP_VERSION = "0.3.0-MLProfile" - - -class MLProfileMetadata(BaseModel): - version: str - generation_date: datetime.datetime - taxonomy: SupportedTaxonomiesFunction - profiler: SupportedProfilerFunction - - -class MLProfileResult(BaseModel): - name: str - metadata: MLProfileMetadata - source: list[Any] # TODO: fix any - outputs: dict[str, Any] - - -class ProfileNotebookParams(BaseModel): - notebook_file_stem: str - python_content: str - parser_elements: list[ParserSubgraph] - taxonomy_name: SupportedTaxonomiesFunction - profiler_name: SupportedProfilerFunction - - -@app.post("/profile") -def profile_notebook(params: ProfileNotebookParams) -> MLProfileResult: - """Compute the ML profile for the given notebook. - - Parameters - ---------- - params : ProfileNotebookParams - the notebook file step (e.g. abc/myfile.ipynb -> myfile), - corresponding python code content and Moose parsing result - - Returns - ------- - list[Any] - the LLM profiling result - """ - profile_json = MLProfileResult( - name=params.notebook_file_stem, - metadata=MLProfileMetadata( - version=APP_VERSION, - generation_date=datetime.datetime.now(), - taxonomy=params.taxonomy_name, - profiler=params.profiler_name, - ), - source=[], - outputs={}, - ) - prev_step = None - - profiler = get_profiler( - profile_json.metadata.profiler, - params.python_content, - profile_json.metadata.taxonomy, - ) - - for subgraph in params.parser_elements: - line = subgraph.source - res = { - "id": subgraph.id, - "algoFamily": None, - # "algoFamily": algorithms_classified_line["algorithm_family"], - "algoName": None, - # "algoName": algorithms_classified_line["algorithm_name"], - "library": subgraph.library, - "function": subgraph.function, - "tasks": [{"name": line, "tasks": []}], - "metadata": {} - } +from app.constants import APP_VERSION +from app.routers import profile_router - if subgraph.step_name == "Library Loading": - current_step = "Library Loading" - perplexity = 1 - logprobs = 100 - else: - current_step, perplexity, logprobs = profiler.profile_subgraph(subgraph, "Others") - if ( - current_step == "Library Loading" - and subgraph.step_name != "Library Loading" - ): - current_step = "Others" - - res["metadata"]["perplexity"] = perplexity - res["metadata"]["logprobs"] = logprobs +app = FastAPI(version=APP_VERSION) +app.include_router(profile_router.router, prefix="/v2/profile") - if prev_step == current_step: - profile_json.source[-1]["tasks"].append(res) - else: - profile_json.source.append( - { - "name": current_step, - "tasks": [res], - "outputs_ids": [], - } - ) - prev_step = current_step - return profile_json +@app.get("/health") +async def health() -> dict[str, str]: + return {"status": "ok"} diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/models/parser.py b/app/models/parser.py new file mode 100644 index 0000000..b448446 --- /dev/null +++ b/app/models/parser.py @@ -0,0 +1,31 @@ +from enum import Enum + +from pydantic import BaseModel, TypeAdapter + + +class ParserFunction(str, Enum): + DSPIPELINES = "dspipelines" + VESPUCCI = "vespucci" + + +class ParserSubgraphLine(BaseModel): + start: int + end: int + + +class ParserSubgraphCursor(BaseModel): + start: int + end: int + + +class ParserSubgraph(BaseModel): + id: str + library: str + function: str + source: str + step_name: str + line: ParserSubgraphLine + cursor: ParserSubgraphCursor + + +ParserSubgraphList = TypeAdapter(list[ParserSubgraph]) diff --git a/app/models/profiler.py b/app/models/profiler.py new file mode 100644 index 0000000..8366179 --- /dev/null +++ b/app/models/profiler.py @@ -0,0 +1,15 @@ +from enum import Enum + +from pydantic import BaseModel + + +class ProfilerFunction(str, Enum): + LLM = "llm" + DSPIPELINES = "dspipelines" + HEADERGEN = "headergen" + EMBEDDING = "embedding" + + +class ProfileResult(BaseModel): + step: str + perplexity: float diff --git a/app/models/taxonomy.py b/app/models/taxonomy.py new file mode 100644 index 0000000..e7e5d74 --- /dev/null +++ b/app/models/taxonomy.py @@ -0,0 +1,134 @@ +from enum import Enum + +from pydantic import BaseModel + + +class TaxonomyFunction(str, Enum): + DSPIPELINES = "dspipelines" + DASWOW = "daswow" + HEADERGEN = "headergen" + + +class TaxonomyElement(BaseModel): + name: str + definition: str + + +class Taxonomy(BaseModel): + name: str + elements: list[TaxonomyElement] + default_step: str + + def get_steps_names(self) -> list[str]: + return [e.name for e in self.elements] + + +DEFAULT_STEP_NAME = "Other" + +TAXONOMY_BY_NAME = { + TaxonomyFunction.DSPIPELINES.value: Taxonomy( + name=TaxonomyFunction.DSPIPELINES.value, + elements=[ + TaxonomyElement( + name="Data Acquisition", + definition="this code reads/loads new data", + ), + TaxonomyElement( + name="Data Preparation", + definition="the code contributes to the preparation of data so that it is suitable for further processing and analysis", + ), + TaxonomyElement( + name="Modeling", + definition="the code instantiates or builds a model", + ), + TaxonomyElement( + name="Training", + definition="the code trains a model", + ), + TaxonomyElement( + name="Evaluation", + definition="the code evaluates a model", + ), + TaxonomyElement( + name="Prediction", + definition="the code makes inference using a model", + ), + TaxonomyElement( + name=DEFAULT_STEP_NAME, + definition="", + ), + ], + default_step=DEFAULT_STEP_NAME, + ), + TaxonomyFunction.DASWOW.value: Taxonomy( + name=TaxonomyFunction.DASWOW.name, + elements=[ + TaxonomyElement( + name="helper_functions", + definition="Code that is not directly related to the data science activity at hand, but provides useful scripting functions (e. g. importing or configuring libraries).", + ), + TaxonomyElement( + name="load_data", + definition="The process of loading a dataset of any type (e.g., .csv, .pkl) into a Jupyter notebook environment.", + ), + TaxonomyElement( + name="data_preprocessing", + definition="The process of preparing the dataset(s) for the subsequent analysis. It includes tasks such as cleaning, instance selection, normalisation, data transformation, and feature selection.", + ), + TaxonomyElement( + name="data_exploration", + definition="The process of inspecting the content and shape of a dataset to understand the nature and characteristics of the data. Note that it may involve the usage of visualisation techniques but differs in its purpose.", + ), + TaxonomyElement( + name="modelling", + definition="The process of applying statistical models and learning-based algorithms to learn from sample data.", + ), + TaxonomyElement( + name="evaluation", + definition="The process of assessing a model using one/various evaluation metric(s) such as goodness of fit and accuracy.", + ), + TaxonomyElement( + name="model_inference", + definition="The process of applying a model trained on a set of data to other or newly arriving pieces of data to forecast new values.", + ), + TaxonomyElement( + name="result_visualization", + definition="The process of obtaining a graphical representation (e.g., tables, plots, graphs) of a/several measurement(s).", + ), + TaxonomyElement( + name="save_results", + definition="The process of serialising and storing the data.", + ), + TaxonomyElement( + name="comment_only", + definition="Lines of comment including commented code.", + ), + TaxonomyElement( + name=DEFAULT_STEP_NAME, + definition="", + ), + ], + default_step=DEFAULT_STEP_NAME, + ), + TaxonomyFunction.HEADERGEN.value: Taxonomy( + name=TaxonomyFunction.HEADERGEN.name, + elements=[ + TaxonomyElement(name="Library Loading", definition=""), + TaxonomyElement(name="Visualization", definition=""), + TaxonomyElement(name="Data Loading", definition=""), + TaxonomyElement(name="Exploratory Data Analysis", definition=""), + TaxonomyElement(name="Data Preparation", definition=""), + TaxonomyElement( + name="Data Sub-sampling and Train-test Splitting", definition="" + ), + TaxonomyElement(name="Feature Transformation", definition=""), + TaxonomyElement(name="Feature Selection", definition=""), + TaxonomyElement(name="Model Assembling", definition=""), + TaxonomyElement(name="Model Parameter Tuning", definition=""), + TaxonomyElement(name="Model Training", definition=""), + TaxonomyElement(name="Model Validation", definition=""), + TaxonomyElement(name=DEFAULT_STEP_NAME, definition=""), + ], + default_step=DEFAULT_STEP_NAME, + ), +} diff --git a/app/parsers/__init__.py b/app/parsers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/parsers/base.py b/app/parsers/base.py new file mode 100644 index 0000000..6ad9b2f --- /dev/null +++ b/app/parsers/base.py @@ -0,0 +1,22 @@ +from abc import ABC, abstractmethod + +from app.models.parser import ParserSubgraph + + +class BaseMLParser(ABC): + def __init__(self) -> None: + super().__init__() + + @abstractmethod + def parse_code( + self, python_code: str, parse_subscript: bool = True + ) -> list[ParserSubgraph]: + """Parse a given python code to extract the instructions to classify. + + Args: + python_code (str): the python code to parse + parse_subscript (bool): whether to parse subscripts instructions or not (e.g., df[...]) + + Returns: + list[ParserSubgraph]: the retrieved subgraphs from the given code + """ diff --git a/app/parsers/dspipelines.py b/app/parsers/dspipelines.py new file mode 100644 index 0000000..7c88b67 --- /dev/null +++ b/app/parsers/dspipelines.py @@ -0,0 +1,318 @@ +import ast +import uuid + +from app.models.parser import ParserSubgraph, ParserSubgraphLine +from app.parsers.base import BaseMLParser + +#### TODO: adapted from https://github.com/sumonbis/DS-Pipeline below + + +class DSPipelinesParser(BaseMLParser): + def __init__(self): + super().__init__() + self.__subgraphs: list[ParserSubgraph] = [] + + def parse_code( + self, python_code: str, parse_subscript: bool = True + ) -> list[ParserSubgraph]: + tree = ast.parse(python_code) + visitor = FuncLister(parse_subscript=parse_subscript) + visitor.s_list = [] + visitor.f_name = [] + visitor.f_dict = {} + visitor.symb_dict = {} + visitor.arg_arr = [] + visitor.symb_arr = [] + visitor.visit(tree) + + edges: list[tuple[int, int]] = [] + pipe: list[str] = [] + + for i in range(len(visitor.s_list)): + ins = list(set(visitor.arg_arr[i])) + for j in ins: + found = False + for k in range(i - 1, -1, -1): + for sym in visitor.symb_arr[k]: + if j == sym: + edges.append((k, i)) + found = True + break + if found: + break + + rec: list[dict] = [] + self.build_pipe(visitor.s_list[i], visitor.f_dict, pipe, rec) + + computed_subgraphs = [*self.__subgraphs] + self.__subgraphs.clear() + return computed_subgraphs + + def build_pipe(self, elem, dict, pipe, rec): + if elem["root"].startswith("9"): + # if custom function call + func_name = elem["root"][1:] + if len(dict[func_name]) > 0: + for ss in dict[func_name]: + if len(rec) < 100: + rec.append(ss) + self.build_pipe(ss, dict, pipe, rec) + rec = [] + elif (len(pipe) == 0) or (pipe[-1] != elem): + pipe.append(elem["root"]) + splitted_libfunc = elem["api"].split(".") + self.__subgraphs.append( + ParserSubgraph( + id=str(uuid.uuid4()), + library="", + function=splitted_libfunc[-1].split(" ")[0], + value={}, + source=ast.unparse(elem["node"]), + line=ParserSubgraphLine( + start=elem["node"].lineno, + end=elem["node"].end_lineno, + ), + step_name="", + ) + ) + + +class FuncLister(ast.NodeVisitor): + trailler = "" + isClass = False + isFunc = 0 + + symb_dict: dict[str, str] = {} + s_list: list[dict] = [] + f_name: list[str] = [] + f_dict: dict[str, list[dict]] = {} + arg_arr: list[list[str] | str] = [] + symb_arr: list[list[str] | str] = [] + symb: list[str] = [] + + def __init__(self, parse_subscript: bool = True) -> None: + super().__init__() + self.parse_subscript = parse_subscript + + def visit_ClassDef(self, node) -> None: + self.generic_visit(node) + + def visit_FunctionDef(self, node) -> None: + self.isFunc += 1 + self.f_name.append(node.name) + self.f_dict[self.f_name[-1]] = [] + self.generic_visit(node) + self.f_name.pop() + self.isFunc -= 1 + + def visit_Assign(self, node) -> None: + self.symb = [] + sym = "" + for target in node.targets: + if isinstance(target, ast.Tuple): + for e in target.elts: + sym = self.getSymbol(e) + self.symb.append(sym) + self.symb_dict[sym] = "-" + else: + sym = self.getSymbol(target) + self.symb.append(sym) + self.symb_dict[sym] = "-" + self.generic_visit(node) + self.symb = [] + + def getSymbol(self, target) -> str: + if isinstance(target, ast.Name): + return target.id + elif isinstance(target, ast.Subscript): + if ( + isinstance(target.value, ast.Name) + and isinstance(target.slice, ast.Index) + and isinstance(target.value, ast.Name) + ): + return target.value.id + "[" + target.value.id + "]" + return "?" + elif isinstance(target, ast.Attribute): + return target.attr + else: + return "?" + + def visit_Call(self, node) -> None: + api = "" + ar = "" + arg = [] + if isinstance(node.func, ast.Name): + try: + arg = FuncLister.get_args(node) + ar = str(arg) + except: # noqa: E722 + ar = "[?]" + api_name = str(node.func.id) + if api_name.endswith("app.run"): + api_name = "main" + api = api_name + " " + ar + + elif isinstance(node.func, ast.Attribute): + n = node.func + FuncLister.trailler = "" + AttrLister().visit(n) + tr = FuncLister.trailler + if tr.endswith("app.run"): + tr = "main" + try: + arg = FuncLister.get_args(node) + ar = str(arg) + except: # noqa: E722 + ar = "[?]" + api = tr + " " + ar + + if api != "": + root = Utils.get_root(api, self.f_dict.keys()) + + # TODO: implement this condition in the profiling function + # if s == "0" or s == "8": + # pass + # else: + if self.isFunc > 0: + self.f_dict[self.f_name[-1]].append( + {"root": root, "node": node, "api": api} + ) + else: + self.s_list.append({"root": root, "node": node, "api": api}) + self.arg_arr.append(arg) + self.symb_arr.append(self.symb) + + self.generic_visit(node) + + def visit_Subscript(self, node): + # ADDED + if self.parse_subscript: + if isinstance(node.value, ast.Name): + self.s_list.append({"root": "", "node": node, "api": "subscript"}) + self.arg_arr.append("") + self.symb_arr.append("") + elif isinstance(node.value, ast.Attribute): + self.s_list.append({"root": "", "node": node, "api": "subscript"}) + self.arg_arr.append("") + self.symb_arr.append("") + self.generic_visit(node) + + @staticmethod + def get_args(node) -> list[str]: + a = [] + b = [] + for arg in node.args: + if isinstance(arg, ast.Starred): + a.append(arg.value.id) # type: ignore + # elif isinstance(arg, ast.Str): + elif isinstance(arg, ast.BinOp): + a.append( + Utils.get_val(arg.left) + + Utils.get_bin_op(arg.op) + + Utils.get_val(arg.right) + ) + else: + a.append(Utils.get_val(arg)) + if isinstance(arg, ast.Name): + b.append(Utils.get_val(arg)) ### + for kw in node.keywords: + if isinstance(kw, ast.keyword): + if kw.arg is None: + a.append("**" + kw.value.id) # type: ignore + else: + a.append(kw.arg + "=" + Utils.get_val(kw.value)) + return b + + +class AttrLister(ast.NodeVisitor): + def visit_Attribute(self, node)-> None: + if isinstance(node.value, ast.Attribute): + if FuncLister.trailler == "": + FuncLister.trailler = node.attr + else: + FuncLister.trailler = node.attr + "." + FuncLister.trailler + if isinstance(node.value, ast.Name): + if FuncLister.trailler == "": + FuncLister.trailler = node.value.id + "." + node.attr + else: + FuncLister.trailler = ( + node.value.id + "." + node.attr + "." + FuncLister.trailler + ) + self.generic_visit(node) + + +class Utils: + @classmethod + def get_val(cls, node) -> str: + if isinstance(node, ast.Num): + return str(node.n) + elif isinstance(node, ast.Str): + return str(node.s) + elif isinstance(node, ast.Name): + return str(node.id) + elif isinstance(node, ast.NameConstant): + return str(node.value) + elif isinstance(node, ast.Call): + return "CALL" + elif isinstance(node, ast.Subscript): + return str( + cls.get_val(node.value) + ) # + handle subcript Slice(Index, Slice or ExtSlice) + elif isinstance(node, ast.Attribute): + FuncLister.trailler = "" + AttrLister().visit(node) + return FuncLister.trailler + elif isinstance(node, ast.List): + return str(cls.get_elts(node)) + elif isinstance(node, ast.Tuple): + return str(cls.get_elts(node)) + else: + return "UNKNOWN" + + @classmethod + def get_elts(cls, node) -> str: + a = [] + for e in node.elts: + a.append(cls.get_val(e)) + return str(a) + + @classmethod + def get_bin_op(cls, node) -> str: + if isinstance(node, ast.Add): + return " + " + elif isinstance(node, ast.Sub): + return " - " + elif isinstance(node, ast.Mult): + return " * " + elif isinstance(node, ast.Div): + return " / " + elif isinstance(node, ast.FloorDiv): + return " // " + elif isinstance(node, ast.Mod): + return " % " + elif isinstance(node, ast.Pow): + return " ** " + elif isinstance(node, ast.LShift): + return " << " + elif isinstance(node, ast.RShift): + return " >> " + elif isinstance(node, ast.BitAnd): + return " B_AND " + elif isinstance(node, ast.BitOr): + return " B_OR " + elif isinstance(node, ast.BitXor): + return " B_XOR " + else: + assert False + + @classmethod + def get_root(cls, api, fs) -> str: + # TODO: add doc + name = api.split(" [")[0] + parts = name.split(".") + root = parts[-1] + + if root in fs: + s = "9" + root + return s + return root diff --git a/app/parsers/factory.py b/app/parsers/factory.py new file mode 100644 index 0000000..9cdcfd5 --- /dev/null +++ b/app/parsers/factory.py @@ -0,0 +1,14 @@ +from app.models.parser import ParserFunction +from app.parsers.base import BaseMLParser +from app.parsers.dspipelines import DSPipelinesParser +from app.parsers.vespucci import VespucciParser + + +def get_parser(parser_name: ParserFunction) -> BaseMLParser: + match parser_name: + case ParserFunction.DSPIPELINES.value: + return DSPipelinesParser() + case ParserFunction.VESPUCCI.value: + return VespucciParser() + case _: + raise ValueError(f"Invalid parser name [{parser_name}].") diff --git a/app/parsers/vespucci.py b/app/parsers/vespucci.py new file mode 100644 index 0000000..f3d2c74 --- /dev/null +++ b/app/parsers/vespucci.py @@ -0,0 +1,26 @@ +import httpx + +from app.constants import PARSER_API_URL_PREFIX +from app.models.parser import ParserSubgraph, ParserSubgraphList +from app.parsers.base import BaseMLParser + +PARSER_API_TIMEOUT = 2 + + +class VespucciParser(BaseMLParser): + def __init__(self): + super().__init__() + + def parse_code( + self, python_code: str, parse_subscript: bool = True + ) -> list[ParserSubgraph]: + parser_response = httpx.post( + f"{PARSER_API_URL_PREFIX}/parse", + json={ + "source": python_code, + }, + timeout=PARSER_API_TIMEOUT, + ) + parser_response.raise_for_status() + parser_response = parser_response.json() + return sorted(ParserSubgraphList.validate_python(parser_response), key=lambda pr: (pr.line.start, pr.cursor.start)) diff --git a/app/profiling_functions/__init__.py b/app/profiling_functions/__init__.py index 2b607ed..e69de29 100644 --- a/app/profiling_functions/__init__.py +++ b/app/profiling_functions/__init__.py @@ -1,3 +0,0 @@ -from app.profiling_functions.llm import * -from app.profiling_functions.dspipelines import * -from app.profiling_functions._factory import * diff --git a/app/profiling_functions/_base.py b/app/profiling_functions/_base.py deleted file mode 100644 index 1f1c882..0000000 --- a/app/profiling_functions/_base.py +++ /dev/null @@ -1,32 +0,0 @@ -from abc import ABC, abstractmethod - -from app.custom_types import ParserSubgraph, SupportedTaxonomiesFunction -from app.profiling_functions._utils import load_taxonomy - - -class BaseMLProfiler(ABC): - - def __init__( - self, python_content: str, taxonomy_name: SupportedTaxonomiesFunction - ) -> None: - super().__init__() - - self.python_content = python_content - self.taxonomy = load_taxonomy(taxonomy_name) - - @abstractmethod - def profile_subgraph( - self, subgraph: ParserSubgraph, default_step: str - ) -> tuple[str, float | None, list[list[tuple[str, float]]]]: - """Profile a given subgraph based on the steps taxonomy. - - Args: - subgraph (ParserSubgraph): the famix subgraph to profile - default_step (str): the default step to use when the profiling - result is out of the taxonomy - - Returns: - tuple[str, float | None, list[list[tuple[str, float]]]]: a tuple containing the step, - overall perplexity and logprobs for each next token. - Default values are used when not using a LLM - """ diff --git a/app/profiling_functions/_factory.py b/app/profiling_functions/_factory.py deleted file mode 100644 index bad3e2e..0000000 --- a/app/profiling_functions/_factory.py +++ /dev/null @@ -1,27 +0,0 @@ -from app.custom_types import SupportedProfilerFunction -from app.profiling_functions._base import BaseMLProfiler -from app.profiling_functions.dspipelines import DSPipelinesProfiler -from app.profiling_functions.headergen import HeaderGenProfiler -from app.profiling_functions.llm import LLMProfiler - - -def get_profiler( - profiler_name: SupportedProfilerFunction, - python_content: str, - taxonomy_name: str, -) -> BaseMLProfiler: - match (profiler_name): - case "llm": - return LLMProfiler( - python_content=python_content, taxonomy_name=taxonomy_name - ) - case "dspipelines": - return DSPipelinesProfiler( - python_content=python_content, taxonomy_name=taxonomy_name - ) - case "headergen": - return HeaderGenProfiler( - python_content=python_content, taxonomy_name=taxonomy_name - ) - case _: - raise ValueError(f"Invalid profiler name [{profiler_name}].") diff --git a/app/profiling_functions/_utils.py b/app/profiling_functions/_utils.py deleted file mode 100644 index c5335d9..0000000 --- a/app/profiling_functions/_utils.py +++ /dev/null @@ -1,38 +0,0 @@ -from pathlib import Path - -from pydantic import BaseModel - -from app.custom_types import SupportedTaxonomiesFunction - - -class TaxonomyElement(BaseModel): - compatible_name: str - original_name: str - definition: str - stage: str - - -class Taxonomy(BaseModel): - name: SupportedTaxonomiesFunction - elements: list[TaxonomyElement] - - def get_original_steps_names(self) -> list[str]: - return [e.original_name for e in self.elements] - - def get_compatible_steps_names(self) -> list[str]: - return [e.compatible_name for e in self.elements] - - def get_original_name_from_compatible( - self, compatible_name: str, default_name: str - ): - for e in self.elements: - if e.compatible_name == compatible_name: - return e.original_name - return default_name - - -def load_taxonomy(taxonomy_name: SupportedTaxonomiesFunction) -> Taxonomy: - raw_taxonomy = Path( - f"resources/taxonomies/{taxonomy_name}_taxonomy.json" - ).read_text() - return Taxonomy.model_validate_json(raw_taxonomy) diff --git a/app/profiling_functions/base.py b/app/profiling_functions/base.py new file mode 100644 index 0000000..c6cd5d6 --- /dev/null +++ b/app/profiling_functions/base.py @@ -0,0 +1,48 @@ +import asyncio +import uuid +from abc import ABC, abstractmethod + +from app.models.parser import ParserSubgraph +from app.models.profiler import ProfileResult +from app.models.taxonomy import Taxonomy + +MAXIMUM_LINE_SIZE = 10 +MAXIMUM_LINE_LENGTH = 79 # Use PEP standard to trim long lines https://peps.python.org/pep-0008/#maximum-line-length + + + +class BaseMLProfiler(ABC): + def __init__(self, source_code: str, taxonomy: Taxonomy): + super().__init__() + self._taxonomy: Taxonomy = taxonomy + self._source_code = source_code + self._source_code_split = self._source_code.split('\n') + self._session_id = f"session-{uuid.uuid4().__str__()}" + + @property + def taxonomy(self): + return self._taxonomy + + @property + def source_code(self): + return self._source_code + + @property + def session_id(self): + return self._session_id + + def compute_source_code_context(self, target_line: int, size: int = MAXIMUM_LINE_SIZE): + from_line = max(0, target_line - size) + to_line = min(len(self._source_code_split), target_line + size) + + context = self._source_code_split[from_line:to_line] + context = [c[:MAXIMUM_LINE_LENGTH] for c in context] + + return '\n'.join(context) + + @abstractmethod + async def profile_subgraph(self, subgraph: ParserSubgraph) -> ProfileResult: + ... + + async def profile_multiple_subgraphs(self, subgraphs: list[ParserSubgraph]) -> list[ProfileResult]: + return await asyncio.gather(*(self.profile_subgraph(subgraph) for subgraph in subgraphs)) diff --git a/app/profiling_functions/dspipelines.py b/app/profiling_functions/dspipelines.py index d684d37..0954156 100644 --- a/app/profiling_functions/dspipelines.py +++ b/app/profiling_functions/dspipelines.py @@ -1,9 +1,9 @@ import json - from pathlib import Path -from app.custom_types import ParserSubgraph, SupportedTaxonomiesFunction -from app.profiling_functions._base import BaseMLProfiler +from app.models.parser import ParserSubgraph +from app.models.profiler import ProfileResult +from app.profiling_functions.base import BaseMLProfiler, Taxonomy steps_functions_mapping_path = Path( "./resources/taxonomies/extra/dspipelines_steps_functions_mapping.json" @@ -14,17 +14,9 @@ class DSPipelinesProfiler(BaseMLProfiler): + def __init__(self, source_code: str, taxonomy: Taxonomy): + super().__init__(source_code, taxonomy) - def __init__(self, python_content: str, taxonomy_name: SupportedTaxonomiesFunction): - super().__init__(python_content, taxonomy_name) - - def profile_subgraph( - self, subgraph: ParserSubgraph, default_step: str - ) -> tuple[str, float | None, list[list[tuple[str, float]]]]: - retrieved_step = steps_functions_mapping.get(subgraph.function, default_step) - verified_retrieved_step = ( - retrieved_step - if retrieved_step in self.taxonomy.get_original_steps_names() - else default_step - ) - return (verified_retrieved_step, 1, [[(verified_retrieved_step, 1.0)]]) + async def profile_subgraph(self, subgraph: ParserSubgraph) -> ProfileResult: + retrieved_step = steps_functions_mapping.get(subgraph.function, self.taxonomy.default_step) + return ProfileResult(step=retrieved_step, perplexity=1) diff --git a/app/profiling_functions/embedding.py b/app/profiling_functions/embedding.py new file mode 100644 index 0000000..872cc0b --- /dev/null +++ b/app/profiling_functions/embedding.py @@ -0,0 +1,82 @@ +import onnxruntime as rt +from sentence_transformers import SentenceTransformer + +from app.models.parser import ParserSubgraph +from app.models.profiler import ProfileResult +from app.profiling_functions.base import BaseMLProfiler, Taxonomy + +LABELS_NAMES = [ + "Data Preparation", + "Data Collection", + "Model Evaluation", + "Data Modeling", + "Model Deployment", + "Save Results", +] + + +class EmbeddingModelSingleton: + __INSTANCE: SentenceTransformer | None = None + __INSTANCE_NAME: str = "" + + @classmethod + def get_instance(cls, instance_name: str) -> SentenceTransformer: + if cls.__INSTANCE and cls.__INSTANCE_NAME == instance_name: + return cls.__INSTANCE + cls.__INSTANCE_NAME = instance_name + cls.__INSTANCE = SentenceTransformer(instance_name) + return cls.__INSTANCE + + +class InferenceSessionSingleton: + __INSTANCE: rt.InferenceSession | None = None + __INSTANCE_NAME: str = "" + + @classmethod + def get_instance(cls, instance_name: str) -> SentenceTransformer: + if cls.__INSTANCE and cls.__INSTANCE_NAME == instance_name: + return cls.__INSTANCE + cls.__INSTANCE_NAME = instance_name + cls.__INSTANCE = rt.InferenceSession( + instance_name, + providers=["CUDAExecutionProvider", "CPUExecutionProvider"], + ) + return cls.__INSTANCE + + +class EmbeddingProfiler(BaseMLProfiler): + def __init__(self, source_code: str, taxonomy: Taxonomy): + super().__init__(source_code, taxonomy) + + self.embedding_model = EmbeddingModelSingleton.get_instance( + "Qwen/Qwen3-Embedding-0.6B" + ) + self.session = InferenceSessionSingleton.get_instance( + "resources/SVC_mlpipelines_classifier_model.onnx", + ) + + async def profile_subgraph(self, subgraph: ParserSubgraph) -> ProfileResult: + embedded_code = self.embedding_model.encode(subgraph.source) + input_name = self.session.get_inputs()[0].name + label_name = self.session.get_outputs()[0].name + # predicted_class_id = self.session.run([label_name], {input_name: embedded_code})[0] + predicted_class_id = self.session.run( + [label_name], + {input_name: embedded_code.reshape((1, embedded_code.shape[0]))}, + )[0][0] + + retrieved_step = LABELS_NAMES[ + predicted_class_id + ] # self.taxonomy.get_steps_names()[predicted_class_id] + return ProfileResult(step=retrieved_step, perplexity=-1) + + async def profile_multiple_subgraphs(self, subgraphs: list[ParserSubgraph]) -> list[ProfileResult]: + embedded_code = self.embedding_model.encode( + [subgraph.source for subgraph in subgraphs] + ) + input_name = self.session.get_inputs()[0].name + label_name = self.session.get_outputs()[0].name + predicted_class_id = self.session.run( + [label_name], {input_name: embedded_code} + )[0] + return [ProfileResult(step=LABELS_NAMES[class_id], perplexity=-1) for class_id in predicted_class_id] diff --git a/app/profiling_functions/factory.py b/app/profiling_functions/factory.py new file mode 100644 index 0000000..08e5933 --- /dev/null +++ b/app/profiling_functions/factory.py @@ -0,0 +1,20 @@ +from app.models.profiler import ProfilerFunction +from app.profiling_functions.base import BaseMLProfiler, Taxonomy +from app.profiling_functions.dspipelines import DSPipelinesProfiler +from app.profiling_functions.embedding import EmbeddingProfiler +from app.profiling_functions.headergen import HeaderGenProfiler +from app.profiling_functions.llm import LLMProfiler + + +def get_profiler(source_code: str, taxonomy: Taxonomy, profiler_name: ProfilerFunction) -> BaseMLProfiler: + match profiler_name: + case "embedding": + return EmbeddingProfiler(source_code=source_code, taxonomy=taxonomy) + case "llm": + return LLMProfiler(source_code=source_code, taxonomy=taxonomy) + case "dspipelines": + return DSPipelinesProfiler(source_code=source_code, taxonomy=taxonomy) + case "headergen": + return HeaderGenProfiler(source_code=source_code, taxonomy=taxonomy) + case _: + raise ValueError(f"Invalid profiler name [{profiler_name}].") diff --git a/app/profiling_functions/headergen.py b/app/profiling_functions/headergen.py index 8eb2125..c2e4713 100644 --- a/app/profiling_functions/headergen.py +++ b/app/profiling_functions/headergen.py @@ -1,35 +1,37 @@ -import httpx - from collections import Counter -from app.custom_types import ParserSubgraph, SupportedTaxonomiesFunction -from app.profiling_functions._base import BaseMLProfiler +import httpx + +from app.models.parser import ParserSubgraph +from app.models.profiler import ProfileResult +from app.profiling_functions.base import BaseMLProfiler, Taxonomy class HeaderGenProfiler(BaseMLProfiler): + def __init__(self, source_code: str, taxonomy: Taxonomy): + super().__init__(source_code, taxonomy) - def __init__(self, python_content: str, taxonomy_name: SupportedTaxonomiesFunction): - super().__init__(python_content, taxonomy_name) - - def profile_subgraph( - self, subgraph: ParserSubgraph, default_step: str - ) -> tuple[str, float | None, list[list[tuple[str, float]]]]: + async def profile_subgraph(self, subgraph: ParserSubgraph) -> ProfileResult: # TODO: add docstring to payload for fair comparison ml_label_response = httpx.post( "http://headergen:54068/get_ml_labels", json={f"{subgraph.library}.{subgraph.function}": {"docstring": ""}}, ) if ml_label_response.is_error: - return (default_step, 1, [[(default_step, 1.0)]]) - retrieved_steps = ml_label_response.json() - compatible_steps = [ - step - for step in retrieved_steps - if step in self.taxonomy.get_original_steps_names() - ] - if not compatible_steps: - return (default_step, 1, [[(default_step, 1.0)]]) - compatible_steps_counter = Counter(compatible_steps) + return ProfileResult( + step=self.taxonomy.default_step, + perplexity=1 + ) + retrieved_steps: list[str] = ml_label_response.json() + if not retrieved_steps: + return ProfileResult( + step=self.taxonomy.default_step, + perplexity=1 + ) + retrieved_steps_counter = Counter(retrieved_steps) # TODO: currently only supporting the first step - retrieved_step = compatible_steps_counter.most_common(1)[0][0] - return (retrieved_step, 1, [[(retrieved_step, 1.0)]]) + retrieved_step = retrieved_steps_counter.most_common(1)[0][0] + return ProfileResult( + step=retrieved_step, + perplexity=1 + ) diff --git a/app/profiling_functions/llm.py b/app/profiling_functions/llm.py index 9e95a4e..305a631 100644 --- a/app/profiling_functions/llm.py +++ b/app/profiling_functions/llm.py @@ -1,113 +1,144 @@ -# Templating configuration - import json -import os +from typing import Any import numpy as np - from jinja2 import Environment, FileSystemLoader, select_autoescape try: # trying monitored by default - from langfuse.openai import OpenAI -except: - from openai import OpenAI - from openai.types.chat import ChatCompletion + from langfuse import get_client, propagate_attributes + from langfuse.openai import AsyncOpenAI + + langfuse = get_client() +except ValueError: + from contextlib import contextmanager + + from openai import AsyncOpenAI + -from app.custom_types import ParserSubgraph, SupportedTaxonomiesFunction -from app.profiling_functions._base import BaseMLProfiler + def propagate_attributes(*args, **kwargs): + ... -INFERENCE_API_URL = os.getenv("INFERENCE_API_URL", "mlprofiler_vllm:11434") -MODEL_ID = os.getenv("MODEL_ID", "qwen2.5-coder:7b") + class LangfuseMock: + @contextmanager + def start_as_current_observation(self, *args, **kwargs): + ... + + + langfuse = LangfuseMock() + +from openai.types.chat import ChatCompletion + +from app.constants import INFERENCE_API_URL_PREFIX +from app.models.parser import ParserSubgraph +from app.models.profiler import ProfileResult +from app.profiling_functions.base import BaseMLProfiler, Taxonomy env = Environment( loader=FileSystemLoader("./templates"), autoescape=select_autoescape() ) -class LLMProfiler(BaseMLProfiler): +class InferenceClientSingleton: + _INSTANCE_BY_NAME: dict[str, dict[str, Any]] = {} - def __init__(self, python_content: str, taxonomy_name: SupportedTaxonomiesFunction): - super().__init__(python_content, taxonomy_name) + @classmethod + async def get_instance(cls, name: str) -> tuple[AsyncOpenAI, str]: + instance = cls._INSTANCE_BY_NAME.get(name) + if instance is None: + client = AsyncOpenAI(base_url=name, api_key="inference-key") + cls._INSTANCE_BY_NAME[name] = { + 'client': client, + 'model_id': (await client.models.list()).data[0].id, + } + instance = cls._INSTANCE_BY_NAME[name] + return instance['client'], instance['model_id'] + + +class LLMProfiler(BaseMLProfiler): + def __init__(self, source_code: str, taxonomy: Taxonomy): + super().__init__(source_code, taxonomy) # loading the LLM system and user prompt templates - system_prompt_template = env.get_template("system_prompt.jinja") - self.user_prompt_template = env.get_template( - f"user_prompt_{taxonomy_name}_taxonomy.jinja" - ) + self._system_prompt_template = env.get_template("system_prompt.jinja") + self._user_prompt_template = env.get_template("user_prompt_taxonomy.jinja") # loading the LLM classification response schema classification_response_schema_template = env.get_template( "classification_response_schema.jinja" ) - self.classification_response_schema = ( + self.__classification_response_schema = ( classification_response_schema_template.render( - compatible_step_names=self.taxonomy.get_compatible_steps_names() + compatible_step_names=self.taxonomy.get_steps_names() ) ) - self.system_prompt_content = system_prompt_template.render( - all_python_code=python_content - ) - # LLM client - - self.client: OpenAI = OpenAI( - base_url=f"http://{INFERENCE_API_URL}/v1", api_key="inference-key" + async def profile_subgraph(self, subgraph: ParserSubgraph) -> ProfileResult: + context_source_code = self.compute_source_code_context(target_line=subgraph.line.start) + system_prompt_content = self._system_prompt_template.render( + all_python_code=context_source_code ) - - def profile_subgraph( - self, subgraph: ParserSubgraph, default_step: str - ) -> tuple[str, float | None, list[list[tuple[str, float]]]]: - user_prompt_content = self.user_prompt_template.render( + user_prompt_content = self._user_prompt_template.render( taxonomy=self.taxonomy, python_code_line=subgraph.source, - subgraph_library=subgraph.library, - subgraph_function=subgraph.function, + expected_class=None, + is_multi_class=False ) - - completion: ChatCompletion = self.client.chat.completions.create( - model=MODEL_ID, - messages=[ - { - "role": "system", - "content": self.system_prompt_content, - }, - {"role": "user", "content": user_prompt_content}, - ], - response_format=json.loads(self.classification_response_schema), - extra_body={ - # "guided_choice": self.taxonomy.compatible_steps_names, - "chat_template_kwargs": {"enable_thinking": False}, - }, - # see https://cookbook.openai.com/examples/multiclass_classification_for_transactions - temperature=0, - max_tokens=15, - top_p=1, - frequency_penalty=0, - presence_penalty=0, - # see https://cookbook.openai.com/examples/using_logprobs - logprobs=True, - top_logprobs=len(self.taxonomy.get_compatible_steps_names()), - ) - - completion_content = completion.choices[0].message.content + client, model_id = await InferenceClientSingleton.get_instance(INFERENCE_API_URL_PREFIX) + with langfuse.start_as_current_observation(as_type="span", name="OpenAI-generation"): + # Propagate session_id to all observations including OpenAI generation + with propagate_attributes(session_id=self.session_id): + completion: ChatCompletion = await client.chat.completions.create( + model=model_id, + messages=[ + { + "role": "system", + "content": system_prompt_content, + }, + { + "role": "user", + "content": user_prompt_content + }, + ], + response_format=json.loads(self.__classification_response_schema), + extra_body={ + "chat_template_kwargs": {"enable_thinking": False}, + }, + # see https://cookbook.openai.com/examples/multiclass_classification_for_transactions + temperature=0, + max_tokens=15, + top_p=1, + frequency_penalty=0, + presence_penalty=0, + # see https://cookbook.openai.com/examples/using_logprobs + logprobs=True, + top_logprobs=len(self.taxonomy.get_steps_names()), + ) + + completion_content = completion.choices[0].message.content if not completion_content: - return default_step, -1, [] + return ProfileResult(step=self.taxonomy.default_step, perplexity=-1) - classified_line = json.loads(completion_content) - predicted_class = classified_line["class"] + try: + classified_line = json.loads(completion_content) + predicted_class = classified_line["class"] + except (json.JSONDecodeError, TypeError): + print("Bad format response") + return ProfileResult(step=self.taxonomy.default_step, perplexity=-1) # excluding json structured output specific tokens to avoid biasing probs # and perplexity score (as the linear probs of these tokens are most of the # time close to 100) leading to a low perplexity score + """ relevant_top_logprobs = [ logprob_content.top_logprobs for logprob_content in completion.choices[0].logprobs.content if logprob_content.top_logprobs[0].token - and logprob_content.top_logprobs[0].token in predicted_class + and logprob_content.top_logprobs[0].token in predicted_class ] + """ relevant_content_logprobs = [ token.logprob for token in completion.choices[0].logprobs.content @@ -115,6 +146,7 @@ def profile_subgraph( ] # converting all logprobs to linear probabilities + """ all_linear_probs = [ [ linear_prob @@ -130,16 +162,16 @@ def profile_subgraph( for top_logprob in relevant_top_logprobs ] ] + """ # we compute the perplexity_score (excluding the json structured output specific tokens to avoid biases) perplexity_score = np.exp( -np.mean([logprob for logprob in relevant_content_logprobs]) ) - return ( - self.taxonomy.get_original_name_from_compatible( - predicted_class, default_step - ), - perplexity_score, - all_linear_probs, + retrieved_step = ( + predicted_class + if predicted_class in self.taxonomy.get_steps_names() + else self.taxonomy.default_step ) + return ProfileResult(step=retrieved_step, perplexity=perplexity_score) diff --git a/app/routers/profile_router.py b/app/routers/profile_router.py new file mode 100644 index 0000000..c77aceb --- /dev/null +++ b/app/routers/profile_router.py @@ -0,0 +1,97 @@ +import asyncio +import datetime +from typing import Any, Generator + +import ujson +from fastapi import APIRouter, UploadFile + +from app.constants import APP_VERSION +from app.models.parser import ParserFunction +from app.models.profiler import ProfilerFunction +from app.models.taxonomy import TAXONOMY_BY_NAME, TaxonomyFunction +from app.parsers.factory import get_parser +from app.profiling_functions.factory import get_profiler + +router = APIRouter() + + +class NotebookCellIterator: + def __init__(self, content_raw: str): + self._content_parsed = ujson.loads(content_raw) + self._generator = self._iterate_cells() + + def __iter__(self): + self._generator = self._iterate_cells() + return self + + def __next__(self) -> dict: + return next(self._generator) + + def _iterate_cells(self) -> Generator[dict, Any, None]: + # yield only cells with source code, also transform source code to str + for cell in self._content_parsed["cells"]: + if cell["cell_type"] == "code": + if isinstance(cell["source"], list): + cell["source"] = "".join(cell["source"]) + if cell["source"]: + yield cell + + +async def _profile_notebook( + notebook_file: UploadFile, + taxonomy_name: TaxonomyFunction, + profiler_name: ProfilerFunction, + parser_name: ParserFunction = ParserFunction.VESPUCCI, +): + file_content = await notebook_file.read() + source_code = "\n".join( + c["source"] for c in NotebookCellIterator(file_content.decode("utf8")) + ) + + subgraphs = get_parser(parser_name).parse_code(source_code) + + taxonomy = TAXONOMY_BY_NAME[taxonomy_name] + profiler = get_profiler(source_code, taxonomy, profiler_name) + results = await profiler.profile_multiple_subgraphs(subgraphs) + + source: list[dict] = [] + for subgraph, profiler_result in zip(subgraphs, results): + element = { + "id": subgraph.id, + "algoFamily": None, + "algoName": None, + "library": subgraph.library, + "function": subgraph.function, + "tasks": [{"name": subgraph.source, "tasks": []}], + "metadata": {"perplexity": profiler_result.perplexity}, + } + if source and source[-1]["name"] == profiler_result.step: + source[-1]["tasks"].append(element) + else: + step = {"name": profiler_result.step, "tasks": [element], "outputs_ids": []} + source.append(step) + return { + "name": notebook_file.filename, + "metadata": { + "version": APP_VERSION, + "generation_date": datetime.datetime.now().isoformat(), + "session_id": profiler.session_id, + "taxonomy": taxonomy_name, + "profiler": profiler_name, + "parser": parser_name.value, + }, + "source": source, + "outputs": {}, + } + + +@router.post("") +async def profile( + notebook_files: list[UploadFile], + taxonomy: TaxonomyFunction, + profiler: ProfilerFunction, +): + profiles = await asyncio.gather( + *[_profile_notebook(notebook_file, taxonomy, profiler) for notebook_file in notebook_files] + ) + return profiles diff --git a/data/student_a.ipynb b/data/student_a.ipynb deleted file mode 100644 index 03973f7..0000000 --- a/data/student_a.ipynb +++ /dev/null @@ -1,818 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Sentiment analysis with an MLP and BOW representation" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Imports" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2022-10-12T12:53:44.910524Z", - "start_time": "2022-10-12T12:53:44.898708Z" - } - }, - "outputs": [], - "source": [ - "import ssl\n", - "ssl._create_default_https_context = ssl._create_unverified_context" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2022-10-12T12:53:45.512632Z", - "start_time": "2022-10-12T12:53:45.507753Z" - } - }, - "outputs": [], - "source": [ - "import warnings\n", - "warnings.filterwarnings('ignore')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2022-10-12T12:54:03.325657Z", - "start_time": "2022-10-12T12:53:46.023244Z" - }, - "id": "vEgOX76J5RwL" - }, - "outputs": [], - "source": [ - "import os\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "import seaborn as sns\n", - "import pandas as pd\n", - "import time" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2022-10-12T12:54:05.131352Z", - "start_time": "2022-10-12T12:54:03.353326Z" - } - }, - "outputs": [], - "source": [ - "from sklearn import set_config\n", - "set_config(display=\"diagram\")\n", - "\n", - "from sklearn.model_selection import train_test_split\n", - "\n", - "from sklearn.pipeline import Pipeline\n", - "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n", - "from sklearn.preprocessing import OneHotEncoder\n", - "\n", - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn.metrics import classification_report, confusion_matrix, f1_score, accuracy_score\n", - "\n", - "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2022-10-12T12:54:09.055834Z", - "start_time": "2022-10-12T12:54:05.208739Z" - } - }, - "outputs": [], - "source": [ - "import nltk\n", - "from nltk import word_tokenize, sent_tokenize \n", - "from nltk.stem import PorterStemmer\n", - "from nltk import FreqDist" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2022-10-12T12:54:48.447022Z", - "start_time": "2022-10-12T12:54:09.081333Z" - } - }, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "import keras_tuner as kt\n", - "from tensorflow.keras.models import Model\n", - "from tensorflow.keras import layers, callbacks, utils" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Load the dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_vlbVhVpvq_s" - }, - "source": [ - "* **Training Dataset:** The sample of data used to fit the model.\n", - "* **Validation Dataset:** The sample of data used to provide an unbiased evaluation of a model fit on the training dataset while tuning model hyperparameters. The evaluation becomes more biased as skill on the validation dataset is incorporated into the model configuration.\n", - "* **Test Dataset:** The sample of data used to provide an unbiased evaluation of a final model fit on the training dataset." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2022-10-12T12:54:50.078178Z", - "start_time": "2022-10-12T12:54:48.452355Z" - }, - "colab": { - "base_uri": "https://localhost:8080/", - "height": 204 - }, - "executionInfo": { - "elapsed": 2638, - "status": "ok", - "timestamp": 1634734106855, - "user": { - "displayName": "Antoine Collin", - "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64", - "userId": "01107865217976062482" - }, - "user_tz": -120 - }, - "id": "MLWzp9CAvq_t", - "outputId": "abb836f6-8137-43a7-decb-f1d7a8a981be" - }, - "outputs": [], - "source": [ - "TRAIN = pd.read_csv(\"http://www.i3s.unice.fr/~riveill/dataset/Amazon_Unlocked_Mobile/train.csv.gz\")\n", - "VAL = pd.read_csv(\"http://www.i3s.unice.fr/~riveill/dataset/Amazon_Unlocked_Mobile/val.csv.gz\")\n", - "TEST = pd.read_csv(\"http://www.i3s.unice.fr/~riveill/dataset/Amazon_Unlocked_Mobile/test.csv.gz\")\n", - "\n", - "TRAIN.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As I'm going to be doing cross validation, I'm going to merge the train part and the val part." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# I chose to merge TRAIN and VAL\n", - "TRAIN = pd.concat([TRAIN, VAL], axis=0)\n", - "TRAIN.shape, TEST.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# I choose to replace missing review by empty one\n", - "TRAIN = TRAIN.fillna(\"\")\n", - "TEST = TEST.fillna(\"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "o2eC1L4fvq_0" - }, - "source": [ - "## Build X (features vectors) and y (labels)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2022-10-12T12:57:10.769051Z", - "start_time": "2022-10-12T12:57:10.631394Z" - }, - "colab": { - "base_uri": "https://localhost:8080/" - }, - "executionInfo": { - "elapsed": 284, - "status": "ok", - "timestamp": 1634738820951, - "user": { - "displayName": "Antoine Collin", - "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64", - "userId": "01107865217976062482" - }, - "user_tz": -120 - }, - "id": "dOnscXEBvq_0", - "outputId": "4364f457-545c-49d3-c2fe-5540d52b0f58" - }, - "outputs": [], - "source": [ - "# Construct X_train and y_train\n", - "X_train = np.array(TRAIN['Reviews']).reshape(-1,1)\n", - "y_train = np.array(TRAIN['Rating']-np.min(TRAIN['Rating'])).reshape(-1,1)\n", - "X_train.shape, y_train.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2022-10-12T12:57:11.822545Z", - "start_time": "2022-10-12T12:57:11.779828Z" - }, - "colab": { - "base_uri": "https://localhost:8080/" - }, - "executionInfo": { - "elapsed": 6, - "status": "ok", - "timestamp": 1634738821203, - "user": { - "displayName": "Antoine Collin", - "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64", - "userId": "01107865217976062482" - }, - "user_tz": -120 - }, - "id": "338OMkmHvq_1", - "outputId": "aea649ad-4a81-4aa7-976f-4b97c3461644" - }, - "outputs": [], - "source": [ - "# Construct X_test and y_test\n", - "X_test = np.array(TEST['Reviews']).reshape(-1,1)\n", - "y_test = np.array(TEST['Rating']-np.min(TRAIN['Rating'])).reshape(-1,1)\n", - "X_test.shape, y_test.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "assert np.min(y_train)==0\n", - "assert np.min(y_test)==0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2022-10-12T12:57:12.344115Z", - "start_time": "2022-10-12T12:57:12.330335Z" - } - }, - "outputs": [], - "source": [ - "del TRAIN, VAL, TEST" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RR3A2Imxvq_v" - }, - "source": [ - "## Very small EDA (Exploratory Data Analysis)\n", - "\n", - "To choose certain constants (size of vocabulary, length of a line, etc.), it is good to know the dataset used.\n", - "\n", - "You can also add new features to the dataset." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2022-10-12T12:54:57.879689Z", - "start_time": "2022-10-12T12:54:50.184619Z" - }, - "colab": { - "base_uri": "https://localhost:8080/" - }, - "executionInfo": { - "elapsed": 2186, - "status": "ok", - "timestamp": 1634738626074, - "user": { - "displayName": "Antoine Collin", - "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64", - "userId": "01107865217976062482" - }, - "user_tz": -120 - }, - "id": "_nfBsi81vq_w", - "outputId": "190bea69-e01d-498d-da18-7399cc9b125d" - }, - "outputs": [], - "source": [ - "# What is the vocabulary size ?\n", - "\n", - "# Tokenized the reviews\n", - "reviews_tokenized = [word_tokenize(review) for review in X_train.ravel()]\n", - "\n", - "# Count the vocabulary\n", - "flatten_reviews = [item for sublist in reviews_tokenized for item in sublist]\n", - "vocabulary_size = len(set(flatten_reviews))\n", - "vocabulary_size" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize max_length with 90 % of the review not truncated\n", - "from nltk import FreqDist\n", - "\n", - "n = 0.90 # 90 % of reviews are not truncated\n", - "lengths = [len(txt.split()) for txt in X_train.ravel()]\n", - "lengths_fdist = FreqDist(lengths) \n", - "first_25 = FreqDist(dict(lengths_fdist.most_common()[:25]))\n", - "first_25.plot()\n", - "\n", - "cumul = 0\n", - "for length in range(max(lengths)):\n", - " cumul += lengths_fdist[length]\n", - " if cumul>=n*len(X_train):\n", - " break\n", - "\n", - "max_len = length\n", - "max_len # With this length, 90% of reviews are not truncated" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Intialize vocab_size in order to keep words that frequency is more than 1 %% of the sentence\n", - "from nltk import FreqDist\n", - "\n", - "fdist = FreqDist([w for txt in X_train.ravel() for w in txt.split()]) \n", - "first_25 = FreqDist(dict(fdist.most_common()[:25]))\n", - "first_25.plot()\n", - "\n", - "threshold = 0.001 * len(X_train) # We keep all the words that appear in at least 1 %% of the number of documents\n", - "\n", - "for i, (word, nb) in enumerate(fdist.most_common()):\n", - " if nb