Ocena zestawów danych konwersacji przy użyciu zestawu SDK Microsoft Foundry (wersja zapoznawcza)

pip install "azure-ai-projects>=2.2.0"

import os
from openai.types.eval_create_params import DataSourceConfigCustom
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import TestingCriterionAzureAIEvaluator

endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"]
model_deployment_name = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"]

with (
    DefaultAzureCredential() as credential,
    AIProjectClient(endpoint=endpoint, credential=credential) as project_client,
    project_client.get_openai_client() as openai_client,
):
    data_source_config = DataSourceConfigCustom(
        type="custom",
        item_schema={
            "type": "object",
            "properties": {
                "messages": {"type": "array"},
                "tool_definitions": {"type": "array"},
            },
            "required": ["messages"],
        },
        include_sample_schema=False,
    )

    testing_criteria = [
        TestingCriterionAzureAIEvaluator(
            type="azure_ai_evaluator",
            name="conversation_coherence",
            evaluator_name="builtin.coherence",
            initialization_parameters={"model": model_deployment_name},
            data_mapping={"messages": "{{item.messages}}"},
        ),
        TestingCriterionAzureAIEvaluator(
            type="azure_ai_evaluator",
            name="groundedness",
            evaluator_name="builtin.groundedness",
            initialization_parameters={"model": model_deployment_name},
            data_mapping={"messages": "{{item.messages}}"},
        ),
    ]

  object dataSourceConfig = new
  {
    type = "custom",
    item_schema = new
    {
      type = "object",
      properties = new
      {
        messages = new { type = "array" },
        tool_definitions = new { type = "array" }
      },
      required = new[] { "messages" }
    },
    include_sample_schema = false
  };
  object[] testingCriteria =
  [
    new
    {
      type = "azure_ai_evaluator",
      name = "customer_satisfaction",
      evaluator_name = "builtin.customer_satisfaction",
      initialization_parameters = new { model = modelDeploymentName },
      data_mapping = new { messages = "{{item.messages}}" }
    },
    new
    {
      type = "azure_ai_evaluator",
      name = "task_completion",
      evaluator_name = "builtin.task_completion",
      initialization_parameters = new { model = modelDeploymentName },
      data_mapping = new { messages = "{{item.messages}}" }
    },
    new
    {
      type = "azure_ai_evaluator",
      name = "conversation_coherence",
      evaluator_name = "builtin.coherence",
      initialization_parameters = new { model = modelDeploymentName },
      data_mapping = new { messages = "{{item.messages}}" }
    },
    new
    {
      type = "azure_ai_evaluator",
      name = "groundedness",
      evaluator_name = "builtin.groundedness",
      initialization_parameters = new { model = modelDeploymentName },
      data_mapping = new { messages = "{{item.messages}}" }
    }
  ];

Bieżące przykłady zestawu SDK języka JavaScript/TypeScript nie pokazują oceny na poziomie konwersacji. Użyj zakładki Python lub cURL w tym przepływie.

curl --request POST \
  --url "https://${ACCOUNT}.services.ai.azure.com/api/projects/${PROJECT}/openai/evals?api-version=2025-11-15-preview" \
  --header "Authorization: Bearer ${TOKEN}" \
  --header "Content-Type: application/json" \
  --data '{
    "name": "multiturn-conversation-evaluation",
    "data_source_config": {
      "type": "custom",
      "item_schema": {
        "type": "object",
        "properties": {
          "messages": {"type": "array"},
          "tool_definitions": {"type": "array"}
        },
        "required": ["messages"]
      },
      "include_sample_schema": false
    },
    "testing_criteria": [
      {
        "type": "azure_ai_evaluator",
        "name": "conversation_coherence",
        "evaluator_name": "builtin.coherence",
        "initialization_parameters": {"model": "gpt-5-mini"},
        "data_mapping": {"messages": "{{item.messages}}"}
      },
      {
        "type": "azure_ai_evaluator",
        "name": "groundedness",
        "evaluator_name": "builtin.groundedness",
        "initialization_parameters": {"model": "gpt-5-mini"},
        "data_mapping": {"messages": "{{item.messages}}"}
      }
    ]
  }'

Pobierz plik sample_data_multiturn_conversations.jsonl przed uruchomieniem przykładu.

from openai.types.evals.create_eval_jsonl_run_data_source_param import (
    CreateEvalJSONLRunDataSourceParam,
    SourceFileID,
)

# Upload conversation data
data_id = project_client.datasets.upload_file(
    name="multiturn-conversation-data",
    version="1",
    file_path="./sample_data_multiturn_conversations.jsonl",
).id

# Create the evaluation
eval_object = openai_client.evals.create(
    name="Multi-turn Conversation Evaluation",
    data_source_config=data_source_config,
    testing_criteria=testing_criteria,
)

# Create a run with evaluation_level set to "conversation"
eval_run = openai_client.evals.runs.create(
    eval_id=eval_object.id,
    name="multiturn-conversation-run",
    data_source=CreateEvalJSONLRunDataSourceParam(
        type="jsonl",
        source=SourceFileID(
            type="file_id",
            id=data_id,
        ),
    ),
    extra_body={"evaluation_level": "conversation"},
)

Pobierz plik sample_data_multiturn_conversations.jsonl, a następnie uruchom następujący kod:

  FileDataset conversations = await projectClient.Datasets.UploadFileAsync(
    name: "multiturn-conversation-data",
    version: "1",
    filePath: "./sample_data_multiturn_conversations.jsonl");

  BinaryData evaluationData = BinaryData.FromObjectAsJson(new
  {
    name = "Multi-turn Conversation Evaluation",
    data_source_config = dataSourceConfig,
    testing_criteria = testingCriteria
  });
  using BinaryContent evaluationContent = BinaryContent.Create(evaluationData);
  ClientResult evaluation = await evaluationClient.CreateEvaluationAsync(
    evaluationContent);
  string evaluationId = GetString(evaluation, "id");

  BinaryData runData = BinaryData.FromObjectAsJson(new
  {
    name = "multiturn-conversation-run",
    evaluation_level = "conversation",
    data_source = new
    {
      type = "jsonl",
      source = new { type = "file_id", id = conversations.Id }
    }
  });
  using BinaryContent runContent = BinaryContent.Create(runData);
  ClientResult evaluationRun = await evaluationClient.CreateEvaluationRunAsync(
    evaluationId: evaluationId,
    content: runContent);
  Console.WriteLine($"Evaluation run created: {GetString(evaluationRun, "id")}");

Dokumentacja: AIProjectDatasetsOperations.UploadFileAsync i EvaluationClient metody protokołu.

Bieżące przykłady zestawu SDK języka JavaScript/TypeScript nie pokazują oceny na poziomie konwersacji. Użyj zakładki Python lub cURL w tym przepływie.

curl --request POST \
  --url "https://${ACCOUNT}.services.ai.azure.com/api/projects/${PROJECT}/openai/evals/${EVAL_ID}/runs?api-version=2025-11-15-preview" \
  --header "Authorization: Bearer ${TOKEN}" \
  --header "Content-Type: application/json" \
  --data '{
    "name": "multiturn-conversation-run",
    "evaluation_level": "conversation",
    "data_source": {
      "type": "jsonl",
      "source": {
        "type": "file_id",
        "id": "YOUR_DATASET_ID"
      }
    }
  }'