> ## Documentation Index
> Fetch the complete documentation index at: https://documentation.datalab.to/llms.txt
> Use this file to discover all available pages before exploring further.

# Create Eval

> Run variants of a pipeline on uploaded files or a collection, and compare their outputs.

Each variant runs the pipeline's steps with its Convert settings merged in. When the
variants of a document have their outputs, the Eval finds where they differ and
decides which variant reads each difference as the page does. Each comparison bills
at the eval rate.



## OpenAPI

````yaml https://www.datalab.to/openapi.json post /api/v1/pipelines/{pipeline_id}/evals
openapi: 3.1.0
info:
  title: Datalab API
  version: 0.0.1
servers:
  - url: https://www.datalab.to
    description: Datalab API
security: []
paths:
  /api/v1/pipelines/{pipeline_id}/evals:
    post:
      tags:
        - evals
      summary: Create Eval
      description: >-
        Run variants of a pipeline on uploaded files or a collection, and
        compare their outputs.


        Each variant runs the pipeline's steps with its Convert settings merged
        in. When the

        variants of a document have their outputs, the Eval finds where they
        differ and

        decides which variant reads each difference as the page does. Each
        comparison bills

        at the eval rate.
      operationId: create_eval_api_v1_pipelines__pipeline_id__evals_post
      parameters:
        - name: pipeline_id
          in: path
          required: true
          schema:
            type: string
            title: Pipeline Id
        - name: wos-session
          in: cookie
          required: false
          schema:
            type: string
            title: Wos-Session
        - name: datalab_active_team
          in: cookie
          required: false
          schema:
            type: string
            title: Datalab Active Team
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateEvalRequest'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvalResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      security:
        - APIKeyHeader: []
components:
  schemas:
    CreateEvalRequest:
      properties:
        collection_id:
          anyOf:
            - type: integer
            - type: 'null'
          title: Collection Id
        file_ids:
          anyOf:
            - items:
                type: integer
              type: array
              maxItems: 100
              minItems: 1
            - type: 'null'
          title: File Ids
        variants:
          anyOf:
            - items:
                $ref: '#/components/schemas/EvalVariant'
              type: array
              maxItems: 4
              minItems: 1
            - type: 'null'
          title: Variants
        version:
          anyOf:
            - type: integer
              minimum: 0
            - type: 'null'
          title: Version
      additionalProperties: false
      type: object
      title: CreateEvalRequest
    EvalResponse:
      properties:
        eval_id:
          type: integer
          title: Eval Id
        pipeline_id:
          type: string
          title: Pipeline Id
        pipeline_name:
          anyOf:
            - type: string
            - type: 'null'
          title: Pipeline Name
        pipeline_version:
          anyOf:
            - type: integer
            - type: 'null'
          title: Pipeline Version
        compared_step:
          type: string
          title: Compared Step
          default: convert
        collection_id:
          anyOf:
            - type: integer
            - type: 'null'
          title: Collection Id
        file_ids:
          anyOf:
            - items:
                type: integer
              type: array
            - type: 'null'
          title: File Ids
        files:
          anyOf:
            - items:
                $ref: '#/components/schemas/EvalFile'
              type: array
            - type: 'null'
          title: Files
        status:
          type: string
          title: Status
        variants:
          items:
            $ref: '#/components/schemas/EvalVariantResult'
          type: array
          title: Variants
        created:
          anyOf:
            - type: string
              format: date-time
            - type: 'null'
          title: Created
      type: object
      required:
        - eval_id
        - pipeline_id
        - pipeline_version
        - collection_id
        - file_ids
        - status
        - variants
        - created
      title: EvalResponse
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    EvalVariant:
      properties:
        name:
          type: string
          maxLength: 100
          minLength: 1
          title: Name
        steps:
          additionalProperties:
            additionalProperties: true
            type: object
          type: object
          title: Steps
      additionalProperties: false
      type: object
      required:
        - name
      title: EvalVariant
    EvalFile:
      properties:
        file_id:
          type: integer
          title: File Id
        file_name:
          type: string
          title: File Name
        comparison:
          type: string
          title: Comparison
          default: pending
        comparison_error:
          anyOf:
            - type: string
            - type: 'null'
          title: Comparison Error
      type: object
      required:
        - file_id
        - file_name
      title: EvalFile
    EvalVariantResult:
      properties:
        name:
          type: string
          title: Name
        steps:
          additionalProperties:
            additionalProperties: true
            type: object
          type: object
          title: Steps
        status:
          type: string
          title: Status
        share:
          anyOf:
            - type: number
            - type: 'null'
          title: Share
        won:
          type: integer
          title: Won
        decided:
          type: integer
          title: Decided
        documents_compared:
          type: integer
          title: Documents Compared
        page_breaks_passed:
          type: integer
          title: Page Breaks Passed
          default: 0
        page_breaks_total:
          type: integer
          title: Page Breaks Total
          default: 0
        documents_total:
          type: integer
          title: Documents Total
        convert_settings:
          additionalProperties: true
          type: object
          title: Convert Settings
        extract_settings:
          additionalProperties: true
          type: object
          title: Extract Settings
        undecided:
          anyOf:
            - type: integer
            - type: 'null'
          title: Undecided
          default: 0
        rate_per_1000_pages_cents:
          anyOf:
            - type: number
            - type: 'null'
          title: Rate Per 1000 Pages Cents
        seconds_per_page:
          anyOf:
            - type: number
            - type: 'null'
          title: Seconds Per Page
        documents:
          anyOf:
            - items:
                $ref: '#/components/schemas/EvalDocumentResult'
              type: array
            - type: 'null'
          title: Documents
      type: object
      required:
        - name
        - steps
        - status
        - share
        - won
        - decided
        - documents_compared
        - documents_total
      title: EvalVariantResult
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
              - type: string
              - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
        ctx:
          type: object
          title: Context
      type: object
      required:
        - loc
        - msg
        - type
      title: ValidationError
    EvalDocumentResult:
      properties:
        file_name:
          type: string
          title: File Name
        file_id:
          anyOf:
            - type: integer
            - type: 'null'
          title: File Id
        status:
          type: string
          title: Status
        lookup_key:
          anyOf:
            - type: string
            - type: 'null'
          title: Lookup Key
        request_id:
          type: integer
          title: Request Id
        share:
          anyOf:
            - type: number
            - type: 'null'
          title: Share
        won:
          type: integer
          title: Won
          default: 0
        decided:
          type: integer
          title: Decided
          default: 0
        undecided:
          anyOf:
            - type: integer
            - type: 'null'
          title: Undecided
          default: 0
      type: object
      required:
        - file_name
        - status
        - request_id
      title: EvalDocumentResult
  securitySchemes:
    APIKeyHeader:
      type: apiKey
      in: header
      name: X-API-Key

````

This documentation is built and hosted on [Mintlify](https://mintlify.com), a developer documentation platform.