openapi: 3.0.3
info:
  title: "Qualifire AI API"
  description: "API for Qualifire AI services, including evaluation and prompt management."
  version: "1.0.0"
servers:
  - url: https://api.qualifire.ai/api
paths:
  /v1/evaluation/evaluate:
    post:
      tags:
        - Evaluation
      summary: Evaluate a prompt using Qualifire's detectors
      description: Evaluates given input, output, or messages using Qualifire's detectors. Supports checks for hallucinations, grounding, PII, prompt injections, content moderation, policy assertions, topic scoping, and tool use quality.
      requestBody:
        description: Evaluation request
        content:
          application/json:
            schema:
              $ref: "#/components/schemas/models.ApiEvaluationEvaluateRequest"
        required: true
      responses:
        "200":
          description: OK
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/models.EvaluationResponse"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/models.Error"
        "422":
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/models.Error"
  /v1/evaluation/invoke:
    post:
      tags:
        - Evaluation
      summary: Invoke a pre-configured evaluation
      description: Invokes a pre-configured evaluation by its ID. The evaluation configuration (which checks to run, modes, assertions, etc.) is defined in the Qualifire dashboard.
      requestBody:
        description: Evaluation invoke request
        content:
          application/json:
            schema:
              $ref: "#/components/schemas/models.EvaluationInvokeRequest"
        required: true
      responses:
        "200":
          description: OK
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/models.EvaluationResponse"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/models.Error"
        "404":
          description: Evaluation not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/models.Error"
        "422":
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/models.Error"
  /v1/studio/prompts/{promptId}:
    get:
      tags:
        - Studio
      summary: Get a prompt
      description: Retrieves a prompt with its messages and parameters. Supports lookup by prompt ID (cuid) or textId (deprecated).
      parameters:
        - name: promptId
          in: path
          description: Prompt ID (cuid) or textId (deprecated)
          required: true
          schema:
            type: string
        - name: revision
          in: query
          description: Revision ID (cuid) - preferred method
          schema:
            type: string
        - name: revisionNumber
          in: query
          description: Revision number (deprecated, use revision instead)
          deprecated: true
          schema:
            type: string
      responses:
        "200":
          description: OK
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/models.Prompt"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/models.Error"
        "404":
          description: Prompt not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/models.Error"
  /v1/studio/prompts/{promptId}/compile:
    post:
      tags:
        - Studio
      summary: Compile a prompt
      description: Compiles a prompt by replacing variable placeholders ({VAR} or {{VAR}}) with the provided values. Returns the prompt with all variables substituted.
      parameters:
        - name: promptId
          in: path
          description: Prompt ID (cuid) or textId (deprecated)
          required: true
          schema:
            type: string
        - name: revision
          in: query
          description: Revision ID (cuid) - preferred method
          schema:
            type: string
        - name: revisionNumber
          in: query
          description: Revision number (deprecated, use revision instead)
          deprecated: true
          schema:
            type: string
      requestBody:
        description: Variables to substitute in the prompt
        content:
          application/json:
            schema:
              type: object
              required:
                - variables
              properties:
                variables:
                  type: object
                  additionalProperties:
                    type: string
                  description: Key-value map of variable names to their values
        required: true
      responses:
        "200":
          description: OK
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/models.CompiledPrompt"
        "400":
          description: Bad Request - missing required variables
          content:
            application/json:
              schema:
                type: object
                required:
                  - error
                  - missing_variables
                properties:
                  error:
                    type: string
                  missing_variables:
                    type: array
                    items:
                      type: string
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/models.Error"
        "404":
          description: Prompt not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/models.Error"
security:
  - ApiKeyAuth: []
components:
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: X-Qualifire-API-Key
  schemas:
    models.Error:
      type: object
      required:
        - error
      properties:
        error:
          type: string
          description: A human-readable error message
          example: Invalid API key
    models.LlmToolCall:
      type: object
      required:
        - function
      properties:
        id:
          type: string
          description: Unique identifier for the tool call
        type:
          type: string
          description: The type of tool call (e.g., "function")
        function:
          type: object
          required:
            - name
            - arguments
          properties:
            name:
              type: string
              description: Name of the tool being called
            arguments:
              type: object
              additionalProperties: true
              description: Arguments passed to the tool (key-value pairs)
    models.LlmToolDefinition:
      type: object
      required:
        - name
      properties:
        name:
          type: string
          description: Name of the tool
        description:
          type: string
          description: Description of what the tool does
        parameters:
          type: object
          additionalProperties: true
          description: JSON Schema describing the tool's parameters
    models.LlmMessage:
      type: object
      required:
        - role
      properties:
        role:
          type: string
          description: The role of the message sender
          enum:
            - system
            - user
            - assistant
            - tool
            - developer
        content:
          type: string
          nullable: true
          description: The text content of the message
        tool_calls:
          type: array
          description: Tool calls made by the assistant
          items:
            $ref: "#/components/schemas/models.LlmToolCall"
    models.DetectionResult:
      type: object
      required:
        - name
        - score
        - label
        - confidence_score
        - reason
        - flagged
      properties:
        name:
          type: string
        score:
          type: number
        label:
          type: string
        confidence_score:
          type: number
        reason:
          type: string
        quote:
          type: string
        flagged:
          type: boolean
          description: Whether this result was flagged as a violation
        data:
          type: string
          description: Additional context data (e.g., the assertion text for policy checks)
          nullable: true
    models.ApiEvaluationEvaluateRequest:
      type: object
      description: "At least one of `input`, `output`, or `messages` must be provided."
      properties:
        input:
          type: string
          description: The input text (e.g., user prompt)
        output:
          type: string
          description: The output text (e.g., LLM response)
        messages:
          type: array
          description: Conversation messages for multi-turn evaluation
          items:
            $ref: "#/components/schemas/models.LlmMessage"
        available_tools:
          type: array
          description: List of tools available to the LLM. Required when tool_use_quality_check is enabled.
          items:
            $ref: "#/components/schemas/models.LlmToolDefinition"
        hallucinations_check:
          type: boolean
          description: Check for factual inaccuracies or hallucinations
        hallucinations_mode:
          type: string
          description: Quality/speed tradeoff for hallucination checks
          enum:
            - speed
            - balanced
            - quality
        grounding_check:
          type: boolean
          description: Check if the output is grounded in the provided input/context
        grounding_mode:
          type: string
          description: Quality/speed tradeoff for grounding checks
          enum:
            - speed
            - balanced
            - quality
        grounding_multi_turn_mode:
          type: boolean
          description: Use full conversation context for grounding checks
        prompt_injections:
          type: boolean
          description: Check for prompt injection attempts
        pii_check:
          type: boolean
          description: Check for personally identifiable information
        content_moderation_check:
          type: boolean
          description: Check for harmful content (dangerous content, harassment, hate speech, sexual content)
        tool_use_quality_check:
          type: boolean
          description: Check the quality of tool selection and usage. Requires messages and available_tools.
        tuq_mode:
          type: string
          description: Quality/speed tradeoff for tool use quality checks
          enum:
            - speed
            - balanced
            - quality
        assertions:
          type: array
          description: Custom policy assertions to evaluate against (e.g., "don't give medical advice")
          items:
            type: string
        assertions_mode:
          type: string
          description: Quality/speed tradeoff for assertion checks
          enum:
            - speed
            - balanced
            - quality
        policy_target:
          type: string
          description: Whether policy assertions apply to input, output, or both
          enum:
            - input
            - output
            - both
        policy_multi_turn_mode:
          type: boolean
          description: Use full conversation context for policy checks
        policy_include_tools:
          type: boolean
          description: Include tool definitions and tool calls in policy assertion context. When true, assertions can reference available_tools and tool_calls from messages.
        allowed_topics:
          type: array
          description: List of allowed topics for topic scoping checks
          items:
            type: string
        topic_scoping_mode:
          type: string
          description: Quality/speed tradeoff for topic scoping checks
          enum:
            - speed
            - balanced
            - quality
        topic_scoping_target:
          type: string
          description: Whether topic scoping applies to input, output, or both
          enum:
            - input
            - output
            - both
        topic_scoping_multi_turn_mode:
          type: boolean
          description: Use full conversation context for topic scoping checks
        metadata:
          type: object
          additionalProperties:
            type: string
          description: Optional key-value pairs (string values only) to attach to the evaluation invocation
    models.EvaluationInvokeRequest:
      type: object
      description: "At least one of `input`, `output`, or `messages` must be provided."
      required:
        - evaluation_id
      properties:
        evaluation_id:
          type: string
          description: The ID of the pre-configured evaluation to invoke
        input:
          type: string
          description: The input text (e.g., user prompt)
        output:
          type: string
          description: The output text (e.g., LLM response)
        messages:
          type: array
          description: Conversation messages for multi-turn evaluation
          items:
            $ref: "#/components/schemas/models.LlmMessage"
        available_tools:
          type: array
          description: List of tools available to the LLM
          items:
            $ref: "#/components/schemas/models.LlmToolDefinition"
        metadata:
          type: object
          additionalProperties:
            type: string
          description: Optional key-value pairs (string values only) to attach to the evaluation invocation
    models.EvaluationResponse:
      type: object
      required:
        - status
        - score
        - evaluationResults
      properties:
        status:
          type: string
          description: Overall evaluation status
          enum:
            - success
            - warning
            - failed
        score:
          type: number
          nullable: true
          description: Overall evaluation score (0-100)
        evaluationResults:
          type: array
          items:
            $ref: "#/components/schemas/models.EvaluationResult"
    models.EvaluationResult:
      type: object
      required:
        - type
        - results
      properties:
        type:
          type: string
          description: The evaluation type (e.g., hallucinations, grounding, policy, pii, prompt_injections, safety, tuq, topic_scoping)
        results:
          type: array
          items:
            $ref: "#/components/schemas/models.DetectionResult"
    models.Prompt:
      type: object
      required:
        - id
        - name
        - revision
        - description
        - created_at
        - messages
        - tools
        - parameters
      properties:
        id:
          type: string
        name:
          type: string
          nullable: true
        description:
          type: string
          nullable: true
        created_at:
          type: string
          nullable: true
        revision:
          type: integer
        messages:
          type: array
          items:
            type: object
            required:
              - role
              - content
            properties:
              role:
                type: string
              content:
                type: string
                nullable: true
        tools:
          type: array
          items:
            $ref: "#/components/schemas/models.ToolResponse"
        parameters:
          type: object
          properties:
            model:
              type: string
            temperature:
              type: number
            top_p:
              type: number
            frequency_penalty:
              type: number
            presence_penalty:
              type: number
            max_tokens:
              type: integer
            reasoning_effort:
              type: string
              description: Reasoning effort level for supported models
    models.CompiledPrompt:
      type: object
      required:
        - id
        - name
        - revision
        - messages
        - tools
        - parameters
      properties:
        id:
          type: string
        name:
          type: string
          nullable: true
        revision:
          type: integer
        messages:
          type: array
          items:
            type: object
            required:
              - role
              - content
            properties:
              role:
                type: string
              content:
                type: string
                description: Fully compiled content with variables substituted
        tools:
          type: array
          items:
            $ref: "#/components/schemas/models.ToolResponse"
        parameters:
          type: object
          additionalProperties: true
    models.ToolResponse:
      type: object
      required:
        - type
        - function
      properties:
        type:
          type: string
        function:
          type: object
          required:
            - name
          properties:
            name:
              type: string
            description:
              type: string
            parameters:
              type: object
              additionalProperties: true