Evaluations Framework

import Evaluations
import Testing
import FoundationModels

// MARK: - Domain Types

struct BookReview {
    let title: String
    let reviewText: String
}

struct BookTags {
    let tags: [String]
}

// MARK: - Simulated Service

actor BookTaggingService {
    func generateTags(for review: BookReview) async throws -> BookTags {
        // In production this calls a LanguageModelSession.
        // Simulated here for demonstration.
        return BookTags(tags: ["classic", "romance", "19th-century", "british", "fiction"])
    }
}

// MARK: - Evaluation Definition

struct BookTaggingEvaluation: Evaluation {
    typealias Input = BookReview
    typealias Output = BookTags

    // Step 1: Define what code you are measuring.
    func subject(from input: BookReview) async throws -> BookTags {
        let service = BookTaggingService()
        return try await service.generateTags(for: input)
    }

    // Step 2: Define the dataset.
    var dataset: [ModelSample<BookReview, BookTags>] {
        [
            ModelSample(
                input: BookReview(title: "Pride & Prejudice",
                                  reviewText: "A witty exploration of marriage and social class in Regency England."),
                expectedOutput: BookTags(tags: ["classic", "romance", "regency", "british"])
            ),
            ModelSample(
                input: BookReview(title: "Dracula",
                                  reviewText: "A gothic horror novel told through letters and diary entries."),
                expectedOutput: BookTags(tags: ["horror", "gothic", "vampire", "victorian"])
            ),
            ModelSample(
                input: BookReview(title: "The Secret Garden",
                                  reviewText: "A neglected garden transforms a lonely child in this timeless classic."),
                expectedOutput: BookTags(tags: ["classic", "children", "nature", "british"])
            )
        ]
    }

    // Step 3: Define metrics.
    var metrics: [any Metric<BookTags>] {
        [
            Metric(name: "TagCount") { output in
                let count = output.tags.count
                return (3...8).contains(count) ? .pass : .fail
            }
        ]
    }

    // Step 4: Summarize across all samples.
    func aggregateMetrics(using results: EvaluationResults<BookTags>) -> [AggregateMetric] {
        let tagCountMetrics = results.metrics(named: "TagCount")
        let passRate = tagCountMetrics.passRate
        return [
            AggregateMetric(name: "TagCountPassRate", value: passRate)
        ]
    }
}

// MARK: - Swift Testing Integration

@Suite("Book Tagging Quality", .tags(.evaluation))
struct BookTaggingTests {
    let evaluation = BookTaggingEvaluation()

    @Test("Tag count meets quality bar",
          .evaluates(BookTaggingEvaluation(),
                     notes: ["model": "on-device", "version": "1.0"]))
    func tagCountQuality(results: EvaluationResultsBundle<BookTags>) async throws {
        let passRate = results.aggregateValue(named: "TagCountPassRate") ?? 0.0
        // Assert the service generates the correct number of tags at least 80% of the time.
        #expect(passRate >= 0.80, "TagCount pass rate \(passRate) is below the 0.80 optimization target")
    }
}