Evaluations Framework: Synthetic Data & Agentic Evaluation

import Evaluations
import FoundationModels

// MARK: - Domain model

struct BookSample: Codable, Sendable {
    let review: String
    let tags: [String]
}

// MARK: - Seed data

let seedSamples: [ModelSample<String, [String]>] = [
    ModelSample(
        prompt: "A sweeping romance set in Regency England with wit and social commentary.",
        expectedOutput: ["romance", "classic", "regency", "social-commentary"]
    ),
    ModelSample(
        prompt: "A gothic horror tale of ambition gone terribly wrong and its monstrous consequences.",
        expectedOutput: ["horror", "gothic", "classic", "science-fiction"]
    ),
    ModelSample(
        prompt: "An adventure on the high seas chasing an obsessive white whale to the ends of the earth.",
        expectedOutput: ["adventure", "classic", "sea", "obsession"]
    )
]

// MARK: - Synthetic generation

func generateExpandedDataset() async throws -> [ModelSample<String, [String]>] {
    let generationPrompt = """
        Generate diverse book review samples covering a wide range of genres,
        moods, tones, and lengths. Each review should feel like it was written
        by a real reader. Vary sentence complexity and emotional register.
        """

    let generator = SampleGenerator(
        prompt: generationPrompt,
        dataset: seedSamples,
        targetCount: 20,
        sessionProvider: {
            // Use on-device model; swap to PrivateCloudComputeLanguageModel for larger context
            let session = LanguageModelSession(
                instructions: Instructions("""
                    You generate synthetic book review samples for an evaluation dataset.
                    Rules:
                    - Reviews must be at least 100 characters.
                    - Generate between 3 and 8 lowercase tags per review.
                    - Cover a mix of genres: literary, sci-fi, mystery, romance, horror, non-fiction.
                    """)
            )
            return session
        },
        samplingStrategy: .random,
        validator: { sample in
            // Rule 1: review must be at least 100 characters
            guard sample.prompt.count >= 100 else {
                return .invalid(reason: "Review too short (\(sample.prompt.count) chars)")
            }
            // Rule 2: between 3 and 8 tags
            let tagCount = sample.expectedOutput.count
            guard (3...8).contains(tagCount) else {
                return .invalid(reason: "Tag count out of range: \(tagCount)")
            }
            // Rule 3: all tags must be lowercase
            let allLower = sample.expectedOutput.allSatisfy { $0 == $0.lowercased() }
            guard allLower else {
                return .invalid(reason: "Tags contain uppercase characters")
            }
            return .valid
        }
    )

    var expandedDataset = seedSamples

    for try await newSample in generator.run() {
        expandedDataset.append(newSample)
    }

    print("Valid samples: \(generator.samples.count)")
    print("Invalid samples: \(generator.invalidSamples.count)")

    return expandedDataset
}