Vision Tap-to-Segment API

import Vision
import UIKit
import CoreImage

final class TapToSegmentViewController: UIViewController {

    private let imageView = UIImageView()
    private var sourceImage: UIImage = UIImage(named: "cafe_photo")!
    private var currentRequest: GenerateIterativeSegmentationRequest?
    private var handler: ImageRequestHandler?

    override func viewDidLoad() {
        super.viewDidLoad()
        setupImageView()
        ensureModelReady()
    }

    private func setupImageView() {
        imageView.image = sourceImage
        imageView.contentMode = .scaleAspectFit
        imageView.isUserInteractionEnabled = true
        imageView.frame = view.bounds
        view.addSubview(imageView)

        let tap = UITapGestureRecognizer(target: self, action: #selector(handleTap(_:)))
        imageView.addGestureRecognizer(tap)
    }

    private func ensureModelReady() {
        Task {
            // Check if segmentation assets are downloaded
            let status = await GenerateIterativeSegmentationRequest.assetStatus
            if status != .ready {
                try await GenerateIterativeSegmentationRequest.downloadAssets()
            }
        }
    }

    @objc private func handleTap(_ gesture: UITapGestureRecognizer) {
        let tapPoint = gesture.location(in: imageView)
        let imageSize = imageView.bounds.size

        // Convert UIKit coordinates (origin top-left) to
        // Vision normalized coordinates (origin bottom-left, values 0-1)
        let normalizedPoint = NormalizedPoint(
            x: tapPoint.x / imageSize.width,
            y: 1.0 - (tapPoint.y / imageSize.height)
        )

        Task {
            await segmentObject(at: normalizedPoint)
        }
    }

    private func segmentObject(at point: NormalizedPoint) async {
        guard let cgImage = sourceImage.cgImage else { return }

        // Create handler once and reuse across refinements
        if handler == nil {
            handler = ImageRequestHandler(cgImage)
        }

        do {
            // Build the iterative segmentation request with an initial seed point
            var request = GenerateIterativeSegmentationRequest()
            request.addPoint(point, label: .foreground)

            // If we already have a prior request, carry over its state for refinement
            if let prior = currentRequest {
                request.previousResults = prior.results
            }

            let result = try await handler!.perform(request)

            // result.pixelBuffer is a mask: white = segmented, black = background
            if let mask = result.pixelBuffer {
                let maskImage = CIImage(cvPixelBuffer: mask)
                let composite = applyMask(maskImage, to: CIImage(cgImage: cgImage))
                await MainActor.run {
                    imageView.image = composite
                }
            }

            currentRequest = request
        } catch {
            print("Segmentation failed: \(error)")
        }
    }

    private func applyMask(_ mask: CIImage, to source: CIImage) -> UIImage {
        let context = CIContext()
        let filter = CIFilter.blendWithMask()
        filter.inputImage = source
        filter.maskImage = mask.samplingLinear()
        filter.backgroundImage = CIImage.empty()
        let output = filter.outputImage ?? source
        if let cgOut = context.createCGImage(output, from: output.extent) {
            return UIImage(cgImage: cgOut)
        }
        return sourceImage
    }
}