containerization/Sources/Integration/Suite.swift

//===----------------------------------------------------------------------===//
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//

import ArgumentParser
import Containerization
import ContainerizationError
import ContainerizationExtras
import ContainerizationOCI
import ContainerizationOS
import Foundation
import Logging
import NIOCore
import NIOPosix
import Synchronization

actor UnpackCoordinator {
    private var inFlight: [String: Task<Containerization.Mount, Error>] = [:]

    func unpack(
        key: String,
        operation: @escaping @Sendable () async throws -> Containerization.Mount
    ) async throws -> Containerization.Mount {
        if let existing = inFlight[key] {
            return try await existing.value
        }

        let task = Task {
            try await operation()
        }
        inFlight[key] = task

        defer {
            inFlight.removeValue(forKey: key)
        }

        return try await task.value
    }
}

struct Test: Sendable {
    var name: String
    var work: @Sendable () async throws -> Void

    init(_ name: String, _ work: @escaping @Sendable () async throws -> Void) {
        self.name = name
        self.work = work
    }
}

final class JobQueue<T>: Sendable where T: Sendable {
    struct State: Sendable {
        var next = 0
        var jobs: [T]
    }

    private let lock: Mutex<State>
    init(_ jobs: [T]) {
        self.lock = Mutex(State(jobs: jobs))
    }

    func pop() -> T? {
        self.lock.withLock { state in
            guard state.next < state.jobs.count else {
                return nil
            }
            defer {
                state.next += 1
            }
            return state.jobs[state.next]
        }
    }
}

let log = {
    LoggingSystem.bootstrap(StreamLogHandler.standardError)
    var log = Logger(label: "com.apple.containerization")
    log.logLevel = .debug
    return log
}()

enum IntegrationError: Swift.Error {
    case assert(msg: String)
    case noOutput
}

struct SkipTest: Swift.Error, CustomStringConvertible {
    let reason: String

    var description: String {
        reason
    }
}

@main
struct IntegrationSuite: AsyncParsableCommand {
    static let appRoot: URL = {
        FileManager.default.urls(
            for: .applicationSupportDirectory,
            in: .userDomainMask
        ).first!
        .appendingPathComponent("com.apple.containerization")
    }()

    private static let _contentStore: ContentStore = {
        try! LocalContentStore(path: appRoot.appending(path: "content"))
    }()

    private static let _imageStore: ImageStore = {
        try! ImageStore(
            path: appRoot,
            contentStore: contentStore
        )
    }()

    static let _testDir: URL = {
        FileManager.default.uniqueTemporaryDirectory(create: true)
    }()

    static var testDir: URL {
        _testDir
    }

    static var imageStore: ImageStore {
        _imageStore
    }

    static var contentStore: ContentStore {
        _contentStore
    }

    static let initImage = "vminit:latest"

    private static let unpackCoordinator = UnpackCoordinator()

    @Option(name: .shortAndLong, help: "Path to a directory for boot logs")
    var bootlogDir: String = "./bin/integration-bootlogs"

    @Option(name: .shortAndLong, help: "Path to a kernel binary")
    var kernel: String = Self.defaultKernelPath

    #if arch(arm64)
    private static let kernelCandidates = ["./bin/vmlinux-arm64"]
    #elseif arch(x86_64)
    private static let kernelCandidates = ["./bin/vmlinuz-x86_64", "./bin/vmlinux-x86_64"]
    #else
    private static let kernelCandidates = ["./bin/vmlinux"]
    #endif

    private static let defaultKernelPath: String = {
        let fm = FileManager.default
        for candidate in kernelCandidates where fm.fileExists(atPath: candidate) {
            return candidate
        }
        return kernelCandidates[0]
    }()

    @Option(name: .shortAndLong, help: "Maximum number of concurrent tests")
    var maxConcurrency: Int = 4

    @Option(name: .shortAndLong, help: "Only run tests whose names contain this string")
    var filter: String?

    static func binPath(name: String) -> URL {
        URL(fileURLWithPath: FileManager.default.currentDirectoryPath)
            .appendingPathComponent("bin")
            .appendingPathComponent(name)
    }

    static let eventLoop = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount)

    func bootstrap(_ testID: String) async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager, image: Containerization.Image, bootLog: BootLog) {
        let reference = "ghcr.io/linuxcontainers/alpine:3.20"
        let store = Self.imageStore

        let initImage = try await store.getInitImage(reference: Self.initImage)
        let initfs = try await {
            let p = Self.binPath(name: "init.block")
            do {
                return try await initImage.initBlock(at: p, for: .linuxArm)
            } catch let err as ContainerizationError {
                guard err.code == .exists else {
                    throw err
                }
                return .block(
                    format: "ext4",
                    source: p.absolutePath(),
                    destination: "/",
                    options: ["ro"]
                )
            }
        }()

        var testKernel = Kernel(path: .init(filePath: kernel), platform: .linuxArm)
        testKernel.commandLine.addDebug()
        let image = try await Self.fetchImage(reference: reference, store: store)
        let platform = Platform(arch: "arm64", os: "linux", variant: "v8")

        // Unpack to shared location with coordination to prevent concurrent unpacks
        let fsPath = Self.testDir.appending(component: image.digest)
        let fs = try await Self.unpackCoordinator.unpack(key: fsPath.absolutePath()) {
            do {
                let unpacker = EXT4Unpacker(blockSizeInBytes: 2.gib())
                return try await unpacker.unpack(image, for: platform, at: fsPath)
            } catch let err as ContainerizationError {
                if err.code == .exists {
                    return .block(
                        format: "ext4",
                        source: fsPath.absolutePath(),
                        destination: "/",
                        options: []
                    )
                }
                throw err
            }
        }

        // Clone to test-specific path
        let clPath = Self.testDir.appending(component: "\(testID).ext4").absolutePath()
        try? FileManager.default.removeItem(atPath: clPath)

        let cl = try fs.clone(to: clPath)

        // Create bootLog directory and per-container bootLog path
        let bootlogDirURL = URL(filePath: bootlogDir)
        try? FileManager.default.createDirectory(at: bootlogDirURL, withIntermediateDirectories: true)
        let bootlogURL = bootlogDirURL.appendingPathComponent("\(testID).log")

        return (
            cl,
            VZVirtualMachineManager(
                kernel: testKernel,
                initialFilesystem: initfs,
                group: Self.eventLoop
            ),
            image,
            BootLog.file(path: bootlogURL)
        )
    }

    static func fetchImage(reference: String, store: ImageStore) async throws -> Containerization.Image {
        do {
            return try await store.get(reference: reference)
        } catch let error as ContainerizationError {
            if error.code == .notFound {
                return try await store.pull(reference: reference)
            }
            throw error
        }
    }

    static func adjustLimits() throws {
        var limits = rlimit()
        guard getrlimit(RLIMIT_NOFILE, &limits) == 0 else {
            throw POSIXError(.init(rawValue: errno)!)
        }
        limits.rlim_cur = 65536
        limits.rlim_max = 65536

        guard setrlimit(RLIMIT_NOFILE, &limits) == 0 else {
            throw POSIXError(.init(rawValue: errno)!)
        }
    }

    private func macOS26Tests() -> [Test] {
        if #available(macOS 26.0, *) {
            return [
                Test("container interface custom MTU", testInterfaceMTU),
                Test("container networking disabled", testNetworkingDisabled),
                Test("container networking enabled", testNetworkingEnabled),
                Test("container networking enabled ipv6", testNetworkingEnabledIPv6),
                Test("container IPv6 address", testIPv6AddressAdd),
                Test("container IPv6 default route", testIPv6DefaultRoute),
                Test("container IPv6 gateway outside subnet", testIPv6GatewayOutsideSubnet),
                Test("container IPv6 only default route", testIPv6OnlyDefaultRoute),
                Test("container IPv6 only gateway outside subnet", testIPv6OnlyGatewayOutsideSubnet),
                Test("container IPv6 dual stack", testIPv6DualStack),
                Test("pod IPv6 address", testPodIPv6AddressAdd),
            ]
        }
        return []
    }

    // Why does this exist?
    //
    // We need the virtualization entitlement to execute these tests.
    // There currently does not exist a straightforward way to do this
    // in a pure swift package.
    //
    // In order to not have a dependency on xcode, we create an executable
    // for our integration tests that can be signed then ran.
    //
    // We also can't import Testing as it expects to be run from a runner.
    // Hopefully this improves over time.
    func run() async throws {
        try Self.adjustLimits()
        let suiteStarted = Date().timeIntervalSinceReferenceDate
        log.info("starting integration suite\n")

        let tests: [Test] =
            [
                // Containers
                Test("process true", testProcessTrue),
                Test("process false", testProcessFalse),
                Test("process echo hi", testProcessEchoHi),
                Test("process no executable", testProcessNoExecutable),
                Test("process user", testProcessUser),
                Test("process stdin", testProcessStdin),
                Test("process home envvar", testProcessHomeEnvvar),
                Test("process custom home envvar", testProcessCustomHomeEnvvar),
                Test("process tty ensure TERM", testProcessTtyEnvvar),
                Test("multiple concurrent processes", testMultipleConcurrentProcesses),
                Test("multiple concurrent processes with output stress", testMultipleConcurrentProcessesOutputStress),
                Test("container hostname", testHostname),
                Test("container hostname defaults to container id", testHostnameDefaultsToContainerID),
                Test("container hosts", testHostsFile),
                Test("container mount", testMounts),
                Test("container stop idempotency", testContainerStopIdempotency),
                Test("nested virt", testNestedVirtualizationEnabled),
                Test("container manager", testContainerManagerCreate),
                Test("container reuse", testContainerReuse),
                Test("container /dev/console", testContainerDevConsole),
                Test("container statistics", testContainerStatistics),
                Test("container cgroup limits", testCgroupLimits),
                Test("container memory events OOM kill", testMemoryEventsOOMKill),
                Test("container no serial console", testNoSerialConsole),
                Test("unix socket into guest", testUnixSocketIntoGuest),
                Test("unix socket into guest long container id", testUnixSocketIntoGuestLongContainerID),
                Test("unix socket into guest symlink", testUnixSocketIntoGuestSymlink),
                Test("container non-closure constructor", testNonClosureConstructor),
                Test("container test large stdio ingest", testLargeStdioOutput),
                Test("process delete idempotency", testProcessDeleteIdempotency),
                Test("multiple execs without delete", testMultipleExecsWithoutDelete),
                Test("container bootlog using filehandle", testBootLogFileHandle),
                Test("container capabilities sys admin", testCapabilitiesSysAdmin),
                Test("container capabilities net admin", testCapabilitiesNetAdmin),
                Test("container capabilities OCI default", testCapabilitiesOCIDefault),
                Test("container capabilities all capabilities", testCapabilitiesAllCapabilities),
                Test("container capabilities file ownership", testCapabilitiesFileOwnership),
                Test("container stat", testStat),
                Test("container copy in", testCopyIn),
                Test("container copy in file to existing directory", testCopyInFileToExistingDirectory),
                Test("container copy in file to missing directory fails", testCopyInFileToMissingDirectoryFails),
                Test("container copy in directory over existing file fails", testCopyInDirectoryOverExistingFileFails),
                Test("container copy out", testCopyOut),
                Test("container copy large file", testCopyLargeFile),
                Test("container copy in directory", testCopyInDirectory),
                Test("container copy out directory", testCopyOutDirectory),
                Test("container copy empty file", testCopyEmptyFile),
                Test("container copy empty directory", testCopyEmptyDirectory),
                Test("container copy binary file", testCopyBinaryFile),
                Test("container copy multiple files", testCopyMultipleFiles),
                Test("container copy directory round trip", testCopyDirectoryRoundTrip),
                Test("container copy in create parents", testCopyInCreateParents),
                Test("container copy file permissions", testCopyFilePermissions),
                Test("container copy large directory", testCopyLargeDirectory),
                Test("container read-only rootfs", testReadOnlyRootfs),
                Test("container read-only rootfs hosts file", testReadOnlyRootfsHostsFileWritten),
                Test("container read-only rootfs DNS", testReadOnlyRootfsDNSConfigured),
                Test("container writable layer", testWritableLayer),
                Test("container writable layer journal writeback", testWritableLayerJournalWriteback),
                Test("container writable layer journal ordered", testWritableLayerJournalOrdered),
                Test("container writable layer journal data", testWritableLayerJournalData),
                Test("container writable layer preserves lower", testWritableLayerPreservesLowerLayer),
                Test("container writable layer reads from lower", testWritableLayerReadsFromLower),
                Test("container writable layer with ro lower", testWritableLayerWithReadOnlyLower),
                Test("container writable layer size", testWritableLayerSize),
                Test("container writable layer DNS and hosts", testWritableLayerWithDNSAndHosts),
                Test("container frozen ext4 clone", testFrozenExt4Clone),
                Test("container trim ext4 clone", testTrimExt4Clone),
                Test("large stdin input", testLargeStdinInput),
                Test("exec large stdin input", testExecLargeStdinInput),
                Test("exec custom path resolution", testExecCustomPathResolution),
                Test("stdin explicit close", testStdinExplicitClose),
                Test("stdin binary data", testStdinBinaryData),
                Test("stdin multiple chunks", testStdinMultipleChunks),
                Test("stdin very large", testStdinVeryLarge),
                Test("container single file mount", testSingleFileMount),
                Test("container single file mount read-only", testSingleFileMountReadOnly),
                Test("container single file mount write-back", testSingleFileMountWriteBack),
                Test("container single file mount symlink", testSingleFileMountSymlink),
                Test("container rlimit open files", testRLimitOpenFiles),
                Test("container rlimit multiple", testRLimitMultiple),
                Test("container rlimit exec", testRLimitExec),
                Test("container duplicate virtiofs mount", testDuplicateVirtiofsMount),
                Test("container duplicate virtiofs mount via symlink", testDuplicateVirtiofsMountViaSymlink),
                Test("container useInit basic", testUseInitBasic),
                Test("container useInit exit code propagation", testUseInitExitCodePropagation),
                Test("container useInit signal forwarding", testUseInitSignalForwarding),
                Test("container useInit zombie reaping", testUseInitZombieReaping),
                Test("container useInit with terminal", testUseInitWithTerminal),
                Test("container useInit with stdin", testUseInitWithStdin),
                Test("container sysctl", testSysctl),
                Test("container sysctl multiple", testSysctlMultiple),
                Test("container noNewPrivileges", testNoNewPrivileges),
                Test("container noNewPrivileges disabled", testNoNewPrivilegesDisabled),
                Test("container noNewPrivileges exec", testNoNewPrivilegesExec),
                Test("container workingDir created", testWorkingDirCreated),
                Test("container workingDir exec created", testWorkingDirExecCreated),
                Test("container mount sort by depth", testMountsSortedByDepth),
                Test("container VM resource overhead", testVMResourceOverhead),
                Test("container NBD mount", testContainerNBDMount),
                Test("container NBD read-only", testContainerNBDReadOnly),
                Test("container NBD raw block", testContainerNBDRawBlock),
                Test("container NBD volume identity", testContainerNBDVolumeIdentity),

                // Pods
                Test("pod single container", testPodSingleContainer),
                Test("pod multiple containers", testPodMultipleContainers),
                Test("pod container output", testPodContainerOutput),
                Test("pod concurrent containers", testPodConcurrentContainers),
                Test("pod exec in container", testPodExecInContainer),
                Test("pod exec in container env", testPodExecInContainerEnv),
                Test("pod container hostname", testPodContainerHostname),
                Test("pod container hostname defaults to container id", testPodContainerHostnameDefaultsToContainerID),
                Test("pod stop container idempotency", testPodStopContainerIdempotency),
                Test("pod list containers", testPodListContainers),
                Test("pod container statistics", testPodContainerStatistics),
                Test("pod memory events OOM kill", testPodMemoryEventsOOMKill),
                Test("pod container resource limits", testPodContainerResourceLimits),
                Test("pod container filesystem isolation", testPodContainerFilesystemIsolation),
                Test("pod container PID namespace isolation", testPodContainerPIDNamespaceIsolation),
                Test("pod container independent resource limits", testPodContainerIndependentResourceLimits),
                Test("pod shared PID namespace", testPodSharedPIDNamespace),
                Test("pod read-only rootfs", testPodReadOnlyRootfs),
                Test("pod read-only rootfs DNS", testPodReadOnlyRootfsDNSConfigured),
                Test("pod single file mount", testPodSingleFileMount),
                Test("pod container hosts config", testPodContainerHostsConfig),
                Test("pod multiple containers different DNS", testPodMultipleContainersDifferentDNS),
                Test("pod multiple containers different hosts", testPodMultipleContainersDifferentHosts),
                Test("pod level DNS", testPodLevelDNS),
                Test("pod level DNS with container override", testPodLevelDNSWithContainerOverride),
                Test("pod level hosts", testPodLevelHosts),
                Test("pod level hosts with container override", testPodLevelHostsWithContainerOverride),
                Test("pod level hostname", testPodLevelHostname),
                Test("pod level hostname with container override", testPodLevelHostnameWithContainerOverride),
                Test("pod rlimit open files", testPodRLimitOpenFiles),
                Test("pod rlimit exec", testPodRLimitExec),
                Test("pod useInit basic", testPodUseInitBasic),
                Test("pod useInit exit code propagation", testPodUseInitExitCodePropagation),
                Test("pod useInit signal forwarding", testPodUseInitSignalForwarding),
                Test("pod useInit multiple containers", testPodUseInitMultipleContainers),
                Test("pod useInit with shared PID namespace", testPodUseInitWithSharedPIDNamespace),
                Test("pod unix socket into guest symlink", testPodUnixSocketIntoGuestSymlink),
                Test("pod sysctl", testPodSysctl),
                Test("pod sysctl multiple containers", testPodSysctlMultipleContainers),
                Test("pod shared NBD volume", testPodSharedNBDVolume),
                Test("pod multiple NBD volumes", testPodMultipleNBDVolumes),
                Test("pod unreferenced NBD volume", testPodUnreferencedVolume),
                Test("pod NBD volume persistence", testPodNBDVolumePersistence),
                Test("pod NBD concurrent writes", testPodNBDConcurrentWrites),
                Test("pod NBD volume identity", testPodNBDVolumeIdentity),
                Test("pod invalid volume reference", testPodInvalidVolumeReference),
                Test("pod duplicate volume name", testPodDuplicateVolumeName),
                Test("pod filesystem operation", testPodFilesystemOperation),
            ] + macOS26Tests()

        let filteredTests: [Test]
        if let filter {
            filteredTests = tests.filter { $0.name.contains(filter) }
            log.info("filter '\(filter)' matched \(filteredTests.count)/\(tests.count) tests")
        } else {
            filteredTests = tests
        }

        let passed: Atomic<Int> = Atomic(0)
        let skipped: Atomic<Int> = Atomic(0)

        await withTaskGroup(of: Void.self) { group in
            let jobQueue = JobQueue(filteredTests)
            for _ in 0..<maxConcurrency {
                group.addTask { @Sendable in
                    while let job = jobQueue.pop() {
                        do {
                            log.info("test \(job.name) started...")

                            let started = Date().timeIntervalSinceReferenceDate
                            try await job.work()
                            let lasted = Date().timeIntervalSinceReferenceDate - started

                            log.info("✅ test \(job.name) complete in \(lasted)s.")
                            passed.add(1, ordering: .relaxed)
                        } catch let err as SkipTest {
                            log.info("⏭️ skipped test: \(err)")
                            skipped.add(1, ordering: .relaxed)
                        } catch {
                            log.error("❌ test \(job.name) failed: \(error)")
                        }
                    }
                }
            }
            await group.waitForAll()
        }

        let passedCount = passed.load(ordering: .acquiring)
        let skippedCount = skipped.load(ordering: .acquiring)

        let ended = Date().timeIntervalSinceReferenceDate - suiteStarted
        var finishingText = "\n\nIntegration suite completed in \(ended)s with \(passedCount)/\(filteredTests.count) passed"
        if skipped.load(ordering: .acquiring) > 0 {
            finishingText += " and \(skippedCount)/\(filteredTests.count) skipped"
        }
        finishingText += "!"

        log.info("\(finishingText)")

        try? FileManager.default.removeItem(at: Self.testDir)
        if passedCount + skippedCount < filteredTests.count {
            log.error("❌")
            throw ExitCode(1)
        }
    }
}