Container isolation: fix stdio stall + per-exec connection leak; add guest image pipeline
Host-side (ships with a normal swift build): - LinuxProcess: non-blocking stdio relay (O_NONBLOCK + nucleicDrainNonBlocking) so a wedged stream can't head-of-line-block sibling execs' relays; atomic stdio-or-abort start (patches #5, #6). - Vminitd: bounded deleteProcess timeout so teardown can't hang a wedged channel (patch #7). - ContainerizedProcessHandle: call LinuxProcess.delete() after exit and on force-close — fixes a per-turn leak (per-exec vsock/gRPC connection + runConnections() task) in the long-lived shared control container. Likely the "degrades until app restart" root cause. - ClaudeCodeBackend: map the atomic-start abort to a recoverable AgentError so a failed launch settles as retryable instead of locking the composer. Guest-side (rides the custom vminitd initfs; inert until the image is built): - ManagedProcess: offload the blocking start off the gRPC event loop (patch #8). - Per-exec cgroups (patch #9) recorded as design only — cross-cutting. Pipeline: - .github/workflows/vminit-image.yml builds vminitd from the vendored source and pushes ghcr.io/abkslm/vminit; ContainerEngine.vminitReference repointed at the custom image. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
@@ -128,6 +128,42 @@ public final class LinuxProcess: Sendable {
|
||||
}
|
||||
|
||||
extension LinuxProcess {
|
||||
/// [Nucleic vendored patch] Put a connected stdio FileHandle's fd into non-blocking mode so the
|
||||
/// relay's reads (``nucleicDrainNonBlocking``) can never park the shared readability queue. No-op
|
||||
/// if the handle is nil.
|
||||
static func nucleicSetNonBlocking(_ handle: FileHandle?) {
|
||||
guard let fd = handle?.fileDescriptor else { return }
|
||||
let flags = fcntl(fd, F_GETFL, 0)
|
||||
if flags >= 0 { _ = fcntl(fd, F_SETFL, flags | O_NONBLOCK) }
|
||||
}
|
||||
|
||||
/// [Nucleic vendored patch] Drain `fd` (already O_NONBLOCK) without ever blocking. Returns the
|
||||
/// bytes read this pass plus whether the stream hit EOF (or a hard error). On EAGAIN it returns
|
||||
/// what it has with `eof == false`; the readability `DispatchSource` fires again when more data
|
||||
/// arrives. Upstream read with `FileHandle.availableData`, a *blocking* read: if one exec's guest
|
||||
/// stdout wedged mid-stream, that read parked Foundation's shared readability thread and
|
||||
/// head-of-line-blocked EVERY other exec's stdout/stderr relay (the "one stuck session freezes the
|
||||
/// others" failure). A non-blocking drain can never park that thread, so a wedged stream is
|
||||
/// contained to its own exec.
|
||||
static func nucleicDrainNonBlocking(_ fd: Int32) -> (data: Data, eof: Bool) {
|
||||
var out = Data()
|
||||
var buf = [UInt8](repeating: 0, count: 64 * 1024)
|
||||
while true {
|
||||
let n = buf.withUnsafeMutableBytes { read(fd, $0.baseAddress, $0.count) }
|
||||
if n > 0 {
|
||||
out.append(contentsOf: buf[0..<n])
|
||||
} else if n == 0 {
|
||||
return (out, true) // EOF: guest closed the write side
|
||||
} else if errno == EINTR {
|
||||
continue
|
||||
} else if errno == EAGAIN || errno == EWOULDBLOCK {
|
||||
return (out, false) // drained for now; not EOF
|
||||
} else {
|
||||
return (out, true) // hard error → treat as EOF so the relay finishes
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func setupIO(listeners: [VsockListener?]) async throws -> [FileHandle?] {
|
||||
let handles = try await Timeout.run(seconds: 3) {
|
||||
try await withThrowingTaskGroup(of: (Int, FileHandle?).self) { group in
|
||||
@@ -170,36 +206,43 @@ extension LinuxProcess {
|
||||
let (stream, cc) = AsyncStream<Void>.makeStream()
|
||||
if let stdout = self.ioSetup.stdout {
|
||||
configuredStreams += 1
|
||||
// [Nucleic vendored patch] Non-blocking relay (see nucleicDrainNonBlocking): mark the
|
||||
// connected fd O_NONBLOCK and drain it without a blocking read, so a wedged guest stdout
|
||||
// can't head-of-line-block sibling execs' relays on Foundation's shared readability queue.
|
||||
Self.nucleicSetNonBlocking(handles[1])
|
||||
handles[1]?.readabilityHandler = { handle in
|
||||
do {
|
||||
let data = handle.availableData
|
||||
if data.isEmpty {
|
||||
// This block is called when the producer (the guest) closes
|
||||
// the fd it is writing into.
|
||||
handles[1]?.readabilityHandler = nil
|
||||
cc.yield()
|
||||
return
|
||||
let (data, eof) = Self.nucleicDrainNonBlocking(handle.fileDescriptor)
|
||||
if !data.isEmpty {
|
||||
do {
|
||||
try stdout.writer.write(data)
|
||||
} catch {
|
||||
self.logger?.error("failed to write to stdout: \(error)")
|
||||
}
|
||||
try stdout.writer.write(data)
|
||||
} catch {
|
||||
self.logger?.error("failed to write to stdout: \(error)")
|
||||
}
|
||||
if eof {
|
||||
// The guest closed the fd it was writing into.
|
||||
handles[1]?.readabilityHandler = nil
|
||||
cc.yield()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let stderr = self.ioSetup.stderr {
|
||||
configuredStreams += 1
|
||||
// [Nucleic vendored patch] Non-blocking relay — same rationale as stdout above.
|
||||
Self.nucleicSetNonBlocking(handles[2])
|
||||
handles[2]?.readabilityHandler = { handle in
|
||||
do {
|
||||
let data = handle.availableData
|
||||
if data.isEmpty {
|
||||
handles[2]?.readabilityHandler = nil
|
||||
cc.yield()
|
||||
return
|
||||
let (data, eof) = Self.nucleicDrainNonBlocking(handle.fileDescriptor)
|
||||
if !data.isEmpty {
|
||||
do {
|
||||
try stderr.writer.write(data)
|
||||
} catch {
|
||||
self.logger?.error("failed to write to stderr: \(error)")
|
||||
}
|
||||
try stderr.writer.write(data)
|
||||
} catch {
|
||||
self.logger?.error("failed to write to stderr: \(error)")
|
||||
}
|
||||
if eof {
|
||||
handles[2]?.readabilityHandler = nil
|
||||
cc.yield()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -288,6 +331,27 @@ extension LinuxProcess {
|
||||
)
|
||||
|
||||
let result = try await t.value
|
||||
|
||||
// [Nucleic vendored patch] Atomic stdio-or-abort. If a *configured* stdio stream never
|
||||
// connected from the guest (its FileHandle came back nil — the failure logged in setupIO),
|
||||
// starting the process would run it with a dead stream: stdin never delivered (it hangs)
|
||||
// or stdout/stderr never read ("no output, just a spinner" — the 60s stall in Nucleic
|
||||
// Control). Rather than launch a black-hole process, tear the just-created exec back down
|
||||
// and fail fast so the caller gets a clean, retryable start error instead of an eternal
|
||||
// silent stall the watchdog has to guess at.
|
||||
let configured = [
|
||||
self.ioSetup.stdin != nil, self.ioSetup.stdout != nil, self.ioSetup.stderr != nil,
|
||||
]
|
||||
let streamLabels = ["stdin", "stdout", "stderr"]
|
||||
if let missing = (0..<3).first(where: { configured[$0] && result[$0] == nil }) {
|
||||
try? await self.agent.deleteProcess(id: self.id, containerID: self.owningContainer)
|
||||
throw ContainerizationError(
|
||||
.internalError,
|
||||
message:
|
||||
"process \(self.id): \(streamLabels[missing]) stream never connected from the guest before start; aborting so the stdio transport stall surfaces as a retryable start error"
|
||||
)
|
||||
}
|
||||
|
||||
let pid = try await self.agent.startProcess(
|
||||
id: self.id,
|
||||
containerID: self.owningContainer
|
||||
|
||||
@@ -323,7 +323,16 @@ extension Vminitd: VirtualMachineAgent {
|
||||
$0.containerID = containerID
|
||||
}
|
||||
}
|
||||
_ = try await client.deleteProcess(request)
|
||||
// [Nucleic vendored patch] Bound the teardown RPC so a wedged agent channel can't hang an
|
||||
// exec's cleanup forever. Nucleic fires `LinuxProcess.delete()` after every turn to reclaim
|
||||
// the per-exec connection; if `deleteProcess` never returned, that reclaim task would leak
|
||||
// and the connection would stay open — reintroducing the very accumulation the delete exists
|
||||
// to prevent. Generous: a healthy delete returns in milliseconds, so this only trips a
|
||||
// genuinely stuck channel, and `LinuxProcess.performDeletion` still closes the agent
|
||||
// connection on the thrown deadline.
|
||||
var callOpts = GRPCCore.CallOptions.defaults
|
||||
callOpts.timeout = .seconds(30)
|
||||
_ = try await client.deleteProcess(request, options: callOpts)
|
||||
}
|
||||
|
||||
public func closeProcessStdin(id: String, containerID: String?) async throws {
|
||||
|
||||
Reference in New Issue
Block a user