diff --git a/Makefile b/Makefile index e177fef95..6bcf7e29e 100644 --- a/Makefile +++ b/Makefile @@ -211,7 +211,8 @@ INTEGRATION_TEST_SUITES := \ TestCLIKernelSet \ TestCLIAnonymousVolumes \ TestCLINotFound \ - TestCLINoParallelCases + TestCLINoParallelCases \ + TestCLISystemDF empty := space := $(empty) $(empty) diff --git a/Sources/ContainerResource/Common/FileManager+AllocatedSize.swift b/Sources/ContainerResource/Common/FileManager+AllocatedSize.swift new file mode 100644 index 000000000..721081be0 --- /dev/null +++ b/Sources/ContainerResource/Common/FileManager+AllocatedSize.swift @@ -0,0 +1,43 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation + +extension FileManager { + /// Total bytes allocated on disk for all files in a directory (recursive). + public func allocatedSize(of directory: URL) -> UInt64 { + guard + let enumerator = self.enumerator( + at: directory, + includingPropertiesForKeys: [.totalFileAllocatedSizeKey], + options: [.skipsHiddenFiles] + ) + else { + return 0 + } + + var size: UInt64 = 0 + for case let fileURL as URL in enumerator { + guard let resourceValues = try? fileURL.resourceValues(forKeys: [.totalFileAllocatedSizeKey]), + let fileSize = resourceValues.totalFileAllocatedSize + else { + continue + } + size += UInt64(fileSize) + } + return size + } +} diff --git a/Sources/Plugins/CoreImages/ImagesHelper.swift b/Sources/Plugins/CoreImages/ImagesHelper.swift index 6ba3318b5..c0a21862a 100644 --- a/Sources/Plugins/CoreImages/ImagesHelper.swift +++ b/Sources/Plugins/CoreImages/ImagesHelper.swift @@ -92,10 +92,11 @@ extension ImagesHelper { private func initializeImagesService(root: URL, containerSystemConfig: ContainerSystemConfig, log: Logger, routes: inout [String: XPCServer.RouteHandler]) throws { let contentStore = RemoteContentStoreClient() + let contentBlobsPath = root.appendingPathComponent("content/blobs/sha256") let imageStore = try ImageStore(path: root, contentStore: contentStore) let unpackStrategy = SnapshotStore.defaultUnpackStrategy(initImage: containerSystemConfig.vminit.image) let snapshotStore = try SnapshotStore(path: root, unpackStrategy: unpackStrategy, log: log) - let service = try ImagesService(contentStore: contentStore, imageStore: imageStore, snapshotStore: snapshotStore, log: log) + let service = try ImagesService(contentStore: contentStore, contentBlobsPath: contentBlobsPath, imageStore: imageStore, snapshotStore: snapshotStore, log: log) let harness = ImagesServiceHarness(service: service, log: log) routes[ImagesServiceXPCRoute.imagePull.rawValue] = XPCServer.route(harness.pull) diff --git a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift index 8bcf8095f..929e7b2e1 100644 --- a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift +++ b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift @@ -216,7 +216,7 @@ public actor ContainersService { for (id, state) in await self.containers { let bundlePath = self.containerRoot.appendingPathComponent(id) - let containerSize = Self.calculateDirectorySize(at: bundlePath.path) + let containerSize = FileManager.default.allocatedSize(of: bundlePath) totalSize += containerSize if state.snapshot.status == .running { @@ -243,39 +243,6 @@ public actor ContainersService { } } - /// Calculate directory size using APFS-aware resource keys - /// - Parameter path: Path to directory - /// - Returns: Total allocated size in bytes - private static nonisolated func calculateDirectorySize(at path: String) -> UInt64 { - let url = URL(fileURLWithPath: path) - let fileManager = FileManager.default - - guard - let enumerator = fileManager.enumerator( - at: url, - includingPropertiesForKeys: [.totalFileAllocatedSizeKey], - options: [.skipsHiddenFiles] - ) - else { - return 0 - } - - var totalSize: UInt64 = 0 - for case let fileURL as URL in enumerator { - guard - let resourceValues = try? fileURL.resourceValues( - forKeys: [.totalFileAllocatedSizeKey] - ), - let fileSize = resourceValues.totalFileAllocatedSize - else { - continue - } - totalSize += UInt64(fileSize) - } - - return totalSize - } - /// Create a new container from the provided id and configuration. public func create(configuration: ContainerConfiguration, kernel: Kernel, options: ContainerCreateOptions, initImage: String? = nil, runtimeData: Data? = nil) async throws { log.debug( @@ -871,7 +838,7 @@ public actor ContainersService { let containerPath = self.containerRoot.appendingPathComponent(id).path - return Self.calculateDirectorySize(at: containerPath) + return FileManager.default.allocatedSize(of: URL(fileURLWithPath: containerPath)) } public func exportRootfs(id: String, archive: URL) async throws { diff --git a/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift b/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift index a90343bc9..0e27f9eb3 100644 --- a/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift +++ b/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift @@ -158,7 +158,7 @@ public actor VolumesService { } let volumePath = self.volumePath(for: name) - return self.calculateDirectorySize(at: volumePath) + return FileManager.default.allocatedSize(of: URL(fileURLWithPath: volumePath)) } /// Calculate disk usage for volumes @@ -201,7 +201,7 @@ public actor VolumesService { // Calculate sizes for volume in allVolumes { let volumePath = self.volumePath(for: volume.name) - let volumeSize = self.calculateDirectorySize(at: volumePath) + let volumeSize = FileManager.default.allocatedSize(of: URL(fileURLWithPath: volumePath)) totalSize += volumeSize if !inUseSet.contains(volume.name) { @@ -214,33 +214,6 @@ public actor VolumesService { } } - private nonisolated func calculateDirectorySize(at path: String) -> UInt64 { - let url = URL(fileURLWithPath: path) - let fileManager = FileManager.default - - guard - let enumerator = fileManager.enumerator( - at: url, - includingPropertiesForKeys: [.totalFileAllocatedSizeKey], - options: [.skipsHiddenFiles] - ) - else { - return 0 - } - - var totalSize: UInt64 = 0 - for case let fileURL as URL in enumerator { - guard let resourceValues = try? fileURL.resourceValues(forKeys: [.totalFileAllocatedSizeKey]), - let fileSize = resourceValues.totalFileAllocatedSize - else { - continue - } - totalSize += UInt64(fileSize) - } - - return totalSize - } - private func parseSize(_ sizeString: String) throws -> UInt64 { let measurement = try Measurement.parse(parsing: sizeString) let bytes = measurement.converted(to: .bytes).value diff --git a/Sources/Services/ContainerImagesService/Server/ImagesService.swift b/Sources/Services/ContainerImagesService/Server/ImagesService.swift index 2fb06ed1c..418306346 100644 --- a/Sources/Services/ContainerImagesService/Server/ImagesService.swift +++ b/Sources/Services/ContainerImagesService/Server/ImagesService.swift @@ -29,11 +29,13 @@ import TerminalProgress public actor ImagesService { private let log: Logger private let contentStore: ContentStore + private let contentBlobsPath: URL private let imageStore: ImageStore private let snapshotStore: SnapshotStore - public init(contentStore: ContentStore, imageStore: ImageStore, snapshotStore: SnapshotStore, log: Logger) throws { + public init(contentStore: ContentStore, contentBlobsPath: URL, imageStore: ImageStore, snapshotStore: SnapshotStore, log: Logger) throws { self.contentStore = contentStore + self.contentBlobsPath = contentBlobsPath self.imageStore = imageStore self.snapshotStore = snapshotStore self.log = log @@ -270,8 +272,7 @@ public actor ImagesService { /// Calculate disk usage for images /// - Parameter activeReferences: Set of image references currently in use by containers - /// - Returns: Tuple of (total count, active count, total size, reclaimable size) - public func calculateDiskUsage(activeReferences: Set) async throws -> (Int, Int, UInt64, UInt64) { + public func calculateDiskUsage(activeReferences: Set) async throws -> (totalCount: Int, activeCount: Int, totalSize: UInt64, reclaimableSize: UInt64) { self.log.debug( "ImagesService: enter", metadata: [ @@ -290,49 +291,41 @@ public actor ImagesService { } let images = try await self._list() - var totalSize: UInt64 = 0 - var reclaimableSize: UInt64 = 0 var activeCount = 0 + var activeContentSizes: [String: UInt64] = [:] + var activeSnapshotSizes: [String: UInt64] = [:] + var processedDigests = Set() for image in images { - // Calculate size for all platform variants - let imageSize = try await self.calculateImageSize(image) - totalSize += imageSize - - // Check if image is referenced by any container - let isActive = activeReferences.contains(image.reference) - if isActive { - activeCount += 1 - } else { - reclaimableSize += imageSize + guard activeReferences.contains(image.reference) else { continue } + activeCount += 1 + let imageDigest = image.digest.trimmingDigestPrefix + guard processedDigests.insert(imageDigest).inserted else { continue } + + var seen = Set() + for digest in try await image.referencedDigests() where seen.insert(digest).inserted { + guard let content: Content = try await self.contentStore.get(digest: digest) else { continue } + activeContentSizes[digest] = try self.contentDiskSize(content) + } + for (digest, size) in try await self.snapshotStore.getSnapshotSizes(for: image) { + activeSnapshotSizes[digest] = size } } - return (images.count, activeCount, totalSize, reclaimableSize) - } - - /// Calculate total size for an image including all platform variants - private func calculateImageSize(_ image: Containerization.Image) async throws -> UInt64 { - var totalSize: UInt64 = 0 - let index = try await image.index() - - for descriptor in index.manifests { - // Skip attestation manifests - if let refType = descriptor.annotations?["vnd.docker.reference.type"], - refType == "attestation-manifest" - { - continue - } + let snapshotDiskSize = await self.snapshotStore.totalAllocatedSize() + let totalOnDisk = FileManager.default.allocatedSize(of: self.contentBlobsPath) + snapshotDiskSize + let activeSize = activeContentSizes.values.reduce(0, +) + activeSnapshotSizes.values.reduce(0, +) + let reclaimable = totalOnDisk > activeSize ? totalOnDisk - activeSize : 0 - guard descriptor.platform != nil else { continue } + return (images.count, activeCount, totalOnDisk, reclaimable) + } - // Get snapshot size for this platform - if let snapshotSize = try? await self.snapshotStore.getSnapshotSize(descriptor: descriptor) { - totalSize += snapshotSize - } + private func contentDiskSize(_ content: Content) throws -> UInt64 { + let values = try? content.path.resourceValues(forKeys: [.totalFileAllocatedSizeKey]) + if let allocatedSize = values?.totalFileAllocatedSize { + return UInt64(allocatedSize) } - - return totalSize + return try content.size() } } diff --git a/Sources/Services/ContainerImagesService/Server/SnapshotStore.swift b/Sources/Services/ContainerImagesService/Server/SnapshotStore.swift index 282569a9d..643713f13 100644 --- a/Sources/Services/ContainerImagesService/Server/SnapshotStore.swift +++ b/Sources/Services/ContainerImagesService/Server/SnapshotStore.swift @@ -183,7 +183,7 @@ public actor SnapshotStore { guard self.fm.fileExists(atPath: unpackedPath.absolutePath()) else { continue } - deletedBytes += (try? self.fm.directorySize(dir: unpackedPath)) ?? 0 + deletedBytes += self.fm.allocatedSize(of: unpackedPath) try self.fm.removeItem(at: unpackedPath) } return deletedBytes @@ -213,38 +213,28 @@ public actor SnapshotStore { } /// Get the disk size for a specific snapshot descriptor - public func getSnapshotSize(descriptor: Descriptor) throws -> UInt64 { + public func getSnapshotSize(descriptor: Descriptor) -> UInt64 { let snapshotPath = self.snapshotDir(descriptor) guard self.fm.fileExists(atPath: snapshotPath.path) else { return 0 } - return try self.fm.directorySize(dir: snapshotPath) + return self.fm.allocatedSize(of: snapshotPath) } -} - -extension FileManager { - fileprivate func directorySize(dir: URL) throws -> UInt64 { - var size: UInt64 = 0 - let resourceKeys: [URLResourceKey] = [.totalFileAllocatedSizeKey] - guard - let enumerator = self.enumerator( - at: dir, - includingPropertiesForKeys: resourceKeys, - options: [.skipsHiddenFiles] - ) - else { - return 0 + /// Returns (trimmed digest, size) pairs for every unpackable snapshot owned by the image. + public func getSnapshotSizes(for image: Containerization.Image) async throws -> [(digest: String, size: UInt64)] { + var results: [(digest: String, size: UInt64)] = [] + for descriptor in try await image.unpackableDescriptors() { + let size = self.getSnapshotSize(descriptor: descriptor) + guard size > 0 else { continue } + results.append((descriptor.digest.trimmingDigestPrefix, size)) } + return results + } - for case let fileURL as URL in enumerator { - if let resourceValues = try? fileURL.resourceValues(forKeys: [.totalFileAllocatedSizeKey]), - let fileSize = resourceValues.totalFileAllocatedSize - { - size += UInt64(fileSize) - } - } - return size + /// Total allocated bytes across all snapshot storage (including orphans). + public func totalAllocatedSize() -> UInt64 { + self.fm.allocatedSize(of: self.path) } } diff --git a/Tests/CLITests/Subcommands/System/TestCLISystemDF.swift b/Tests/CLITests/Subcommands/System/TestCLISystemDF.swift new file mode 100644 index 000000000..eb184e7c3 --- /dev/null +++ b/Tests/CLITests/Subcommands/System/TestCLISystemDF.swift @@ -0,0 +1,100 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Foundation +import Testing + +@Suite(.serialSuites, .serialized) +final class TestCLISystemDF: CLITest { + private struct DiskUsageStats: Decodable { + let images: ResourceUsage + } + + private struct ResourceUsage: Decodable { + let active: Int + let reclaimable: UInt64 + let sizeInBytes: UInt64 + let total: Int + } + + // Issue #1526: reported image size must include content blobs, not just unpacked snapshots. + @Test func imageDiskUsageIsPopulatedAfterPull() throws { + try withCleanImageStore { + try doPull(imageName: alpine) + let stats = try systemDiskUsage() + #expect(stats.images.total >= 1) + #expect(stats.images.active == 0) + #expect(stats.images.sizeInBytes > 0) + #expect(stats.images.reclaimable == stats.images.sizeInBytes) + } + } + + // Issue #1527: tagging the same image must not double-count its storage. + @Test func tagsDoNotDoubleCountImageStorage() throws { + try withCleanImageStore { + try doPull(imageName: alpine) + let before = try systemDiskUsage() + + try doImageTag(image: alpine, newName: "local/system-df-alpine:tag-one") + try doImageTag(image: alpine, newName: "local/system-df-alpine:tag-two") + let after = try systemDiskUsage() + + #expect(after.images.total == before.images.total + 2) + #expect(after.images.sizeInBytes == before.images.sizeInBytes) + #expect(after.images.reclaimable == before.images.reclaimable) + } + } + + // Issue #1527: removing one of several tags must not free shared storage. + // Assumes no background GC runs between operations; blobs stay until all references are removed. + @Test func deletingOneOfMultipleTagsPreservesSharedStorage() throws { + try withCleanImageStore { + let baseline = try systemDiskUsage() + + try doPull(imageName: alpine) + try doImageTag(image: alpine, newName: "local/system-df-alpine:delete-probe") + let beforeDelete = try systemDiskUsage() + + try doRemoveImages(images: ["local/system-df-alpine:delete-probe"]) + let afterAliasDelete = try systemDiskUsage() + + #expect(afterAliasDelete.images.total == beforeDelete.images.total - 1) + #expect(afterAliasDelete.images.sizeInBytes == beforeDelete.images.sizeInBytes) + #expect(afterAliasDelete.images.reclaimable == beforeDelete.images.reclaimable) + + _ = try? run(arguments: ["image", "rm", "--all"]) + let afterFullClean = try systemDiskUsage() + #expect(afterFullClean.images.total <= baseline.images.total) + #expect(afterFullClean.images.sizeInBytes <= baseline.images.sizeInBytes) + } + } + + private func withCleanImageStore(_ body: () throws -> Void) throws { + _ = try? run(arguments: ["image", "rm", "--all"]) + defer { + _ = try? run(arguments: ["image", "rm", "--all"]) + } + try body() + } + + private func systemDiskUsage() throws -> DiskUsageStats { + let (data, _, error, status) = try run(arguments: ["system", "df", "--format", "json"]) + guard status == 0 else { + throw CLIError.executionFailed("system df failed: \(error)") + } + return try JSONDecoder().decode(DiskUsageStats.self, from: data) + } +}