Refactor Docker image handling by replacing ArchiveService with LayerProcessor for layer merging and extraction

bostrot · bostrot · commit 2d066f7f1130 · 2025-12-11T01:14:10.000+01:00
diff --git a/lib/api/docker_images.dart b/lib/api/docker_images.dart
@@ -7,7 +7,7 @@ import 'package:chunked_downloader/chunked_downloader.dart';
 import 'package:dio/dio.dart';
 import 'package:flutter/foundation.dart';
 import 'package:localization/localization.dart';
-import 'package:wsl2distromanager/api/archive.dart';
+import 'package:wsl2distromanager/api/layer_processor.dart';
 import 'package:wsl2distromanager/api/safe_paths.dart';
 import 'package:wsl2distromanager/components/helpers.dart';
 import 'package:wsl2distromanager/components/logging.dart';
@@ -145,12 +145,10 @@ class DockerImage {
   String? distroName;
   final Dio dio;
   final ChunkedDownloaderFactory chunkedDownloaderFactory;
-  final ArchiveService archiveService;
 
   DockerImage(
       {Dio? dio,
       ChunkedDownloaderFactory? chunkedDownloaderFactory,
-      ArchiveService? archiveService,
       String? registryUrl,
       this.authUrl = 'https://auth.docker.io/token',
       this.svcUrl = 'registry.docker.io'})
@@ -174,8 +172,7 @@ class DockerImage {
                     chunkSize: chunkSize ?? 1024 * 1024,
                     onProgress: onProgress,
                     onDone: onDone,
-                    onError: onError)),
-        archiveService = archiveService ?? ArchiveService() {
+                    onError: onError)) {
     String? mirror = prefs.getString('DockerMirror');
     if (mirror != null && mirror.isNotEmpty) {
       this.registryUrl = mirror;
@@ -349,8 +346,85 @@ class DockerImage {
             ImageManifest.fromMap(await _getManifest(image, token, digest));
 
         final config = imageManifest.config.digest;
-        await _downloadBlob(image, token, config,
-            SafePath(path).file('config.json'), (p0, p1) {});
+        final configPath = SafePath(path).file('config.json');
+        await _downloadBlob(image, token, config, configPath, (p0, p1) {});
+
+        // Parse config.json for metadata (V2)
+        try {
+          if (await File(configPath).exists()) {
+            final configContent = await File(configPath).readAsString();
+            final configJson = json.decode(configContent);
+
+            if (configJson['config'] != null) {
+              final parsedConfig = configJson['config'];
+              final env = parsedConfig['Env'];
+              final cmd = parsedConfig['Cmd'];
+              final entrypoint = parsedConfig['Entrypoint'];
+              final user = parsedConfig['User'];
+
+              // Handle User
+              if (user != null && user is String && user.isNotEmpty) {
+                var userStr = user;
+                if (userStr.contains(':')) {
+                  userStr = userStr.split(':')[0];
+                }
+                if (int.tryParse(userStr) == null) {
+                  prefs.setString('StartUser_$distroName', userStr);
+                } else {
+                  Notify.message(
+                      'Not implemented yet: Docker USER is a number.');
+                }
+              }
+
+              // Handle Env, Entrypoint, Cmd
+              var entrypointCmd = '';
+              if (entrypoint != null && entrypoint is List) {
+                entrypointCmd = entrypoint.map((e) => e.toString()).join(' ');
+              }
+
+              String exportEnv = '';
+              if (env != null && env is List) {
+                exportEnv = env.map((e) => 'export $e;').join(' ');
+              }
+
+              // Use distroName for StartCmd so it applies to the instance
+              if (cmd != null && cmd is List) {
+                prefs.setString('StartCmd_$distroName',
+                    '$exportEnv $entrypointCmd; ${cmd.map((e) => e.toString()).join(' ')}');
+              } else if (entrypointCmd.isNotEmpty) {
+                prefs.setString(
+                    'StartCmd_$distroName', '$exportEnv $entrypointCmd');
+              }
+            }
+
+            // Handle history for UserCmds/GroupCmds
+            // These need to be saved under image filename to be picked up by create_dialog
+            String imageName = filename(image, tag);
+            List<String> userCmds = [];
+            List<String> groupCmds = [];
+
+            if (configJson['history'] != null &&
+                configJson['history'] is List) {
+              for (var item in configJson['history']) {
+                final createdBy = item['created_by'] as String?;
+                if (createdBy != null) {
+                  if (createdBy.contains('adduser') ||
+                      createdBy.contains('useradd')) {
+                    userCmds.add(createdBy);
+                  }
+                  if (createdBy.contains('groupadd') ||
+                      createdBy.contains('addgroup')) {
+                    groupCmds.add(createdBy);
+                  }
+                }
+              }
+            }
+            prefs.setStringList('UserCmds_$imageName', userCmds);
+            prefs.setStringList('GroupCmds_$imageName', groupCmds);
+          }
+        } catch (e, stack) {
+          logDebug(e, stack, 'Failed to parse V2 config');
+        }
       } catch (e, stackTrace) {
         if (kDebugMode) {
           print(e);
@@ -456,8 +530,8 @@ class DockerImage {
       // Set image specific commands
       String name = filename(image, tag);
       if (cmd != null) {
-        prefs.setString(
-            'StartCmd_$name', '$exportEnv $entrypointCmd; ${cmd.join(' ')}');
+        prefs.setString('StartCmd_$distroName',
+            '$exportEnv $entrypointCmd; ${cmd.join(' ')}');
       }
       prefs.setStringList('UserCmds_$name', userCmds);
       prefs.setStringList('GroupCmds_$name', groupCmds);
@@ -570,38 +644,17 @@ class DockerImage {
     int retry = 0;
 
     final parentPath = SafePath(tmpImagePath);
-    String outTar = parentPath.file('$imageName.tar');
     String outTarGz = SafePath(distroPath).file('$imageName.tar.gz');
     while (retry < 2) {
       try {
-        // More than one layer
-        List<String> paths = [];
-        if (layers != 1) {
-          for (var i = 0; i < layers; i++) {
-            // Read archives layers
-            if (kDebugMode) {
-              print('Extracting layer $i of $layers');
-            }
-            // progress(i, layers, -1, -1);
-            Notify.message('Extracting layer $i of $layers');
-
-            // Extract layer
-            final layerTarGz = parentPath.file('layer_$i.tar.gz');
-            await archiveService.extract(layerTarGz, parentPath.path);
-            paths.add(parentPath.file('layer_$i.tar'));
-          }
-
-          // Archive as tar then gzip to disk
-          await archiveService.merge(paths, outTar);
-          await archiveService.compress(outTar, outTarGz);
-
-          Notify.message('writingtodisk-text'.i18n());
-        } else if (layers == 1) {
-          // Just copy the file
-          File(SafePath(tmpImagePath).file('layer_0.tar.gz'))
-              .copySync(outTarGz);
+        List<String> layerPaths = [];
+        for (var i = 0; i < layers; i++) {
+          layerPaths.add(parentPath.file('layer_$i.tar.gz'));
         }
 
+        await LayerProcessor()
+            .mergeLayers(layerPaths, outTarGz, (msg) => Notify.message(msg));
+
         retry = 2;
         break;
       } catch (e, stackTrace) {
diff --git a/lib/api/layer_processor.dart b/lib/api/layer_processor.dart
@@ -0,0 +1,113 @@
+import 'dart:async';
+import 'dart:io';
+import 'package:tar/tar.dart';
+import 'package:path/path.dart' as p;
+
+class LayerProcessor {
+  /// Merges Docker layers into a single rootfs tarball, handling whiteouts.
+  /// [layerPaths] is a list of paths to .tar.gz layer files, ordered from bottom (base) to top.
+  /// [outputPath] is the path where the resulting .tar.gz will be written.
+  Future<void> mergeLayers(List<String> layerPaths, String outputPath,
+      Function(String) onStatus) async {
+    final keptFiles = <String, int>{}; // Path -> Layer Index
+
+    onStatus('Scanning layers...');
+    // Pass 1: Determine which files to keep
+    for (var i = 0; i < layerPaths.length; i++) {
+      onStatus('Scanning layer ${i + 1}/${layerPaths.length}...');
+      final layerPath = layerPaths[i];
+      final file = File(layerPath);
+      if (!await file.exists()) continue;
+
+      final reader = TarReader(file.openRead().transform(gzip.decoder));
+      try {
+        while (await reader.moveNext()) {
+          final entry = reader.current;
+          final path = entry.name;
+          final filename = p.posix.basename(path);
+          final dirname = p.posix.dirname(path);
+
+          if (filename.startsWith('.wh.')) {
+            if (filename == '.wh..wh..opq') {
+              // Opaque whiteout: hide all siblings in this directory from lower layers
+              keptFiles.removeWhere((key, value) {
+                final keyDir = p.posix.dirname(key);
+                return keyDir == dirname && value < i;
+              });
+            } else {
+              // Explicit whiteout
+              final realFilename = filename.substring(4);
+              final realPath =
+                  p.posix.normalize(p.posix.join(dirname, realFilename));
+              keptFiles.remove(realPath);
+            }
+          } else {
+            // Normal file/directory
+            final normalizedPath = p.posix.normalize(path);
+            keptFiles[normalizedPath] = i;
+          }
+        }
+      } finally {
+        await reader.cancel();
+      }
+    }
+
+    onStatus('Writing rootfs...');
+    // Pass 2: Write the output tar
+    final outFile = File(outputPath);
+    if (await outFile.exists()) {
+      await outFile.delete();
+    }
+
+    Stream<TarEntry> streamLayers() async* {
+      for (var i = 0; i < layerPaths.length; i++) {
+        onStatus('Merging layer ${i + 1}/${layerPaths.length}...');
+        final layerPath = layerPaths[i];
+        final file = File(layerPath);
+        if (!await file.exists()) continue;
+
+        final reader = TarReader(file.openRead().transform(gzip.decoder));
+        try {
+          while (await reader.moveNext()) {
+            final entry = reader.current;
+            final path = entry.name;
+            final filename = p.posix.basename(path);
+
+            // Skip whiteout files themselves in the output
+            if (filename.startsWith('.wh.')) {
+              continue;
+            }
+
+            final normalizedPath = p.posix.normalize(path);
+
+            if (keptFiles[normalizedPath] == i) {
+              // We need to wait for the entry to be fully consumed by the writer
+              // before moving the reader to the next entry.
+              final completer = Completer();
+              final trackedStream = entry.contents.transform(
+                  StreamTransformer<List<int>, List<int>>.fromHandlers(
+                handleData: (data, sink) => sink.add(data),
+                handleError: (error, stack, sink) =>
+                    sink.addError(error, stack),
+                handleDone: (sink) {
+                  sink.close();
+                  completer.complete();
+                },
+              ));
+
+              yield TarEntry(entry.header, trackedStream);
+              await completer.future;
+            }
+          }
+        } finally {
+          await reader.cancel();
+        }
+      }
+    }
+
+    await streamLayers()
+        .transform(tarWriter)
+        .transform(gzip.encoder)
+        .pipe(outFile.openWrite());
+  }
+}
diff --git a/pubspec.lock b/pubspec.lock
@@ -947,7 +947,7 @@ packages:
     source: hosted
     version: "0.0.3"
   tar:
-    dependency: transitive
+    dependency: "direct main"
     description:
       name: tar
       sha256: b338bacfd24dae6cf527acb4242003a71fc88ce183a9002376fabbc4ebda30c9
diff --git a/pubspec.yaml b/pubspec.yaml
@@ -36,6 +36,7 @@ dependencies:
   shelf_static: ^1.1.3
   system_info2: ^4.1.0
   system_theme: ^3.1.2
+  tar: ^2.0.2
   url_launcher: ^6.3.2
   window_manager: ^0.5.1
   yaml: ^3.1.3
diff --git a/test/docker_images_unit_test.dart b/test/docker_images_unit_test.dart
@@ -125,7 +125,6 @@ void main() {
               onProgress: onProgress,
               onDone: onDone,
               onError: onError),
-      archiveService: MockArchiveService(),
     );
   });
 
@@ -218,7 +217,6 @@ void main() {
               onProgress: onProgress,
               onDone: onDone,
               onError: onError),
-      archiveService: MockArchiveService(),
     );
 
     // Mock token for custom registry (if needed, or just standard auth url if not changed)
diff --git a/test/layer_processor_test.dart b/test/layer_processor_test.dart
