From 05c13a03e2fb27186d8d1f2825807bf156512111 Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Fri, 23 Apr 2021 04:15:14 -0400
Subject: [PATCH 1/5] make-disk-image: Get proper size for automatic size
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On some filesystems, `du` without `--apparent-size` will not give the
actual size for a file. Using `--apparent-size` will give us the actual
file size.

Though, this is not actually correct still. 1000 × 1 bytes is not 1000
bytes. It is 1000 × ceil(filesize/blockSize)*blockSize.

So instead of adding up the actual file sizes. We are adding up the
block sizes.

Note that this also changes the builder to work with *bytes*, rather
than with any other units. Doing maths on bytes is less likely to go
awry than doing it on other units.
---
 nixos/lib/make-disk-image.nix | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/nixos/lib/make-disk-image.nix b/nixos/lib/make-disk-image.nix
index 023d0791a5c..164f2f0f0be 100644
--- a/nixos/lib/make-disk-image.nix
+++ b/nixos/lib/make-disk-image.nix
@@ -163,6 +163,8 @@ let format' = format; in let
 
   closureInfo = pkgs.closureInfo { rootPaths = [ config.system.build.toplevel channelSources ]; };
 
+  blockSize = toString (4 * 1024); # ext4fs block size (not block device sector size)
+
   prepareImage = ''
     export PATH=${binPath}
 
@@ -175,6 +177,15 @@ let format' = format; in let
       echo $(( "$1" * 512  ))
     }
 
+    # Given lines of numbers, adds them together
+    sum_lines() {
+      local acc=0
+      while read -r number; do
+        acc=$((acc+number))
+      done
+      echo "$acc"
+    }
+
     mkdir $out
 
     root="$PWD/root"
@@ -235,12 +246,23 @@ let format' = format; in let
 
     ${if diskSize == "auto" then ''
       ${if partitionTableType == "efi" || partitionTableType == "hybrid" then ''
-        additionalSpace=$(( ($(numfmt --from=iec '${additionalSpace}') + $(numfmt --from=iec '${bootSize}')) / 1000 ))
+        additionalSpace=$(( ($(numfmt --from=iec '${additionalSpace}') + $(numfmt --from=iec '${bootSize}')) ))
       '' else ''
-        additionalSpace=$(( $(numfmt --from=iec '${additionalSpace}') / 1000 ))
+        additionalSpace=$(( $(numfmt --from=iec '${additionalSpace}') ))
       ''}
-      diskSize=$(( $(set -- $(du -d0 $root); echo "$1") + $additionalSpace ))
-      truncate -s "$diskSize"K $diskImage
+
+      # Compute required space in filesystem blocks
+      requiredSpace=$(find . ! -type d -exec 'du' '--apparent-size' '--block-size' "${blockSize}" '{}' ';' | cut -f1 | sum_lines)
+      # Convert to bytes
+      requiredSpace=$(( requiredSpace * ${blockSize} ))
+
+      diskSize=$(( requiredSpace  + additionalSpace ))
+      truncate -s "$diskSize" $diskImage
+
+      printf "Automatic disk size...\n"
+      printf "  Space needed: %d bytes\n" $requiredSpace
+      printf "  Additional space: %d bytes\n" $additionalSpace
+      printf "  Disk image size: %d bytes\n" $diskSize
     '' else ''
       truncate -s ${toString diskSize}M $diskImage
     ''}
@@ -251,9 +273,9 @@ let format' = format; in let
       # Get start & length of the root partition in sectors to $START and $SECTORS.
       eval $(partx $diskImage -o START,SECTORS --nr ${rootPartition} --pairs)
 
-      mkfs.${fsType} -F -L ${label} $diskImage -E offset=$(sectorsToBytes $START) $(sectorsToKilobytes $SECTORS)K
+      mkfs.${fsType} -b ${blockSize} -F -L ${label} $diskImage -E offset=$(sectorsToBytes $START) $(sectorsToKilobytes $SECTORS)K
     '' else ''
-      mkfs.${fsType} -F -L ${label} $diskImage
+      mkfs.${fsType} -b ${blockSize} -F -L ${label} $diskImage
     ''}
 
     echo "copying staging root to image..."

From 9b18a78c739a2a43c3ed386a4a9c52e4f7360650 Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Fri, 23 Apr 2021 23:18:08 -0400
Subject: [PATCH 2/5] make-disk-image: Account for the ext4 reserved space

Reserved space includes:

 - inodes space in use (2 blocks per)
 - about 5.2% of the space

The 5.2% reserved space was computed empirically when working on a
previous EXT4 image builder. It seems to stabilize around 5% even for
much larger filesystems.
---
 nixos/lib/make-disk-image.nix | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/nixos/lib/make-disk-image.nix b/nixos/lib/make-disk-image.nix
index 164f2f0f0be..f1c942181f0 100644
--- a/nixos/lib/make-disk-image.nix
+++ b/nixos/lib/make-disk-image.nix
@@ -186,6 +186,13 @@ let format' = format; in let
       echo "$acc"
     }
 
+    # Approximative percentage of reserved space in an ext4 fs over 512MiB.
+    # 0.05208587646484375
+    #  × 1000, integer part: 52
+    compute_fudge() {
+      echo $(( $1 * 52 / 1000 ))
+    }
+
     mkdir $out
 
     root="$PWD/root"
@@ -252,15 +259,22 @@ let format' = format; in let
       ''}
 
       # Compute required space in filesystem blocks
-      requiredSpace=$(find . ! -type d -exec 'du' '--apparent-size' '--block-size' "${blockSize}" '{}' ';' | cut -f1 | sum_lines)
-      # Convert to bytes
-      requiredSpace=$(( requiredSpace * ${blockSize} ))
+      diskUsage=$(find . ! -type d -exec 'du' '--apparent-size' '--block-size' "${blockSize}" '{}' ';' | cut -f1 | sum_lines)
+      # Each inode takes space!
+      numInodes=$(find . | wc -l)
+      # Convert to bytes, inodes take two blocks each!
+      diskUsage=$(( (diskUsage + 2 * numInodes) * ${blockSize} ))
+      # Then increase the required space to account for the reserved blocks.
+      fudge=$(compute_fudge $diskUsage)
+      requiredFilesystemSpace=$(( diskUsage + fudge ))
 
-      diskSize=$(( requiredSpace  + additionalSpace ))
+      diskSize=$(( requiredFilesystemSpace  + additionalSpace ))
       truncate -s "$diskSize" $diskImage
 
       printf "Automatic disk size...\n"
-      printf "  Space needed: %d bytes\n" $requiredSpace
+      printf "  Closure space use: %d bytes\n" $diskUsage
+      printf "  fudge: %d bytes\n" $fudge
+      printf "  Filesystem size needed: %d bytes\n" $requiredFilesystemSpace
       printf "  Additional space: %d bytes\n" $additionalSpace
       printf "  Disk image size: %d bytes\n" $diskSize
     '' else ''

From ba666011a64aee55c78c9cb4b0e7c272479d9a2e Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Fri, 23 Apr 2021 23:20:49 -0400
Subject: [PATCH 3/5] make-disk-image: Account for reserved disk space

This is a bit of a thorny issue. See, the actual `diskSize` variable is
for the *total* disk size, not for the filesystem!

The automatic numbers are meant to compute the *filesystem* required
space. So we have to add any other reserved space!

We have different requirements for reserved space. E.g. there could be
none (when it's actually a filesystem image). There could also be 1MiB
for alignment for an MBR image, legacy+gpt needs 2MiB, then GPT with an
ESP ("bootSize") needs to take the boot partition and GPT size into
account too!

Though luckily(?) for this latter situation we can cheat! As noted in the
change, `bootSize` is NOT the boot partition size. It is actually the
offset where the target filesystem starts.
---
 nixos/lib/make-disk-image.nix | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/nixos/lib/make-disk-image.nix b/nixos/lib/make-disk-image.nix
index f1c942181f0..26a591646e2 100644
--- a/nixos/lib/make-disk-image.nix
+++ b/nixos/lib/make-disk-image.nix
@@ -15,6 +15,8 @@
 
 , # size of the boot partition, is only used if partitionTableType is
   # either "efi" or "hybrid"
+  # This will be undersized slightly, as this is actually the offset of
+  # the end of the partition. Generally it will be 1MiB smaller.
   bootSize ? "256M"
 
 , # The files and directories to be placed in the target file system.
@@ -253,10 +255,25 @@ let format' = format; in let
 
     ${if diskSize == "auto" then ''
       ${if partitionTableType == "efi" || partitionTableType == "hybrid" then ''
-        additionalSpace=$(( ($(numfmt --from=iec '${additionalSpace}') + $(numfmt --from=iec '${bootSize}')) ))
+        # Add the GPT at the end
+        gptSpace=$(( 512 * 34 * 1 ))
+        # Normally we'd need to account for alignment and things, if bootSize
+        # represented the actual size of the boot partition. But it instead
+        # represents the offset at which it ends.
+        # So we know bootSize is the reserved space in front of the partition.
+        reservedSpace=$(( gptSpace + $(numfmt --from=iec '${bootSize}') ))
+      '' else if partitionTableType == "legacy+gpt" then ''
+        # Add the GPT at the end
+        gptSpace=$(( 512 * 34 * 1 ))
+        # And include the bios_grub partition; the ext4 partition starts at 2MB exactly.
+        reservedSpace=$(( gptSpace + 2 * 1024*1024 ))
+      '' else if partitionTableType == "legacy" then ''
+        # Add the 1MiB aligned reserved space (includes MBR)
+        reservedSpace=$(( 1024*1024 ))
       '' else ''
-        additionalSpace=$(( $(numfmt --from=iec '${additionalSpace}') ))
+        reservedSpace=0
       ''}
+      additionalSpace=$(( $(numfmt --from=iec '${additionalSpace}') + reservedSpace ))
 
       # Compute required space in filesystem blocks
       diskUsage=$(find . ! -type d -exec 'du' '--apparent-size' '--block-size' "${blockSize}" '{}' ';' | cut -f1 | sum_lines)

From 5aa4273e4f5e0ea7d0abd61d57c1c2301e284f23 Mon Sep 17 00:00:00 2001
From: lassulus <lassulus@lassul.us>
Date: Sat, 19 Dec 2020 19:41:11 +0100
Subject: [PATCH 4/5] treewide: use auto diskSize for make-disk-image

(cherry picked from commit f3aa040bcbf39935e7e9ac7a7296eac9da7623ec)
---
 nixos/maintainers/scripts/cloudstack/cloudstack-image.nix | 1 -
 nixos/maintainers/scripts/ec2/amazon-image.nix            | 5 +++--
 nixos/maintainers/scripts/openstack/openstack-image.nix   | 2 +-
 nixos/modules/virtualisation/azure-image.nix              | 5 +++--
 nixos/modules/virtualisation/digital-ocean-image.nix      | 5 +++--
 nixos/modules/virtualisation/google-compute-image.nix     | 5 +++--
 nixos/modules/virtualisation/hyperv-image.nix             | 5 +++--
 nixos/modules/virtualisation/virtualbox-image.nix         | 5 +++--
 nixos/modules/virtualisation/vmware-image.nix             | 5 +++--
 9 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/nixos/maintainers/scripts/cloudstack/cloudstack-image.nix b/nixos/maintainers/scripts/cloudstack/cloudstack-image.nix
index 37b46db059c..005f75476e9 100644
--- a/nixos/maintainers/scripts/cloudstack/cloudstack-image.nix
+++ b/nixos/maintainers/scripts/cloudstack/cloudstack-image.nix
@@ -10,7 +10,6 @@ with lib;
 
   system.build.cloudstackImage = import ../../../lib/make-disk-image.nix {
     inherit lib config pkgs;
-    diskSize = 8192;
     format = "qcow2";
     configFile = pkgs.writeText "configuration.nix"
       ''
diff --git a/nixos/maintainers/scripts/ec2/amazon-image.nix b/nixos/maintainers/scripts/ec2/amazon-image.nix
index 0ecf07669a1..653744986d1 100644
--- a/nixos/maintainers/scripts/ec2/amazon-image.nix
+++ b/nixos/maintainers/scripts/ec2/amazon-image.nix
@@ -40,8 +40,9 @@ in {
     };
 
     sizeMB = mkOption {
-      type = types.int;
-      default = if config.ec2.hvm then 2048 else 8192;
+      type = with types; either (enum [ "auto" ]) int;
+      default = "auto";
+      example = 8192;
       description = "The size in MB of the image";
     };
 
diff --git a/nixos/maintainers/scripts/openstack/openstack-image.nix b/nixos/maintainers/scripts/openstack/openstack-image.nix
index 4c464f43f61..3255e7f3d44 100644
--- a/nixos/maintainers/scripts/openstack/openstack-image.nix
+++ b/nixos/maintainers/scripts/openstack/openstack-image.nix
@@ -12,8 +12,8 @@ with lib;
 
   system.build.openstackImage = import ../../../lib/make-disk-image.nix {
     inherit lib config;
+    additionalSpace = "1024M";
     pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
-    diskSize = 8192;
     format = "qcow2";
     configFile = pkgs.writeText "configuration.nix"
       ''
diff --git a/nixos/modules/virtualisation/azure-image.nix b/nixos/modules/virtualisation/azure-image.nix
index 60fed3222ef..03dd3c05130 100644
--- a/nixos/modules/virtualisation/azure-image.nix
+++ b/nixos/modules/virtualisation/azure-image.nix
@@ -9,8 +9,9 @@ in
 
   options = {
     virtualisation.azureImage.diskSize = mkOption {
-      type = with types; int;
-      default = 2048;
+      type = with types; either (enum [ "auto" ]) int;
+      default = "auto";
+      example = 2048;
       description = ''
         Size of disk image. Unit is MB.
       '';
diff --git a/nixos/modules/virtualisation/digital-ocean-image.nix b/nixos/modules/virtualisation/digital-ocean-image.nix
index b582e235d43..0ff2ee591f2 100644
--- a/nixos/modules/virtualisation/digital-ocean-image.nix
+++ b/nixos/modules/virtualisation/digital-ocean-image.nix
@@ -10,8 +10,9 @@ in
 
   options = {
     virtualisation.digitalOceanImage.diskSize = mkOption {
-      type = with types; int;
-      default = 4096;
+      type = with types; either (enum [ "auto" ]) int;
+      default = "auto";
+      example = 4096;
       description = ''
         Size of disk image. Unit is MB.
       '';
diff --git a/nixos/modules/virtualisation/google-compute-image.nix b/nixos/modules/virtualisation/google-compute-image.nix
index e2332df611a..79c3921669e 100644
--- a/nixos/modules/virtualisation/google-compute-image.nix
+++ b/nixos/modules/virtualisation/google-compute-image.nix
@@ -18,8 +18,9 @@ in
 
   options = {
     virtualisation.googleComputeImage.diskSize = mkOption {
-      type = with types; int;
-      default = 1536;
+      type = with types; either (enum [ "auto" ]) int;
+      default = "auto";
+      example = 1536;
       description = ''
         Size of disk image. Unit is MB.
       '';
diff --git a/nixos/modules/virtualisation/hyperv-image.nix b/nixos/modules/virtualisation/hyperv-image.nix
index fabc9113dfc..6845d675009 100644
--- a/nixos/modules/virtualisation/hyperv-image.nix
+++ b/nixos/modules/virtualisation/hyperv-image.nix
@@ -9,8 +9,9 @@ in {
   options = {
     hyperv = {
       baseImageSize = mkOption {
-        type = types.int;
-        default = 2048;
+        type = with types; either (enum [ "auto" ]) int;
+        default = "auto";
+        example = 2048;
         description = ''
           The size of the hyper-v base image in MiB.
         '';
diff --git a/nixos/modules/virtualisation/virtualbox-image.nix b/nixos/modules/virtualisation/virtualbox-image.nix
index fa580e8b42d..071edda8269 100644
--- a/nixos/modules/virtualisation/virtualbox-image.nix
+++ b/nixos/modules/virtualisation/virtualbox-image.nix
@@ -11,8 +11,9 @@ in {
   options = {
     virtualbox = {
       baseImageSize = mkOption {
-        type = types.int;
-        default = 50 * 1024;
+        type = with types; either (enum [ "auto" ]) int;
+        default = "auto";
+        example = 50 * 1024;
         description = ''
           The size of the VirtualBox base image in MiB.
         '';
diff --git a/nixos/modules/virtualisation/vmware-image.nix b/nixos/modules/virtualisation/vmware-image.nix
index 9da9e145f7a..f6cd12e2bb7 100644
--- a/nixos/modules/virtualisation/vmware-image.nix
+++ b/nixos/modules/virtualisation/vmware-image.nix
@@ -18,8 +18,9 @@ in {
   options = {
     vmware = {
       baseImageSize = mkOption {
-        type = types.int;
-        default = 2048;
+        type = with types; either (enum [ "auto" ]) int;
+        default = "auto";
+        example = 2048;
         description = ''
           The size of the VMWare base image in MiB.
         '';

From 7b8b3fab6dce5357c14fc435b436514fc985f0f7 Mon Sep 17 00:00:00 2001
From: Samuel Dionne-Riel <samuel@dionne-riel.com>
Date: Sun, 25 Apr 2021 15:24:45 -0400
Subject: [PATCH 5/5] make-disk-image: Round image size to the next mebibyte

This ensures the following gptfdisk warning won't happen:

```
Warning: File size is not a multiple of 512 bytes! Misbehavior is likely!
```

Additionally, helps towards aligning the partition to be more optimal
for the underlying storage.

It is actually impossible to align for the actual underlying storage
optimally because we don't know what the block device will be!

But aligning on 1MiB should help.
---
 nixos/lib/make-disk-image.nix | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/nixos/lib/make-disk-image.nix b/nixos/lib/make-disk-image.nix
index 26a591646e2..d87b3ef8da0 100644
--- a/nixos/lib/make-disk-image.nix
+++ b/nixos/lib/make-disk-image.nix
@@ -188,6 +188,8 @@ let format' = format; in let
       echo "$acc"
     }
 
+    mebibyte=$(( 1024 * 1024 ))
+
     # Approximative percentage of reserved space in an ext4 fs over 512MiB.
     # 0.05208587646484375
     #  × 1000, integer part: 52
@@ -266,10 +268,10 @@ let format' = format; in let
         # Add the GPT at the end
         gptSpace=$(( 512 * 34 * 1 ))
         # And include the bios_grub partition; the ext4 partition starts at 2MB exactly.
-        reservedSpace=$(( gptSpace + 2 * 1024*1024 ))
+        reservedSpace=$(( gptSpace + 2 * mebibyte ))
       '' else if partitionTableType == "legacy" then ''
         # Add the 1MiB aligned reserved space (includes MBR)
-        reservedSpace=$(( 1024*1024 ))
+        reservedSpace=$(( mebibyte ))
       '' else ''
         reservedSpace=0
       ''}
@@ -286,6 +288,14 @@ let format' = format; in let
       requiredFilesystemSpace=$(( diskUsage + fudge ))
 
       diskSize=$(( requiredFilesystemSpace  + additionalSpace ))
+
+      # Round up to the nearest mebibyte.
+      # This ensures whole 512 bytes sector sizes in the disk image
+      # and helps towards aligning partitions optimally.
+      if (( diskSize % mebibyte )); then
+        diskSize=$(( ( diskSize / mebibyte + 1) * mebibyte ))
+      fi
+
       truncate -s "$diskSize" $diskImage
 
       printf "Automatic disk size...\n"