diff --git a/.github/workflows/packer.yml b/.github/workflows/packer.yml
index f149685751..2a374fef39 100644
--- a/.github/workflows/packer.yml
+++ b/.github/workflows/packer.yml
@@ -40,8 +40,14 @@ jobs:
     - name: Init Packer
       run: packer init orka.pkr.hcl
       working-directory: ansible/packer
+    
+    - name: Run Packer (base image)
+      # Skip arm64 for now as Xcode download takes too long
+      if: matrix.os != 'sonoma-arm64'
+      run: packer build --only=macstadium-orka.${{ matrix.os }} orka-base.pkr.hcl
+      working-directory: ansible/packer
 
-    - name: Run Packer
+    - name: Run Packer (final image)
       run: packer build --only=macstadium-orka.${{ matrix.os }} orka.pkr.hcl
       working-directory: ansible/packer
       env:
diff --git a/ansible/packer/orka-base.pkr.hcl b/ansible/packer/orka-base.pkr.hcl
index 8b53a27a56..07cc903e1a 100644
--- a/ansible/packer/orka-base.pkr.hcl
+++ b/ansible/packer/orka-base.pkr.hcl
@@ -62,7 +62,9 @@ build {
   }
 
   # Pause the provisioner until user interacts (for install Xcode etc)
-  provisioner "breakpoint" {}
+  provisioner "breakpoint" {
+    only = ["macstadium-orka.sonoma-arm64"]
+  }
 
   # Install homebrew and ansible
   provisioner "shell" {
@@ -85,6 +87,8 @@ EOF
 
   # Install Xcode
   provisioner "ansible-local" {
+    # We only install Xcode on the arm64 VM (build tools is enough for the Intel test VMs)
+    only = ["macstadium-orka.sonoma-arm64"]
     playbook_file = "../playbooks/AdoptOpenJDK_Unix_Playbook/main.yml"
     playbook_dir = "../playbooks/AdoptOpenJDK_Unix_Playbook"
     extra_arguments = [
diff --git a/ansible/packer/orka.pkr.hcl b/ansible/packer/orka.pkr.hcl
index fe61ae3823..7d7e46053d 100644
--- a/ansible/packer/orka.pkr.hcl
+++ b/ansible/packer/orka.pkr.hcl
@@ -47,6 +47,8 @@ build {
 
   # Ensure ansible package is up to date
   provisioner "shell" {
+    # Only needed on arm64 as we rebuild intel base frequently
+    only = ["macstadium-orka.sonoma-arm64"]
     inline = [
       "source /Users/admin/.zprofile; brew upgrade ansible",
     ]