caches: generate outputs in CI like other sections

container/Dockerfile

@@ -2,7 +2,7 @@ FROM ghcr.io/spack/tutorial-ubuntu-26.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-# Install AWS cli
+# Install extra tools used by the tutorial
 RUN apt-get update -y && \
     apt-get install -y --no-install-recommends \
     bash-completion \
@@ -11,7 +11,8 @@ RUN apt-get update -y && \
     jq \
     less \
     vim \
-    rclone
+    rclone \
+    docker.io
 
 # Download the buildcache
 RUN mkdir /mirror

outputs/Makefile

@@ -4,6 +4,13 @@
 DOCKER := docker
 DOCKER_RUN_OPTS :=
 
+# Path to the host Docker socket and its group id. The cache section talks to a
+# real Docker daemon (see run-cache), so its container needs access to the
+# socket. The socket is owned by a group we must join with --group-add; detect
+# its gid portably across Linux (stat -c) and BSD/macOS (stat -f).
+DOCKER_SOCK := /var/run/docker.sock
+DOCKER_SOCK_GID := $(shell stat -c '%g' $(DOCKER_SOCK) 2>/dev/null || stat -f '%g' $(DOCKER_SOCK) 2>/dev/null)
+
 # Use this for Podman
 #DOCKER := podman
 #DOCKER_RUN_OPTS := --userns=keep-id --security-opt label=disable
@@ -29,35 +36,58 @@ update-outputs: run
 	echo "Filtering raw outputs though col"
 	for raw in raw/*/*.out; do \
 		out=$$(echo $$raw | sed 's.raw/..'); \
-		cat $$raw | perl -pe 's/\x1b]0;.+?\x07//g' | perl -pe 's/\x1b\[\d+F\x1b\[J//g' | perl -pe 's/\033\[([01];)?\d+m//g' | col -bp | sed '/^==> Waiting for/d' > $$out; \
+		cat $$raw | \
+			perl -pe 's/\x1b]0;.+?\x07//g' | \
+			perl -pe 's/\x1b\[\d+F\x1b\[J//g' | \
+			perl -pe 's/\033\[([01];)?\d+m//g' | \
+			col -bp | \
+			sed '/^==> Waiting for/d' > $$out; \
 	done
 
 run: run-scripting
-run-scripting: run-dev scripting.sh
-# run-cache: run-dev cache.sh
+run-scripting: run-cache scripting.sh
+#run-cache is defined below; it needs an extra docker arguments
 run-dev: run-packaging dev.sh
 run-packaging: run-stacks packaging.sh
 run-stacks: run-environments stacks.sh
 run-environments: run-basics environments.sh
 run-basics: basics.sh init_spack.sh defs.sh
 
 local: local-scripting
-local-scripting: local-dev scripting.sh
-# local-cache: local-dev cache.sh
+local-scripting: local-cache scripting.sh
+local-cache: local-dev cache.sh
 local-dev: local-packaging dev.sh
-local-packaging: local-environments packaging.sh
-# local-stacks: local-environments stacks.sh
+local-packaging: local-stacks packaging.sh
+local-stacks: local-environments stacks.sh
 local-environments: local-basics environments.sh
 local-basics: basics.sh init_spack.sh defs.sh
 
 local-%: %.sh init_spack.sh defs.sh
 	$(CURDIR)/$(@:local-%=%).sh
 
+# Common container invocation used by all run-<section> targets.
+#   $(1) = script basename (e.g. environments)
+#   $(2) = extra docker run arguments for this section (may be empty)
+define run_section
+$(DOCKER) run $(DOCKER_RUN_OPTS) --rm -t $(2) \
+	--mount type=bind,source=$(CURDIR),target=/project \
+	${container} \
+	/project/$(1).sh
+endef
+
 run-%: %.sh init_spack.sh defs.sh
-	$(DOCKER) run $(DOCKER_RUN_OPTS) --rm -t \
-		--mount type=bind,source=$(CURDIR),target=/project \
-		${container} \
-		/project/$(@:run-%=%).sh && touch $@
+	$(call run_section,$(@:run-%=%),) && touch $@
+
+# The cache section pushes to an OCI registry with 'spack buildcache' and runs
+# real 'docker' commands so the generated output is an honest integration test.
+# The host Docker socket is mounted in so those commands reach a real daemon,
+# and --add-host maps localhost:5000 (used by spack and by the sibling
+# containers' published port) to the host gateway. cache.sh starts the
+# registry container and cleans it up on exit, including leftovers from a
+# previously interrupted run.
+run-cache: run-dev cache.sh init_spack.sh defs.sh
+	$(call run_section,cache,-v $(DOCKER_SOCK):$(DOCKER_SOCK) \
+        --group-add $(DOCKER_SOCK_GID) --add-host localhost:host-gateway) && touch $@
 
 interactive:
 	$(DOCKER) run $(DOCKER_RUN_OPTS) --rm -it \

outputs/cache.sh

@@ -7,64 +7,65 @@ project="$(dirname "$0")"
 rm -rf "${raw_outputs:?}/cache"
 . "$project/init_spack.sh"
 
-example cache/mirror-list-0 "spack mirror list"
+export SPACK_COLOR=never
 
-example cache/setup-scr "cd ~"
-cd ~ || exit
-example cache/setup-scr "spack env create -d cache-env"
-example cache/setup-scr "cd cache-env"
-cd cache-env || exit
-fake_example cache/setup-scr "spacktivate ." "spack env activate ."
-spack env activate .
-example cache/setup-scr "# for now, disable fortran support in all packages"
-example cache/setup-scr 'spack config add "packages:all:variants: ~fortran"'
-example cache/setup-scr "spack add macsio+scr"
+# Clean up state from a previous run. A real Docker daemon is available via the
+# socket mounted by the Makefile, so docker steps below run for real.
+rm -rf ~/myenv
+docker rm -f registry >/dev/null 2>&1 || true
 
-# The packages will already be installed by the dev tutorial
-spack install
-rm spack.lock
-example cache/setup-scr "spack install"
+# Ensure the registry is torn down when the script exits for any reason.
+trap 'docker rm -f registry >/dev/null 2>&1 || true' EXIT
 
-example cache/spack-mirror-single "spack mirror create -d ~/mirror scr"
+# Installing julia from the build cache
+example cache/install-julia "mkdir ~/myenv && cd ~/myenv"
+mkdir -p ~/myenv
+cd ~/myenv || exit
+example cache/install-julia "spack env create --with-view view ."
+example cache/install-julia "spack -e . add julia"
 
-example cache/spack-mirror-config "spack mirror add mymirror ~/mirror"
+example --tee cache/install "spack -e . install"
 
-example cache/spack-mirror-all "spack mirror create -d ~/mirror --all"
+example cache/julia-run "./view/bin/julia -e 'println(1 + 1)'"
 
-example cache/spack-mirror-permissions "umask 750"
-example cache/spack-mirror-permissions "chmod -R g+rs ~/mirror"
-example cache/spack-mirror-permissions "chgrp -R spack ~/mirror"
+# Setting up a local OCI build cache
+example --tee cache/registry "docker run -d --rm -p 5000:5000 --name registry registry"
 
-example cache/spack-mirror-3 "spack add unzip"
-example cache/spack-mirror-3 "spack install"
+example cache/mirror-add "spack -e . mirror add --unsigned my-registry oci+http://localhost:5000/buildcache"
 
-example cache/spack-mirror-4 "spack mirror create -d ~/mirror --all"
+# Pushing to the OCI build cache
+example --tee cache/push "spack -e . buildcache push --without-build-dependencies my-registry"
 
-example cache/trust "spack buildcache keys --install --trust --force"
+# Re-running the push detects that nothing needs to be uploaded
+example --tee cache/push-again "spack -e . buildcache push --without-build-dependencies my-registry"
 
-example cache/binary-cache-1 "cd ~"
-cd ~ || exit
-example cache/binary-cache-1 "mkdir cache-binary"
-example cache/binary-cache-1 "cd cache-binary"
-cd cache-binary || exit
-example cache/binary-cache-1 "spack env create -d ."
-fake_example cache/binary-cache-1 "spacktivate ." "spack env activate ."
-spack env activate .
-example cache/binary-cache-1 "spack add bzip2"
-example cache/binary-cache-1 "spack add zlib"
+# Reinstalling from the OCI build cache
+# Disable the filesystem "tutorial" mirror by changing mirrors: to mirrors::
+sed -i~ 's/^\( *\)mirrors:$/\1mirrors::/' spack.yaml
 
-example cache/binary-cache-2 'spack config add "config:install_tree:padded_length:128"'
-example cache/binary-cache-2 "spack install --no-cache"
+example --tee cache/reinstall "spack -e . install --overwrite -y julia"
 
-example cache/binary-cache-3 'spack gpg create "My Name" "<my.email@my.domain.com>"'
+# Creating runnable container images
+julia_tag="$(spack -e . find --format '{name}-{version}-{hash}' julia).spack"
 
-example cache/binary-cache-3 'mkdir ~/private_gpg_backup'
-example cache/binary-cache-3 'cp ~/spack/opt/spack/gpg/*.gpg ~/private_gpg_backup'
-example cache/binary-cache-3 'cp ~/spack/opt/spack/gpg/pubring.* ~/mirror'
+# Running the image without a base image fails: it has no glibc.
+example --expect-error cache/docker-run-fail \
+    "docker run --rm localhost:5000/buildcache:${julia_tag} julia -e 'println(1 + 1)'"
 
-example cache/binary-cache-4 'spack buildcache push ~/mirror'
+# Push again with a base image that provides a compatible glibc
+example --tee cache/push-base-image "spack -e . buildcache push --force --without-build-dependencies --base-image ubuntu:26.04 my-registry"
 
-# Remove installations from customized prefix
-spack uninstall -ay
+# The image now runs; --pull always fetches the rebuilt image with the base layer
+example cache/docker-run \
+    "docker run --rm --pull always localhost:5000/buildcache:${julia_tag} julia -e 'println(1 + 1)'"
 
-example cache/bootstrap-1 'spack bootstrap mirror --binary-packages ~/mirror'
+# Spack environments as container images
+# Re-enable the "tutorial" mirror so vim is installed from the cache
+sed -i~ 's/^\( *\)mirrors::$/\1mirrors:/' spack.yaml
+
+example --tee cache/install-vim "spack -e . install --add vim"
+
+example --tee cache/push-tag "spack -e . buildcache push --without-build-dependencies --base-image ubuntu:26.04 --tag julia-and-vim my-registry"
+
+# The combined image is run interactively in the tutorial:
+#   $ docker run -it --rm localhost:5000/buildcache:julia-and-vim

outputs/cache/docker-run-fail.out

@@ -0,0 +1,35 @@
+$ docker run --rm localhost:5000/buildcache:julia-1.12.6-cwym5wwcr7mnaap6okxtqeljftjv3lpu.spack julia -e 'println(1 + 1)'
+Unable to find image 'localhost:5000/buildcache:julia-1.12.6-cwym5wwcr7mnaap6okxtqeljftjv3lpu.spack' locally
+julia-1.12.6-cwym5wwcr7mnaap6okxtqeljftjv3lpu.spack: Pulling from buildcache
+
+1B677dae4359: Pulling fs layer
+1B46b4461b49: Pulling fs layer
+1Bb1eec3f584: Pulling fs layer
+1Bc7cd303c5f: Pulling fs layer
+1Bcf8ca48a15: Pulling fs layer
+1B130f64570e: Pulling fs layer
+1B04ad61a343: Pulling fs layer
+1B58bd53dee4: Pulling fs layer
+1B37a6e2b2d2: Pulling fs layer
+1B37a03e6e93: Pulling fs layer
+1Bd3362c5809: Pulling fs layer
+1Befa2c562fe: Pulling fs layer
+1Bc3b5738e05: Pulling fs layer
+1B4247ca54b9: Pulling fs layer
+1B937d98ff5e: Pulling fs layer
+1Bd6bafa5dc2: Pulling fs layer
+1B4f2eeaaedf: Pulling fs layer
+1Bfcb3848f3f: Pulling fs layer
+1B032b1a40b1: Pulling fs layer
+1B054009b974: Pulling fs layer
+1Ba0ffc08165: Pulling fs layer
+1B92dac9dcca: Pulling fs layer
+1Bf78da11cdb: Pulling fs layer
+1B4f64d88d50: Pulling fs layer
+1B532207b9e2: Pulling fs layer
+1Bdf85180220: Pulling fs layer
+1B6f91ffc3c0: Pulling fs layer
+1B4350659f33: Pulling fs layer
+1BDigest:3sha256:1f7410e032fce4b2ac8bdbf30b797eb823c394596f73f098b0de4bbeda5ce19a58.4MB/158.4MBB
+Status: Downloaded newer image for localhost:5000/buildcache:julia-1.12.6-cwym5wwcr7mnaap6okxtqeljftjv3lpu.spack
+exec /home/spack/spack/opt/spack/linux-x86_64_v3/julia-1.12.6-cwym5wwcr7mnaap6okxtqeljftjv3lpu/bin/julia: no such file or directory

outputs/cache/docker-run.out

@@ -0,0 +1,36 @@
+$ docker run --rm --pull always localhost:5000/buildcache:julia-1.12.6-cwym5wwcr7mnaap6okxtqeljftjv3lpu.spack julia -e 'println(1 + 1)'
+julia-1.12.6-cwym5wwcr7mnaap6okxtqeljftjv3lpu.spack: Pulling from buildcache
+
+1B819469700f: Pulling fs layer
+1B679419df18: Pulling fs layer
+1B677dae4359: Pulling fs layer
+1B46b4461b49: Pulling fs layer
+1Bb1eec3f584: Pulling fs layer
+1Bc7cd303c5f: Pulling fs layer
+1Bcf8ca48a15: Pulling fs layer
+1B130f64570e: Pulling fs layer
+1B04ad61a343: Pulling fs layer
+1B58bd53dee4: Pulling fs layer
+1B37a6e2b2d2: Pulling fs layer
+1B37a03e6e93: Pulling fs layer
+1Bd3362c5809: Pulling fs layer
+1Befa2c562fe: Pulling fs layer
+1Bc3b5738e05: Pulling fs layer
+1B4247ca54b9: Pulling fs layer
+1B937d98ff5e: Pulling fs layer
+1Bd6bafa5dc2: Pulling fs layer
+1B4f2eeaaedf: Pulling fs layer
+1Bfcb3848f3f: Pulling fs layer
+1B032b1a40b1: Pulling fs layer
+1B054009b974: Pulling fs layer
+1Ba0ffc08165: Pulling fs layer
+1B92dac9dcca: Pulling fs layer
+1Bf78da11cdb: Pulling fs layer
+1B4f64d88d50: Pulling fs layer
+1B532207b9e2: Pulling fs layer
+1Bdf85180220: Pulling fs layer
+1B6f91ffc3c0: Pulling fs layer
+1B4350659f33: Pulling fs layer
+1BDigest:dsha256:5d0452d90f2ef02df959fcb3277e31d35f4df46cea7453a1d2d96c0002fc44d258.4MB/158.4MBB
+Status: Downloaded newer image for localhost:5000/buildcache:julia-1.12.6-cwym5wwcr7mnaap6okxtqeljftjv3lpu.spack
+2

outputs/cache/install-julia.out

@@ -0,0 +1,6 @@
+$ mkdir ~/myenv && cd ~/myenv
+$ spack env create --with-view view .
+==> Created independent environment in: /home/spack/myenv
+==> Activate with: spack env activate .
+$ spack -e . add julia
+==> Adding julia to environment /home/spack/myenv

outputs/cache/install-vim.out

@@ -0,0 +1,27 @@
+$ spack -e . install --add vim
+==> Warning: the mirror at oci+http://localhost:5000/buildcache cannot be used in concretization (no index found)
+==> Concretized 1 spec
+[b]  2w6tqad  vim@9.2.0000~cscope~gui~lua~perl~python~ruby~x build_system=autotools features=normal platform=linux os=ubuntu26.04 target=x86_64_v3 %c,cxx=gcc@15.2.0
+[+]  mmywg7x	  ^compiler-wrapper@1.1.0 build_system=generic platform=linux os=ubuntu26.04 target=x86_64_v3
+[+]  punbqrx	  ^findutils@4.10.0 build_system=autotools patches:=440b954 platform=linux os=ubuntu26.04 target=x86_64_v3 %c=gcc@15.2.0
+[+]  sle3ix4	      ^gettext@1.0+bzip2+curses+git~libunistring+libxml2+pic+shared+tar+xz build_system=autotools platform=linux os=ubuntu26.04 target=x86_64_v3 %c,cxx=gcc@15.2.0
+[+]  cbtgjrh		  ^bzip2@1.0.8~debug~pic+shared build_system=generic platform=linux os=ubuntu26.04 target=x86_64_v3 %c=gcc@15.2.0
+[b]  ao2onuz		      ^diffutils@3.12 build_system=autotools platform=linux os=ubuntu26.04 target=x86_64_v3 %c=gcc@15.2.0
+[+]  vbwvgwx		  ^libiconv@1.18 build_system=autotools libs:=shared,static platform=linux os=ubuntu26.04 target=x86_64_v3 %c=gcc@15.2.0
+[+]  ujlg2ua		  ^libxml2@2.15.3+pic~python+shared build_system=autotools platform=linux os=ubuntu26.04 target=x86_64_v3 %c=gcc@15.2.0
+[+]  g72d7i3		      ^zlib-ng@2.3.3+compat+new_strategies+opt+pic+shared build_system=autotools platform=linux os=ubuntu26.04 target=x86_64_v3 %c,cxx=gcc@15.2.0
+[+]  f5xe4px		  ^tar@1.35 build_system=autotools zip=pigz platform=linux os=ubuntu26.04 target=x86_64_v3 %c=gcc@15.2.0
+[+]  afklka7		      ^pigz@2.8 build_system=makefile platform=linux os=ubuntu26.04 target=x86_64_v3 %c=gcc@15.2.0
+[+]  63aruxk		      ^zstd@1.5.7+programs build_system=makefile compression:=none libs:=shared,static platform=linux os=ubuntu26.04 target=x86_64_v3 %c,cxx=gcc@15.2.0
+[+]  hhjyyqy		  ^xz@5.8.3~pic build_system=autotools libs:=shared,static platform=linux os=ubuntu26.04 target=x86_64_v3 %c=gcc@15.2.0
+[e]  yjlog5x	  ^gcc@15.2.0+binutils+bootstrap~graphite+libsanitizer~mold~nvptx~piclibs~profiled~strip build_system=autotools build_type=RelWithDebInfo languages:='c,c++,fortran' platform=linux os=ubuntu26.04 target=x86_64_v3
+[+]  xm76mt3	  ^gcc-runtime@15.2.0 build_system=generic platform=linux os=ubuntu26.04 target=x86_64_v3
+[e]  yc4n2pp	  ^glibc@2.43 build_system=autotools platform=linux os=ubuntu26.04 target=x86_64_v3
+[+]  r4lhaok	  ^gmake@4.4.1~guile build_system=generic platform=linux os=ubuntu26.04 target=x86_64_v3 %c=gcc@15.2.0
+[+]  ekvivpv	  ^ncurses@6.6~symlinks+termlib abi=none build_system=autotools patches:=7a351bc platform=linux os=ubuntu26.04 target=x86_64_v3 %c,cxx=gcc@15.2.0
+[+]  yvl6jpi	      ^pkgconf@2.5.1 build_system=autotools platform=linux os=ubuntu26.04 target=x86_64_v3 %c=gcc@15.2.0
+
+[ ] 2w6tqad vim@9.2.0000 fetching from build cache (0s)
+[ ] 2w6tqad vim@9.2.0000 relocating (0s)
+[+] 2w6tqad vim@9.2.0000 /home/spack/spack/opt/spack/linux-x86_64_v3/vim-9.2.0000-2w6tqadqxniojqyzoe4j55jamd4pbbcp (1s)
+==> Updating view at /home/spack/myenv/view