30  Build Final SIF

This notebook builds the final read-only SIF from the validated platform definition.

from __future__ import annotations

def build_container(singularity_def, SINGULARITY_IMAGE):
    from spython.main import Client
    from pathlib import Path
    import os

    job_id = os.environ.get("SLURM_JOB_ID", "local")
    user = os.environ["USER"]
    tmp_root = Path(f"/tmp/{user}-rse-workbench-{job_id}")
    singularity_tmp = tmp_root / "singularity-tmp"
    singularity_cache = tmp_root / "singularity-cache"

    singularity_tmp.mkdir(parents=True, exist_ok=True)
    singularity_cache.mkdir(parents=True, exist_ok=True)

    os.environ["SINGULARITY_TMPDIR"] = str(singularity_tmp)
    os.environ["SINGULARITY_CACHEDIR"] = str(singularity_cache)
    os.environ["APPTAINER_TMPDIR"] = str(singularity_tmp)
    os.environ["APPTAINER_CACHEDIR"] = str(singularity_cache)

    return Client.build(
        image=str(SINGULARITY_IMAGE),
        recipe=str(singularity_def),
        options=["--fakeroot", "--force"],
        sudo=False,
        return_result=True,
    )

import submitit
from rse_workbench.helpers import show_submitit_job, spython_exec, show_spython
from rse_workbench.build_install import build_container

Build:

executor = submitit.AutoExecutor(folder=str(project_contract.submitit_dir / "%j"))
executor.update_parameters(
    timeout_min=project_contract.slurm_build_hours * 60,
    slurm_partition=project_contract.slurm_build_partition,
    mem_gb=float(project_contract.slurm_build_mem.rstrip("G")),
    cpus_per_task=project_contract.slurm_build_cpus,
    slurm_job_name=f"build-{project_contract.project_name}-container",
    mail_user=project_contract.slurm_email,
)

job = executor.submit(build_container, project_contract.singularity_def, project_contract.singularity_image)
job.job_id

result = show_submitit_job(job)
if result is not None:
    print(result)

Install Spack and System Dependencies:

job = executor.submit(
    install_in_container,
    project_dir=project_contract.project_dir,
    SINGULARITY_IMAGE=project_contract.singularity_image,
    runtime_command=project_contract.spack_install_command,
)
job.job_id

result = show_submitit_job(job)
if result is not None:
    print(''.join(result['message']))

Install Data Science Languages and Packages:

job = executor.submit(
    install_in_container,
    project_dir=project_contract.project_dir,
    SINGULARITY_IMAGE=project_contract.singularity_image,
    runtime_command=project_contract.language_install_command,
)
job.job_id

result = show_submitit_job(job)
if result is not None:
    print(''.join(result['message']))

This section verifies that each layer of the environment is functioning correctly independently before testing that the layers communicate correctly.

The architecture is expected to look like:

Container
    ↓
Spack
    ↓
R / Rscript
    ↓
rv-managed R package library
    ↓
Quarto
    ↓
Project render

At each stage we test a single contract before moving to the next.

---

30.1 1. Verify the container runtime

The container should provide the basic executables required for the project.

command = r"""
set -euo pipefail
echo "============================================================"
echo "1. Base container tools (before activation)"
echo "============================================================"

command -v spack
command -v rv
command -v uv
command -v quarto
quarto --version
echo

echo "============================================================"
echo "2. Activate project"
echo "============================================================"

cd /work
source /work/env/activate.sh
echo

echo "============================================================"
echo "3. Tool discovery after activation"
echo "============================================================"

command -v spack
command -v R
command -v Rscript
command -v rv
command -v quarto
echo

echo "============================================================"
echo "4. Spack"
echo "============================================================"

spack env status
spack find r
echo

echo "============================================================"
echo "5. rv"
echo "============================================================"

rv cache
rv sync
echo

echo "============================================================"
echo "6. R"
echo "============================================================"

Rscript -e '
cat("R.home():\n")
print(R.home())
cat("\n.libPaths():\n")
print(.libPaths())

pkgs <- c(
  "rlang",
  "knitr",
  "rmarkdown"
)

for (pkg in pkgs) {
  cat("\n=====================================\n")
  cat(pkg, "\n")
  cat("=====================================\n")
  cat("find.package():\n")
  print(find.package(pkg, quiet = TRUE))
  cat("\nrequireNamespace():\n")
  print(requireNamespace(pkg, quietly = FALSE))
  cat("\nlibrary():\n")
  try(library(pkg, character.only = TRUE))
  cat("\npackageVersion():\n")
  print(packageVersion(pkg))
}
'
echo

echo "============================================================"
echo "7. Quarto discovery"
echo "============================================================"

export QUARTO_R="$(command -v Rscript)"
echo "QUARTO_R=${QUARTO_R}"
quarto check knitr --log-level DEBUG
"""

Expected:

• every executable is found
• Quarto prints its version

---

30.2 2. Verify that Spack owns the R installation

Activate the project environment and inspect the R installation.

show_spython(
    spython_exec(
        command=r'''
cd /work

source /work/env/activate.sh

spack env status
spack find r

Rscript -e '
cat("R.home():\n")
print(R.home())

cat("\n.libPaths():\n")
print(.libPaths())
'
''',
        image=SINGULARITY_IMAGE,
        options=[
            "--home", f"{CONTAINER_HOME}:/home/rse",
            "--bind", f"{CONTAINER_TMP}:/tmp",
        ],
        bind_project=PROJECT_DIR,
    ),
)

Expected:

• the project Spack environment is active
• R.home() points inside .spack-env
• .libPaths() contains

/work/.rv/library
/work/.spack-env/.../R/library

---

30.3 3. Verify that rv owns the project R packages

Synchronize the project packages and verify that they can be loaded.

show_spython(
    spython_exec(
        command=r'''
rv sync

Rscript -e '
cat(".libPaths():\n")
print(.libPaths())

pkgs <- c("rlang", "knitr", "rmarkdown")

for (pkg in pkgs) {
  cat("\n=============================\n")
  cat(pkg, "\n")
  cat("=============================\n")

  print(find.package(pkg, quiet = TRUE))
  print(requireNamespace(pkg, quietly = FALSE))
}
'
''',
        image=SINGULARITY_IMAGE,
        options=[
            "--home", f"{CONTAINER_HOME}:/home/rse",
            "--bind", f"{CONTAINER_TMP}:/tmp",
        ],
        bind_project=PROJECT_DIR,
    ),
)

Expected:

TRUE
TRUE
TRUE

for all three packages.

---

30.4 4. Verify that Quarto discovers the same R installation

Explicitly tell Quarto which R executable to use.

export QUARTO_R="$(command -v Rscript)"

quarto check knitr --log-level DEBUG

Expected:

• Quarto reports the same R installation
• knitr is detected
• rmarkdown is detected

---