diff --git a/.gitignore b/.gitignore index 24a77fce2..1d35394f2 100644 --- a/.gitignore +++ b/.gitignore @@ -168,6 +168,7 @@ logs/ tmp/ temp/ *.tmp +e2e/gpu/images/.build/ # Secrets/credentials (should never be committed) *.pem diff --git a/e2e/gpu/README.md b/e2e/gpu/README.md new file mode 100644 index 000000000..8a9bd1e73 --- /dev/null +++ b/e2e/gpu/README.md @@ -0,0 +1,113 @@ + + + +# GPU workload images + +This directory defines GPU workload images used by OpenShell GPU e2e tests. + +The image definitions live here first so the OpenShell e2e harness can iterate +against a concrete contract. The long-term image ownership should move to +`NVIDIA/OpenShell-Community`; OpenShell should then keep the contract, local +build task, and tests that consume published image refs. + +## Contract + +Each workload image must: + +- Use the OpenShell community base image as its final-stage base. +- Install the workload at `/usr/local/bin/openshell-gpu-workload`. +- Run the same workload as the image default entrypoint for direct + container-engine validation. +- Require no network access after the image is pulled. +- Print `OPENSHELL_GPU_WORKLOAD_SUCCESS` only when validation succeeds. +- Print `OPENSHELL_GPU_WORKLOAD_FAILURE` and exit non-zero when validation + fails. +- Be usable as an OpenShell sandbox image with `openshell sandbox create + --from `. + +OpenShell sandbox creation replaces the image entrypoint with the supervisor and +does not run the OCI image `CMD`. E2e tests that use these images through +OpenShell should run `/usr/local/bin/openshell-gpu-workload` explicitly. + +## Images + +| Source directory | Image name | Purpose | +| --- | --- | --- | +| `smoke-pass` | `gpu-workload-smoke-pass` | Always succeeds and prints the success marker. | +| `smoke-fail` | `gpu-workload-smoke-fail` | Always fails and prints the failure marker. | +| `cuda-basic` | `gpu-workload-cuda-basic` | Runs CUDA `deviceQuery` and `vectorAdd` validation. | + +## Build + +Build all workload images: + +```shell +mise run e2e:gpu:images:build +``` + +Build a subset by source directory name: + +```shell +OPENSHELL_GPU_WORKLOAD_IMAGES=smoke-pass,smoke-fail \ +mise run e2e:gpu:images:build +``` + +The build task uses `tasks/scripts/container-engine.sh`. Set +`CONTAINER_ENGINE=docker` or `CONTAINER_ENGINE=podman` to choose an engine +explicitly. When unset, the helper uses its existing auto-detection behavior. + +Local tags use the current commit short SHA. Dirty local trees append `-dirty`. +Set `OPENSHELL_GPU_WORKLOAD_IMAGE_TAG=` to override the tag. + +The task writes the latest build refs to: + +```text +e2e/gpu/images/.build/latest.env +``` + +Use it in later commands: + +```shell +source e2e/gpu/images/.build/latest.env +``` + +## Direct Validation + +Validate smoke pass: + +```shell +docker run --rm "${OPENSHELL_E2E_GPU_SMOKE_PASS_IMAGE}" +``` + +Validate smoke fail: + +```shell +docker run --rm "${OPENSHELL_E2E_GPU_SMOKE_FAIL_IMAGE}" +``` + +The smoke fail command should exit non-zero and print +`OPENSHELL_GPU_WORKLOAD_FAILURE`. + +Validate CUDA with Docker CDI: + +```shell +docker run --rm --device nvidia.com/gpu=all \ + "${OPENSHELL_E2E_GPU_CUDA_WORKLOAD_IMAGE}" +``` + +Use `podman run` with the same `--device nvidia.com/gpu=all` option on hosts +where Podman CDI is configured. + +Direct container-engine validation catches image, CDI, CUDA, and host GPU setup +issues before OpenShell sandbox behavior is involved. + +## Publish Guidance + +Published tests should reference immutable image refs: + +```shell +OPENSHELL_E2E_GPU_CUDA_WORKLOAD_IMAGE=ghcr.io/nvidia/openshell-community/sandboxes/gpu-workload-cuda-basic@sha256: +``` + +Mutable tags are acceptable for local iteration. CI should use a digest or an +immutable release tag once the images are published from OpenShell-Community. diff --git a/e2e/gpu/images/cuda-basic/Dockerfile b/e2e/gpu/images/cuda-basic/Dockerfile new file mode 100644 index 000000000..a7dde7422 --- /dev/null +++ b/e2e/gpu/images/cuda-basic/Dockerfile @@ -0,0 +1,72 @@ +# syntax=docker/dockerfile:1 + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +ARG CUDA_BUILD_IMAGE=nvcr.io/nvidia/cuda:12.8.1-base-ubuntu22.04 +ARG OPENSHELL_SANDBOX_BASE_IMAGE=ghcr.io/nvidia/openshell-community/sandboxes/base:latest + +FROM ${CUDA_BUILD_IMAGE} AS builder + +ARG DEBIAN_FRONTEND=noninteractive +ARG CUDA_SAMPLES_REF=v12.8 +ARG CUDA_SAMPLES_REPO=https://github.com/NVIDIA/cuda-samples + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + cmake \ + cuda-nvcc-12-8 \ + curl \ + g++ \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /build/cuda-samples + +RUN set -eux; \ + curl -fsSL "${CUDA_SAMPLES_REPO}/archive/refs/tags/${CUDA_SAMPLES_REF}.tar.gz" \ + -o /tmp/cuda-samples.tar.gz; \ + tar -xzf /tmp/cuda-samples.tar.gz \ + --strip-components=1 \ + --wildcards \ + '*/Common/*' \ + '*/cmake/*' \ + '*/Samples/0_Introduction/vectorAdd/*' \ + '*/Samples/1_Utilities/deviceQuery/*' \ + '*/LICENSE'; \ + sed -i 's/CUDA::cudart/CUDA::cudart_static/g' \ + Samples/1_Utilities/deviceQuery/CMakeLists.txt; \ + cmake -S Samples/1_Utilities/deviceQuery -B /tmp/build-device-query \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CUDA_RUNTIME_LIBRARY=Static; \ + cmake --build /tmp/build-device-query --parallel; \ + cmake -S Samples/0_Introduction/vectorAdd -B /tmp/build-vector-add \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CUDA_RUNTIME_LIBRARY=Static; \ + cmake --build /tmp/build-vector-add --parallel; \ + mkdir -p /opt/openshell-gpu-workload; \ + cp /tmp/build-device-query/deviceQuery /opt/openshell-gpu-workload/deviceQuery; \ + cp /tmp/build-vector-add/vectorAdd /opt/openshell-gpu-workload/vectorAdd; \ + cp LICENSE /opt/openshell-gpu-workload/cuda-samples.LICENSE; \ + rm -f /tmp/cuda-samples.tar.gz + +FROM ${OPENSHELL_SANDBOX_BASE_IMAGE} + +ARG CUDA_SAMPLES_REF=v12.8 + +LABEL com.nvidia.openshell.gpu-workload.name="cuda-basic" \ + com.nvidia.openshell.gpu-workload.cuda-samples-ref="${CUDA_SAMPLES_REF}" + +USER root +RUN mkdir -p /usr/local/lib/openshell-gpu-workload \ + /usr/local/share/doc/openshell-gpu-workload +COPY --from=builder /opt/openshell-gpu-workload/deviceQuery /usr/local/lib/openshell-gpu-workload/deviceQuery +COPY --from=builder /opt/openshell-gpu-workload/vectorAdd /usr/local/lib/openshell-gpu-workload/vectorAdd +COPY --from=builder /opt/openshell-gpu-workload/cuda-samples.LICENSE /usr/local/share/doc/openshell-gpu-workload/cuda-samples.LICENSE +COPY workload.sh /usr/local/bin/openshell-gpu-workload +RUN chmod 0755 /usr/local/bin/openshell-gpu-workload \ + /usr/local/lib/openshell-gpu-workload/deviceQuery \ + /usr/local/lib/openshell-gpu-workload/vectorAdd + +USER sandbox +ENTRYPOINT ["/usr/local/bin/openshell-gpu-workload"] diff --git a/e2e/gpu/images/cuda-basic/README.md b/e2e/gpu/images/cuda-basic/README.md new file mode 100644 index 000000000..7445a5cb2 --- /dev/null +++ b/e2e/gpu/images/cuda-basic/README.md @@ -0,0 +1,42 @@ + + + +# GPU workload CUDA basic + +`cuda-basic` validates that a GPU-enabled environment can run a basic CUDA +runtime workload. It is a single image that runs two validation steps: + +1. `deviceQuery` checks CUDA runtime, driver, and device discovery. +2. `vectorAdd` checks kernel launch, device memory allocation, host/device + copies, synchronization, and result validation. + +The image builds the samples from `NVIDIA/cuda-samples` tag `v12.8` with a CUDA +12.8 builder image, then copies only the compiled binaries into the OpenShell +community base final image. + +The workload prints `OPENSHELL_GPU_WORKLOAD_SUCCESS` only after both samples +pass. On failure it prints `OPENSHELL_GPU_WORKLOAD_FAILURE` and exits non-zero. + +Build it with: + +```shell +OPENSHELL_GPU_WORKLOAD_IMAGES=cuda-basic mise run e2e:gpu:images:build +``` + +Run it directly with Docker CDI: + +```shell +source e2e/gpu/images/.build/latest.env +docker run --rm --device nvidia.com/gpu=all \ + "${OPENSHELL_E2E_GPU_CUDA_WORKLOAD_IMAGE}" +``` + +Use `podman run` with the same `--device nvidia.com/gpu=all` option when Podman +CDI is configured. + +The image does not vendor GPU driver libraries such as `libcuda.so.1`. Those +libraries must be provided by the host GPU runtime or CDI injection. + +The CUDA samples are redistributed under the NVIDIA CUDA samples license. The +license text is copied into the image at +`/usr/local/share/doc/openshell-gpu-workload/cuda-samples.LICENSE`. diff --git a/e2e/gpu/images/cuda-basic/workload.sh b/e2e/gpu/images/cuda-basic/workload.sh new file mode 100644 index 000000000..e20a67d96 --- /dev/null +++ b/e2e/gpu/images/cuda-basic/workload.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +readonly SUCCESS_MARKER="OPENSHELL_GPU_WORKLOAD_SUCCESS" +readonly FAILURE_MARKER="OPENSHELL_GPU_WORKLOAD_FAILURE" +readonly WORKLOAD_DIR="/usr/local/lib/openshell-gpu-workload" + +run_sample() { + local name=$1 + local expected=$2 + local binary="${WORKLOAD_DIR}/${name}" + local output + + output="$(mktemp)" + echo "running CUDA sample: ${name}" + if ! "${binary}" >"${output}" 2>&1; then + cat "${output}" + echo "${FAILURE_MARKER} ${name} exited non-zero" >&2 + rm -f "${output}" + exit 1 + fi + + cat "${output}" + if ! grep -Fq "${expected}" "${output}"; then + echo "${FAILURE_MARKER} ${name} did not print expected output: ${expected}" >&2 + rm -f "${output}" + exit 1 + fi + + rm -f "${output}" +} + +run_sample "deviceQuery" "Result = PASS" +run_sample "vectorAdd" "Test PASSED" + +echo "${SUCCESS_MARKER} cuda-basic" diff --git a/e2e/gpu/images/smoke-fail/Dockerfile b/e2e/gpu/images/smoke-fail/Dockerfile new file mode 100644 index 000000000..f74aa3c5e --- /dev/null +++ b/e2e/gpu/images/smoke-fail/Dockerfile @@ -0,0 +1,15 @@ +# syntax=docker/dockerfile:1 + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +ARG OPENSHELL_SANDBOX_BASE_IMAGE=ghcr.io/nvidia/openshell-community/sandboxes/base:latest + +FROM ${OPENSHELL_SANDBOX_BASE_IMAGE} + +USER root +COPY workload.sh /usr/local/bin/openshell-gpu-workload +RUN chmod 0755 /usr/local/bin/openshell-gpu-workload + +USER sandbox +ENTRYPOINT ["/usr/local/bin/openshell-gpu-workload"] diff --git a/e2e/gpu/images/smoke-fail/README.md b/e2e/gpu/images/smoke-fail/README.md new file mode 100644 index 000000000..1c24f4452 --- /dev/null +++ b/e2e/gpu/images/smoke-fail/README.md @@ -0,0 +1,24 @@ + + + +# GPU workload smoke fail + +`smoke-fail` validates negative-path diagnostics in e2e test plumbing. + +The workload does not perform GPU-specific work. It prints +`OPENSHELL_GPU_WORKLOAD_FAILURE`, emits a stable diagnostic, and exits non-zero. + +Build it with: + +```shell +OPENSHELL_GPU_WORKLOAD_IMAGES=smoke-fail mise run e2e:gpu:images:build +``` + +Run it directly: + +```shell +source e2e/gpu/images/.build/latest.env +docker run --rm "${OPENSHELL_E2E_GPU_SMOKE_FAIL_IMAGE}" +``` + +The direct run should fail. diff --git a/e2e/gpu/images/smoke-fail/workload.sh b/e2e/gpu/images/smoke-fail/workload.sh new file mode 100644 index 000000000..8c57624df --- /dev/null +++ b/e2e/gpu/images/smoke-fail/workload.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +echo "OPENSHELL_GPU_WORKLOAD_FAILURE smoke-fail intentional failure" >&2 +exit 42 diff --git a/e2e/gpu/images/smoke-pass/Dockerfile b/e2e/gpu/images/smoke-pass/Dockerfile new file mode 100644 index 000000000..f74aa3c5e --- /dev/null +++ b/e2e/gpu/images/smoke-pass/Dockerfile @@ -0,0 +1,15 @@ +# syntax=docker/dockerfile:1 + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +ARG OPENSHELL_SANDBOX_BASE_IMAGE=ghcr.io/nvidia/openshell-community/sandboxes/base:latest + +FROM ${OPENSHELL_SANDBOX_BASE_IMAGE} + +USER root +COPY workload.sh /usr/local/bin/openshell-gpu-workload +RUN chmod 0755 /usr/local/bin/openshell-gpu-workload + +USER sandbox +ENTRYPOINT ["/usr/local/bin/openshell-gpu-workload"] diff --git a/e2e/gpu/images/smoke-pass/README.md b/e2e/gpu/images/smoke-pass/README.md new file mode 100644 index 000000000..c09946c17 --- /dev/null +++ b/e2e/gpu/images/smoke-pass/README.md @@ -0,0 +1,23 @@ + + + +# GPU workload smoke pass + +`smoke-pass` validates image publishing, sandbox image compatibility, default +entrypoint execution, and success-marker assertion plumbing. + +The workload does not perform GPU-specific work. It prints +`OPENSHELL_GPU_WORKLOAD_SUCCESS` and exits `0`. + +Build it with: + +```shell +OPENSHELL_GPU_WORKLOAD_IMAGES=smoke-pass mise run e2e:gpu:images:build +``` + +Run it directly: + +```shell +source e2e/gpu/images/.build/latest.env +docker run --rm "${OPENSHELL_E2E_GPU_SMOKE_PASS_IMAGE}" +``` diff --git a/e2e/gpu/images/smoke-pass/workload.sh b/e2e/gpu/images/smoke-pass/workload.sh new file mode 100644 index 000000000..76f848f50 --- /dev/null +++ b/e2e/gpu/images/smoke-pass/workload.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +echo "OPENSHELL_GPU_WORKLOAD_SUCCESS smoke-pass" diff --git a/tasks/scripts/e2e-gpu-build-images.sh b/tasks/scripts/e2e-gpu-build-images.sh new file mode 100644 index 000000000..af67204cc --- /dev/null +++ b/tasks/scripts/e2e-gpu-build-images.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +source "${SCRIPT_DIR}/container-engine.sh" + +IMAGES_ROOT="${ROOT}/e2e/gpu/images" +BUILD_DIR="${IMAGES_ROOT}/.build" +BASE_IMAGE="${OPENSHELL_SANDBOX_BASE_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}" +CUDA_BUILD_IMAGE="${CUDA_BUILD_IMAGE:-nvcr.io/nvidia/cuda:12.8.1-base-ubuntu22.04}" +CUDA_SAMPLES_REPO="${CUDA_SAMPLES_REPO:-https://github.com/NVIDIA/cuda-samples}" +CUDA_SAMPLES_REF="${CUDA_SAMPLES_REF:-v12.8}" + +shell_quote() { + local value=$1 + printf "'%s'" "${value//\'/\'\\\'\'}" +} + +write_env_var() { + local name=$1 + local value=$2 + printf 'export %s=%s\n' "${name}" "$(shell_quote "${value}")" +} + +available_image_dirs() { + local dockerfile + local preferred + local seen=" " + + for preferred in smoke-pass smoke-fail cuda-basic; do + if [[ -f "${IMAGES_ROOT}/${preferred}/Dockerfile" ]]; then + echo "${preferred}" + seen+="${preferred} " + fi + done + + find "${IMAGES_ROOT}" -mindepth 2 -maxdepth 2 -name Dockerfile -type f | sort | while IFS= read -r dockerfile; do + name="$(basename "$(dirname "${dockerfile}")")" + [[ "${seen}" == *" ${name} "* ]] && continue + echo "${name}" + done +} + +contains_image() { + local needle=$1 + shift + local item + for item in "$@"; do + [[ "${item}" == "${needle}" ]] && return 0 + done + return 1 +} + +image_env_var() { + case "$1" in + smoke-pass) echo "OPENSHELL_E2E_GPU_SMOKE_PASS_IMAGE" ;; + smoke-fail) echo "OPENSHELL_E2E_GPU_SMOKE_FAIL_IMAGE" ;; + cuda-basic) echo "OPENSHELL_E2E_GPU_CUDA_WORKLOAD_IMAGE" ;; + *) + echo "unsupported GPU workload image source directory: $1" >&2 + exit 1 + ;; + esac +} + +mapfile -t available < <(available_image_dirs) +if [[ ${#available[@]} -eq 0 ]]; then + echo "No GPU workload image Dockerfiles found under ${IMAGES_ROOT}" >&2 + exit 1 +fi + +selected=() +if [[ -n "${OPENSHELL_GPU_WORKLOAD_IMAGES:-}" ]]; then + IFS=',' read -r -a requested <<< "${OPENSHELL_GPU_WORKLOAD_IMAGES}" + for raw in "${requested[@]}"; do + name="${raw//[[:space:]]/}" + [[ -z "${name}" ]] && continue + if ! contains_image "${name}" "${available[@]}"; then + echo "Unknown GPU workload image source directory: ${name}" >&2 + echo "Available: ${available[*]}" >&2 + exit 1 + fi + selected+=("${name}") + done +else + selected=("${available[@]}") +fi + +if [[ ${#selected[@]} -eq 0 ]]; then + echo "No GPU workload images selected" >&2 + exit 1 +fi + +source_sha="$(git -C "${ROOT}" rev-parse HEAD)" +source_short_sha="$(git -C "${ROOT}" rev-parse --short HEAD)" +source_dirty=false +if [[ -n "$(git -C "${ROOT}" status --short)" ]]; then + source_dirty=true +fi + +if [[ -n "${OPENSHELL_GPU_WORKLOAD_IMAGE_TAG:-}" ]]; then + image_tag="${OPENSHELL_GPU_WORKLOAD_IMAGE_TAG}" +else + image_tag="${source_short_sha}" + if [[ "${source_dirty}" == "true" ]]; then + image_tag="${image_tag}-dirty" + fi +fi + +declare -A image_refs=() + +echo "Building GPU workload images with ${CONTAINER_ENGINE}" +echo "Source: ${source_short_sha} (dirty: ${source_dirty})" +echo "Tag: ${image_tag}" + +for name in "${selected[@]}"; do + image_name="gpu-workload-${name}" + image_ref="localhost/openshell/${image_name}:${image_tag}" + context="${IMAGES_ROOT}/${name}" + + build_args=( + --build-arg "OPENSHELL_SANDBOX_BASE_IMAGE=${BASE_IMAGE}" + ) + if [[ "${name}" == "cuda-basic" ]]; then + build_args+=( + --build-arg "CUDA_BUILD_IMAGE=${CUDA_BUILD_IMAGE}" + --build-arg "CUDA_SAMPLES_REPO=${CUDA_SAMPLES_REPO}" + --build-arg "CUDA_SAMPLES_REF=${CUDA_SAMPLES_REF}" + ) + fi + + echo + echo "Building ${name} as ${image_ref}" + ce_build \ + --load \ + --provenance=false \ + -t "${image_ref}" \ + --label "com.nvidia.openshell.gpu-workload.source=${name}" \ + --label "org.opencontainers.image.revision=${source_sha}" \ + "${build_args[@]}" \ + "${context}" + + image_refs["${name}"]="${image_ref}" +done + +mkdir -p "${BUILD_DIR}" +latest_env="${BUILD_DIR}/latest.env" +{ + echo "# Generated by mise run e2e:gpu:images:build" + echo "# Source this file to use the most recently built GPU workload images." + write_env_var OPENSHELL_GPU_WORKLOAD_IMAGE_TAG "${image_tag}" + write_env_var OPENSHELL_GPU_WORKLOAD_IMAGE_SOURCE_PATH "${IMAGES_ROOT}" + write_env_var OPENSHELL_GPU_WORKLOAD_IMAGE_SOURCE_SHA "${source_sha}" + write_env_var OPENSHELL_GPU_WORKLOAD_IMAGE_SOURCE_DIRTY "${source_dirty}" + write_env_var OPENSHELL_GPU_WORKLOAD_CONTAINER_ENGINE "${CONTAINER_ENGINE}" + for name in "${selected[@]}"; do + write_env_var "$(image_env_var "${name}")" "${image_refs[${name}]}" + done +} > "${latest_env}" + +echo +echo "Wrote ${latest_env}" +echo "Built images:" +for name in "${selected[@]}"; do + echo " ${name}: ${image_refs[${name}]}" +done diff --git a/tasks/test.toml b/tasks/test.toml index c6ac82180..30c127045 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -15,6 +15,10 @@ depends = ["e2e:rust", "e2e:python"] description = "Run Docker GPU end-to-end tests" depends = ["e2e:docker:gpu"] +["e2e:gpu:images:build"] +description = "Build local GPU workload images for e2e validation" +run = "bash tasks/scripts/e2e-gpu-build-images.sh" + ["e2e:k3s:gpu"] description = "Run k3s GPU end-to-end tests" depends = ["e2e:python:gpu"]