Build Accessibility Tree from scene

[mysunnytime]

(A draft PR hoping to get early feedback.)

1. build a DOM tree for: Node that are marked as salient, and GUI Controls. In this way, the screen reader can read the page content.
2. make the DOM elements interactive for actionable nodes. So the user can use "tab" key to focus on interactive nodes or controls, and "enter" key to trigger the interactive nodes.
3. when the scene content changes (e.g. show/hide new salient mesh), the DOM tree updates too.

@RaananW @Exolun

Testing:

Testing code 1:
Given a scene that has objects that cover conditions of:

• some are salient, some are not not-salient
• some objects (spheres) are interactive, and could be left and right clicked
• objects have parent-children hierarchy

let createScene = function () {
    let scene = new BABYLON.Scene(engine);
    let camera = new BABYLON.ArcRotateCamera("Camera", Math.PI/2, Math.PI/4, 10, new BABYLON.Vector3(0, 0, 0), scene);
    camera.setTarget(BABYLON.Vector3.Zero());
    camera.attachControl(canvas, true);
    let light = new BABYLON.HemisphericLight("light", new BABYLON.Vector3(0, 1, 0), scene);
    light.intensity = 0.7;

    // add some objects

    // not salient objects
    const parent = new BABYLON.TransformNode('parent');
    parent.accessibilityTag = {
        isSalient: true,
        description: "A parent of all, of the root",
    }

    const boxDecs = new BABYLON.TransformNode('boxDecs');
    boxDecs.parent = parent;
    boxDecs.accessibilityTag = {
        isSalient: true,
        description: "A parent without salient children",
    }
    let boxDec1 = BABYLON.MeshBuilder.CreateBox("boxDec1", {size: 0.3}, scene);
    boxDec1.parent = boxDecs;
    boxDec1.position.x = -3;
    boxDec1.position.z = -4;
    let boxDec2 = BABYLON.MeshBuilder.CreateBox("boxDec2", {size: 0.3}, scene);
    boxDec2.parent = boxDecs;
    boxDec2.position.x = 3;
    boxDec2.position.z = -4;

    // salient objects, static
    let boxes = new BABYLON.TransformNode("boxes");
    boxes.parent = parent;
    boxes.accessibilityTag = {
        isSalient: true,
        description: "A group of boxes",
    }
    let box0 = BABYLON.MeshBuilder.CreateBox("box3", {size: 0.5}, scene);
    box0.parent = boxes;
    box0.position.z = -1;
    box0.position.y = 0.6;
    box0.accessibilityTag = {
        isSalient: true,
        description: "A small box in the middle of the scene",
    }
    let box1 = BABYLON.MeshBuilder.CreateBox("box1", {}, scene);
    box1.parent = boxes;
    box1.position.x = 1;
    box1.accessibilityTag = {
        isSalient: true,
        description: "A big box on the left of the small box",
    }
    let box2 = BABYLON.MeshBuilder.CreateBox("box2", {}, scene);
    box2.parent = boxes;
    box2.position.x = -1;
    box2.accessibilityTag = {
        isSalient: true,
        description: "A big box on the right of the small box",
    }

    // salient objects, interactive
    let sphereWrapper = new BABYLON.TransformNode('sphereWrapper');
    sphereWrapper.accessibilityTag = {
        isSalient: true,
        description: 'A group of spheres',
    };
    // sphereWrapper.parent = parent;
    let mat = new BABYLON.StandardMaterial("Gray", scene);
    mat.diffuseColor = BABYLON.Color3.Gray();
    let spheresCount = 6;
    let alpha = 0;
    for (let index = 0; index < spheresCount; index++) {
        const sphere = BABYLON.Mesh.CreateSphere("Sphere " + index, 32, 1, scene);
        sphere.parent = sphereWrapper;
        sphere.position.x = 3 * Math.cos(alpha);
        sphere.position.z = 3 * Math.sin(alpha);
        sphere.material = mat;
        const sphereOnClicked = () => {
            alert(`You just clicked ${sphere.name}!`)
        }
        const sphereOnLeftClicked2 = () => {
            alert(`You just LEFT clicked ${sphere.name}!`)
        }
        const sphereOnRightClicked = () => {
            alert(`You just RIGHT clicked ${sphere.name}!`)
        }
        sphere.accessibilityTag = {
            isSalient: true,
            description: sphere.name,
        };
        sphere.actionManager = new BABYLON.ActionManager(scene);
        sphere.actionManager.registerAction(new BABYLON.ExecuteCodeAction(BABYLON.ActionManager.OnPickTrigger, sphereOnClicked));
        sphere.actionManager.registerAction(new BABYLON.ExecuteCodeAction(BABYLON.ActionManager.OnLeftPickTrigger, sphereOnLeftClicked2));
        sphere.actionManager.registerAction(new BABYLON.ExecuteCodeAction(BABYLON.ActionManager.OnRightPickTrigger, sphereOnRightClicked));
        alpha += (2 * Math.PI) / spheresCount;
    }

    console.log('Start the show!');
    AccessibilityRenderer.renderAccessibilityTree(scene);
    return scene;
};

Testing code 2:

/**
 * A simple GUI scene. Shows a card GUI for a easter event, with buttons to 'Join' event, and 'close' card.
 */
export let createScene = function ()
{
    let scene = new BABYLON.Scene(engine);
    let camera = new BABYLON.ArcRotateCamera('Camera', -Math.PI/2, Math.PI/2, 5, new BABYLON.Vector3(0, 0, 0), scene);
    camera.setTarget(BABYLON.Vector3.Zero());
    camera.attachControl(canvas, true);
    let light = new BABYLON.HemisphericLight('light', new BABYLON.Vector3(0, 1, 0), scene);
    light.intensity = 0.7;

    let wrapper = new BABYLON.TransformNode("Wrapper");
    wrapper.setEnabled(true);

    let egg = BABYLON.MeshBuilder.CreateSphere("Egg", {diameterX: 0.62, diameterY: 0.8, diameterZ: 0.6}, scene);
    egg.parent = wrapper;
    egg.accessibilityTag = {
        isSalient: true,
        description: "An easter egg"
    }
    egg.actionManager = new BABYLON.ActionManager(scene);
    egg.actionManager.registerAction(new BABYLON.ExecuteCodeAction(
        BABYLON.ActionManager.OnPickTrigger,
        () => {
            wrapper.setEnabled(false);
            card.setEnabled(true);
        })
    );

    let box1 = BABYLON.MeshBuilder.CreateBox("box1", {size: 0.3}, scene);
    box1.parent = wrapper;
    box1.position.x = 1;
    box1.position.z = 0.2;
    box1.accessibilityTag = {
        isSalient: true,
        description: "A small box on the left of the egg",
        isWholeObject: true,
    }
    let box2 = BABYLON.MeshBuilder.CreateBox("box2", {size: 0.3}, scene);
    box2.parent = wrapper;
    box2.position.x = -1;
    box2.position.z = 0.2;
    box2.accessibilityTag = {
        isSalient: true,
        description: "A small box on the right of the egg",
        isWholeObject: true,
    }

    let box = BABYLON.MeshBuilder.CreateBox("box", {size: 0.5}, scene);
    box.position.y = -0.65;
    box.parent = egg;

    // GUI
    let card = BABYLON.MeshBuilder.CreatePlane('card', {size: 3});
    card.setEnabled(false);
    card.accessibilityTag = {
        isSalient: true,
        description: "Easter Event Card"
    }
    card.position.z = 0.5;
    let adt = GUI.AdvancedDynamicTexture.CreateForMesh(card);

    let wrapFront = new GUI.Rectangle('TeamsCardUI_wrapFront');
    wrapFront.width = '80%';
    wrapFront.background = 'white';
    adt.addControl(wrapFront);

    let thumbnailBg = new GUI.Rectangle('TeamsCardUI_ThumbnailBg');
    thumbnailBg.width = '100%';
    thumbnailBg.height = '40%';
    thumbnailBg.background = 'gray';
    thumbnailBg.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
    thumbnailBg.top = 0;
    wrapFront.addControl(thumbnailBg);

    let url = 'https://raw.githubusercontent.com/TNyawara/EasterBunnyGroupProject/master/Assets/Easter_UI/Backgrounds/background_10.png';
    let thumbnailCustomized = new GUI.Image('TeamsCardUI_ThumbnailImage', url);
    thumbnailCustomized.alt = 'Background image';
    thumbnailCustomized.width = '100%';
    thumbnailCustomized.height = '100%';
    thumbnailBg.addControl(thumbnailCustomized);

    const titleFront = new GUI.TextBlock(
    'TeamsCardUIText_Title',
    'Event: Happy Hoppy'
    );
    titleFront.fontSize = 60;
    titleFront.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
    titleFront.textHorizontalAlignment = GUI.Control.HORIZONTAL_ALIGNMENT_LEFT;
    titleFront.paddingLeft = '7.5%';
    titleFront.top = '40%';
    titleFront.height = '10%';
    wrapFront.addControl(titleFront);

    let dateFront = new GUI.TextBlock('TeamsCardUIText_Date', 'Every day');
    wrapFront.addControl(dateFront);
    dateFront.fontSize = 40;
    dateFront.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
    dateFront.textHorizontalAlignment = GUI.Control.HORIZONTAL_ALIGNMENT_LEFT;
    dateFront.paddingLeft = '7.5%';
    dateFront.top = '50%';
    dateFront.height = '5%';
    dateFront.isEnabled = false;

    const timeFront = new GUI.TextBlock('TeamsCardUIText_Time', '00:00 - 23:59');
    wrapFront.addControl(timeFront);
    timeFront.fontSize = 40;
    timeFront.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
    timeFront.textHorizontalAlignment = GUI.Control.HORIZONTAL_ALIGNMENT_LEFT;
    timeFront.paddingLeft = '35%';
    timeFront.top = '50%';
    timeFront.height = '5%';

    const meetingDetail = new GUI.TextBlock(
    'TeamsCardUIText_GroupName',
    "Help the little bunny rabbits get ready for Easter! Look at all the different colors to decorate Easter eggs with and pick out the shapes you'd like to wear in the parade. "
    );
    wrapFront.addControl(meetingDetail);
    meetingDetail.fontSize = 40;
    meetingDetail.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
    meetingDetail.textHorizontalAlignment = GUI.Control.HORIZONTAL_ALIGNMENT_LEFT;
    meetingDetail.paddingLeft = '7.5%';
    meetingDetail.top = '55%';
    meetingDetail.height = '30%';
    meetingDetail.width = '100%';
    meetingDetail.textWrapping = GUI.TextWrapping.WordWrapEllipsis;

    let joinButton = GUI.Button.CreateSimpleButton('TeamsCardUIButton_Join', 'Join');
    joinButton.background = 'black';
    joinButton.color = 'white';
    joinButton.fontSize = 40;
    joinButton.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
    joinButton.textBlock.textHorizontalAlignment = GUI.Control.VERTICAL_ALIGNMENT_CENTER;
    joinButton.top = '85%';
    joinButton.height = '10%';
    joinButton.width = '40%';
    wrapFront.addControl(joinButton);
    joinButton.onPointerClickObservable.add(() => {
        alert('💐🌼Happy Easter! 🐰🥚');
    });

    let closeButton = GUI.Button.CreateSimpleButton('TeamsCardUIButton_Close', 'X');
    closeButton.background = 'black';
    closeButton.color = 'white';
    closeButton.fontSize = 40;
    closeButton.verticalAlignment = GUI.Control.VERTICAL_ALIGNMENT_TOP;
    closeButton.horizontalAlignment = GUI.Control.HORIZONTAL_ALIGNMENT_RIGHT;
    closeButton.textBlock.textHorizontalAlignment = GUI.Control.VERTICAL_ALIGNMENT_CENTER;
    closeButton.top = '0';
    closeButton.height = '12%';
    closeButton.width = '15%';
    wrapFront.addControl(closeButton);
    closeButton.onPointerClickObservable.add(() => {
        card.setEnabled(false);
        wrapper.setEnabled(true);
    });

    console.log('Start the show!');
    AccessibilityRenderer.RenderAccessibilityTree(scene);
    return scene;
}

[deltakosh]

Woot! I love it!! cc @PirateJC and @PatrickRyanMS for awareness

Only thing to consider is that we are in code lock for 5.0 but this is totally 6.0 material

[carolhmj]

+1 for AWESOME FEATURE! I'll love to see it in next release 😄

[sebavan]

I love it a lot !!!! the main issue I am seeing is that it requires the full inspector which will turn out to force loading all of Babylon creating a huge dependency. I bet @RaananW can help with this part as this component should be easily embeddable without pulling all of the framework.

[RaananW]

Sorry, didn't find time for review today, I'll go over this tomorrow.

[deltakosh]

No rush, we will not merge until after the release

[deltakosh]

I guess we can think about continuing this thread

[azure-pipelines]

Please make sure to tag your PR with "bug", "new feature" or "breaking change" tags.

[azure-pipelines]

Reviewer - this PR has made changes to one or more package.json files.

[azure-pipelines]

Snapshot stored with reference name:
refs/pull/12074/merge

Test environment:
https://babylonsnapshots.z22.web.core.windows.net/refs/pull/12074/merge/index.html

To test a playground add it to the URL, for example:

https://babylonsnapshots.z22.web.core.windows.net/refs/pull/12074/merge/index.html#WGZLGJ#4600

To test the snapshot in the playground itself use (for example):

https://playground.babylonjs.com/?snapshot=refs/pull/12074/merge#BCU1XR#0

[mysunnytime]

I updated it so it's now in its own package, and it support GUI (TextBlock and Button only for now). It can now also update when scene updates (add/remove node and GUI control, enable/disable node, visible/invisible GUI control). I think it's in a good shape to be reviewed :) @RaananW, @sebavan, @deltakosh

[deltakosh]

Please bear with us as @RaananW is OOF and I'm expecting him to give us the green light :)

[mysunnytime]

(Answering David's question about why changing core and then have another package for accessibility) Good question! The idea is that the core can allow tagging accessibility metadata to nodes, that developers can assign this metadata and later use this metadata for customized accessibility. And then there's an accessibility package, specifically providing methods to render accessibility DOM for a scene. Right now, it just renders every accessibility object in the scene to DOM, as if we are flattening a 3D scene into a readable 2D document, even some objects might be behind the camera. This will work great for more static style applications like gallery, information display, but won't work that much for applications where the user have to move around, like multi-user meeting app, because it loses the sense of spatial. Later we can extend the package so it can provide another rendering method that represents the user's view when the user moves, e.g. it can render DOM for objects that's inside the current camera view. But with the accessibility tag (change in the core), the developer should be able to customize their own renderer already. Does that make sense? I wonder whether we should change the package to AccessibilityHelper or AccessibilityRenderer to reduce confusion?

…

On Thu, 21 Apr 2022 at 16:19, David Catuhe ***@***.***> wrote: Some questions on my side: - Why using GUI? I believe the goal is to generate DOM elements right? - DO you see this as part of the framework ? (I see it is decorrelated here). THen it will be an independent NPM package? — Reply to this email directly, view it on GitHub <#12074 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/ABJXMNVW5F26UAMSRJP6RB3VGHO6LANCNFSM5PV2POOA> . You are receiving this because you authored the thread.Message ID: ***@***.***>

[azure-pipelines]

Please make sure to tag your PR with "bug", "new feature" or "breaking change" tags.

[azure-pipelines]

Reviewer - this PR has made changes to one or more package.json files.

[azure-pipelines]

Snapshot stored with reference name:
refs/pull/12074/merge

Test environment:
https://babylonsnapshots.z22.web.core.windows.net/refs/pull/12074/merge/index.html

To test a playground add it to the URL, for example:

https://babylonsnapshots.z22.web.core.windows.net/refs/pull/12074/merge/index.html#WGZLGJ#4600

Links to test babylon tools with this snapshot:

https://playground.babylonjs.com/?snapshot=refs/pull/12074/merge
https://sandbox.babylonjs.com/?snapshot=refs/pull/12074/merge
https://gui.babylonjs.com/?snapshot=refs/pull/12074/merge
https://nme.babylonjs.com/?snapshot=refs/pull/12074/merge

To test the snapshot in the playground with a playground ID add it after the snapshot query string:

https://playground.babylonjs.com/?snapshot=refs/pull/12074/merge#BCU1XR#0

[RaananW]

Oh, this is awesome!!! Can't wait to have it as part of the framework :-)

I have a few general questions/suggestions (and will be happy to discuss them all)

1. I think the better place for this would be the tools directory and not the dev directory, as this is a tool using the core libs.
2. The role definition of the component holding the data can be a bit confusing. Both Control and Node should have the accessibility tag added to them, allowing the dev to extend the corresponding (HTML) elements. Those tags can be cached and updated in the React implementation when changed. I wrote in one of the comments that I think the cache and "changed" functionality should be a part of the node/control (or maybe even its own "accessibility class" that will take care of that for both classes as part of core?). This way the dev could change the a11y data when the app is already running and will always see the changes propagate to the react implementation (or any other implementation that will use this API).
3. You are using a lot of instanceof, which can be a little unreliable (and a bit slow, though the differences are not that extreme). If possible, try using feature-detection or use the getClassName function (if possible, of course).

A bit philosophical, but I think that, in general, screen readers don't know what 3D content is. They understand 2D very well, and this is a kind of 3D-to-2D converter. My question here is whether the order (and "level") should be controlled by something other than the tree-structure? it makes sense in GUI (which is "2D" in its nature), but might not fit the node implementation. TBH - I don't have a good answer as to what is better here, but will be happy to talk about it :-)

And in general - I believe the React implementation should use the provided observables in a more dynamic way instead of reading the entire scene structure on every change. So when a node (or control) is added, a new component should be created, and this component should attach to the corresponding node/control's observables to decide whether or not its a11y tag should be rendered. In this case it would also make sense to have a "onA11YTagUpdatedObservable` (and maybe added/removed as well? though "changed" might be enough) and attach to it. This way changes in the scene graph will automatically propagate to the react implementation in a more performant way (instead of checking all nodes for changes on each frame).

[azure-pipelines]

Reviewer - this PR has made changes to one or more package.json files.

[azure-pipelines]

Reviewer - this PR has made changes to one or more package.json files.

[azure-pipelines]

Reviewer - this PR has made changes to one or more package.json files.

[azure-pipelines]

Reviewer - this PR has made changes to one or more package.json files.

[azure-pipelines]

Visualization tests for webgl1 have failed. If some tests failed because the snapshots do not match, the report can be found at

https://babylonsnapshots.z22.web.core.windows.net/refs/pull/12074/merge/testResults/webgl1/index.html

If tests were successful afterwards, this report might not be available anymore.

[RaananW]

Is it now ready for final review and merge?

[azure-pipelines]

Reviewer - this PR has made changes to one or more package.json files.

[azure-pipelines]

Reviewer - this PR has made changes to one or more package.json files.

[azure-pipelines]

Visualization tests for webgl1 have failed. If some tests failed because the snapshots do not match, the report can be found at

https://babylonsnapshots.z22.web.core.windows.net/refs/pull/12074/merge/testResults/webgl1/index.html

If tests were successful afterwards, this report might not be available anymore.

[carolhmj]

🥳

[sebavan]

❤️‍🔥

[PatrickRyanMS]

The Readme and examples look great!

[azure-pipelines]

Visualization tests for webgl1 have failed. If some tests failed because the snapshots do not match, the report can be found at

https://babylonsnapshots.z22.web.core.windows.net/refs/pull/12074/merge/testResults/webgl1/index.html

If tests were successful afterwards, this report might not be available anymore.

[azure-pipelines]

Visualization tests for webgl1 have failed. If some tests failed because the snapshots do not match, the report can be found at

https://babylonsnapshots.z22.web.core.windows.net/refs/pull/12074/merge/testResults/webgl1/index.html

If tests were successful afterwards, this report might not be available anymore.

[azure-pipelines]

Reviewer - this PR has made changes to one or more package.json files.

[azure-pipelines]

Reviewer - this PR has made changes to one or more package.json files.

[azure-pipelines]

Reviewer - this PR has made changes to one or more package.json files.