Add anisotropy tests to render-fidelity-tests (and update Babylon + F…

…ilament to latest) (#4535)

* init submodule for glTF-Sample-Assets.

* fix left over popd in fetch khronos gltf-smaples.

* remove tests from config.json that no longer have corresponding assets in glTF-Sample-Assets

* upgrade babylon + filament, check in their latest goldens for new anistropy tests

* update config.json with the new anisotropy tests.

* add more time for tests.

* add more gltf tests along with goldens for filament,babylon and model-viewer.

* using checkout with submodule support.

* using checkout with submodule support.

* update goldens for model-viewer + filament for khronos-TextureTransformMultiTest.

* update goldens for model-viewer + filament for khronos-TextureTransformMultiTest.

* add more goldens for anisotropy from gltf-sample-viewer and three-gpu-pathtracer

* add more goldens for anisotropy from gltf-sample-viewer and three-gpu-pathtracer

* exclude a few renderers for now to get the PR accepted.

* exclude a few renderers for now to get the PR accepted.

* add gltf-sample viewer goldens for new tests.

* add goldens for AnisotropyDiscTest

* remove barn lamp test for now.

* remove blank golden image.

* remove barn lamp test for now.

* exclude three-gpu-pathtracer from rotation test.

* exclude three-gpu-pathtracer from rotation test.

* remove failing test for now.

* add missing anisotropy goldens, and re-add barn lamp with model-viewer exclusion.

* ensure goldens tests are excluded if renderer is excluded.

* remove blank image.

packages/render-fidelity-tools/package.json

@@ -23,8 +23,8 @@
   },
   "dependencies": {
     "@actions/core": "^1.2.4",
-    "@babylonjs/core": "^5.44.0",
-    "@babylonjs/loaders": "^5.44.0",
+    "@babylonjs/core": "^6.25.0",
+    "@babylonjs/loaders": "^6.25.0",
     "@google/model-viewer": "^3.3.0",
     "@khronosgroup/gltf-viewer": "^1.0.9",
     "@polymer/paper-button": "^3.0.1",
@@ -36,7 +36,7 @@
     "@types/puppeteer": "^5.4.6",
     "@types/rimraf": "^3.0.1",
     "@types/three": "^0.156.0",
-    "filament": "1.31.5",
+    "filament": "1.44.0",
     "http-server": "^14.1.1",
     "lit": "^2.7.2",
     "mkdirp": "^1.0.4",

packages/render-fidelity-tools/src/artifact-creator.ts

@@ -68,6 +68,11 @@ export class ArtifactCreator {
     const {rootDirectory, outputDirectory, goldens} = this;
     const {name: scenarioName, exclude, dimensions} = scenario;
 
+    // skip if this renderer is excluded from this scenario
+    if(exclude != null && exclude.includes(renderer) ) {
+      return;
+    }
+
     console.log(
         `Start to compare ${renderer}'s golden with other renderers' goldens:`);
 
@@ -200,10 +205,16 @@ export class ArtifactCreator {
       const scenarioName = scenarioBase.name;
       const scenario = this[$configReader].scenarioConfig(scenarioName)!;
 
+      // skip if white lists exists and this scenario isn't in it
       if (scenarioWhitelist != null && !scenarioWhitelist.has(scenarioName)) {
         continue;
       }
 
+      // skip if this renderer is excluded from this scenario
+      if(scenario.exclude != null && scenario.exclude.includes(renderer) ) {
+        continue;
+      }
+
       console.log(`\n🎨 Scenario: ${scenarioName}`);
 
       const scenarioOutputDirectory = join(outputDirectory, scenarioName);
@@ -241,6 +252,9 @@ export class ArtifactCreator {
         this.config,
         {scenarios: analyzedScenarios, errors: compareRenderersErrors});
 
+    // ensure directory exists, this can happen if all scenarios are ignored
+    await fs.mkdir(outputDirectory, {recursive: true});
+
     await fs.writeFile(
         join(outputDirectory, 'config.json'), JSON.stringify(finalConfig));
 

packages/render-fidelity-tools/test/config.json

@@ -257,7 +257,8 @@
       },
       "exclude": [
         "stellar",
-        "vray"
+        "vray",
+        "three-gpu-pathtracer"
       ]
     },
     {
@@ -407,6 +408,37 @@
         "three-gpu-pathtracer"
       ]
     },
+    {
+      "name": "khronos-GlassHurricaneCandleHolder",
+      "model": "../../../shared-assets/models/glTF-Sample-Assets/Models/GlassHurricaneCandleHolder/glTF-Binary/GlassHurricaneCandleHolder.glb",
+      "target": {
+        "y": 0.13
+      },
+      "orbit": {
+        "radius": 0.75
+      },
+      "exclude": [
+        "stellar",
+        "vray",
+        "three-gpu-pathtracer"
+      ]
+    },
+    {
+      "name": "khronos-GlassVaseFlowers",
+      "model": "../../../shared-assets/models/glTF-Sample-Assets/Models/GlassVaseFlowers/glTF-Binary/GlassVaseFlowers.glb",
+      "target": {
+        "y": 0.1,
+        "x": 0.05
+      },
+     "orbit": {
+        "radius": 0.5
+      },
+      "exclude": [
+        "stellar",
+        "vray",
+        "three-gpu-pathtracer"
+      ]
+    },
     {
       "name": "khronos-Sponza",
       "model": "../../../shared-assets/models/glTF-Sample-Assets/Models/Sponza/glTF/Sponza.gltf",
@@ -608,13 +640,40 @@
         "vray"
       ]
     },
+    {
+      "name": "khronos-ClearCoatCarPaint",
+      "model": "../../../shared-assets/models/glTF-Sample-Assets/Models/ClearCoatCarPaint/glTF/ClearCoatCarPaint.gltf",
+      "target": {
+        "y": 0.5
+      },
+      "orbit": {
+        "radius": 2
+      },
+      "exclude": [
+        "stellar",
+        "vray",
+        "three-gpu-pathtracer"
+      ]
+    },
     {
       "name": "khronos-ClearCoatTest",
       "model": "../../../shared-assets/models/glTF-Sample-Assets/Models/ClearCoatTest/glTF/ClearCoatTest.gltf",
       "orbit": {
         "radius": 20
       }
     },
+    {
+      "name": "khronos-NegativeScaleTest",
+      "model": "../../../shared-assets/models/glTF-Sample-Assets/Models/NegativeScaleTest/glTF/NegativeScaleTest.gltf",
+      "orbit": {
+        "radius": 13
+      },
+      "exclude": [
+        "stellar",
+        "vray",
+        "three-gpu-pathtracer"
+      ]
+    },
     {
       "name": "khronos-NormalTangentMirrorTest",
       "model": "../../../shared-assets/models/glTF-Sample-Assets/Models/NormalTangentMirrorTest/glTF/NormalTangentMirrorTest.gltf",
@@ -689,6 +748,49 @@
         "stellar"
       ]
     },
+    {
+      "name": "khronos-AnisotropyBarnLamp",
+      "model": "../../../shared-assets/models/glTF-Sample-Assets/Models/AnisotropyBarnLamp/glTF/AnisotropyBarnLamp.gltf",
+      "target": {
+        "y": -0.10
+      },
+      "orbit": {
+        "radius": 0.5
+      },
+      "exclude": [
+        "stellar",
+        "vray",
+        "model-viewer",
+        "three-gpu-pathtracer"
+      ]
+    },
+    {
+      "name": "khronos-AnisotropyRotationTest",
+      "model": "../../../shared-assets/models/glTF-Sample-Assets/Models/AnisotropyRotationTest/glTF/AnisotropyRotationTest.gltf",
+      "orbit": {
+        "radius": 18
+      },
+      "exclude": [
+        "stellar",
+        "vray",
+        "three-gpu-pathtracer"
+      ]
+    },
+    {
+      "name": "khronos-AnisotropyStrengthTest",
+      "model": "../../../shared-assets/models/glTF-Sample-Assets/Models/AnisotropyStrengthTest/glTF/AnisotropyStrengthTest.gltf",
+      "target": {
+        "y": 2.5
+      },
+      "orbit": {
+        "radius": 12
+      },
+      "exclude": [
+        "stellar",
+        "vray",
+        "three-gpu-pathtracer"
+      ]
+    },
     {
       "name": "khronos-TextureTransformMultiTest",
       "model": "../../../shared-assets/models/glTF-Sample-Assets/Models/TextureTransformMultiTest/glTF-Binary/TextureTransformMultiTest.glb",