Per-case quality & failure details
[
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-menu-0",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "Corner Cafe",
"actual": "Corner Cafe",
"neuralExecuted": true,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 900.1999999880791,
"inferenceMs": 305.30000001192093,
"totalMs": 1205.5,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": true,
"imagesPerSecond": 3.275466753884551,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "good",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-menu-1",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "Espresso 3.20",
"actual": "Espresso 3.20",
"neuralExecuted": true,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 859.1999999880791,
"inferenceMs": 305.60000002384186,
"totalMs": 1164.800000011921,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": true,
"imagesPerSecond": 3.2722513086452336,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "good",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-menu-1-blur",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "Espresso 3.20",
"actual": "Espresso 3.20",
"neuralExecuted": true,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 850,
"inferenceMs": 296.90000000596046,
"totalMs": 1146.9000000059605,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": true,
"imagesPerSecond": 3.3681374199391185,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "good",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-menu-2",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "Cappuccino 4.10",
"actual": "Cappuccino 4.10",
"neuralExecuted": true,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 861.3999999761581,
"inferenceMs": 322.5,
"totalMs": 1183.8999999761581,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": true,
"imagesPerSecond": 3.10077519379845,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "good",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-menu-3",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "Green tea 3.50",
"actual": "Green tea 3.50",
"neuralExecuted": true,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 857.0999999940395,
"inferenceMs": 332.19999998807907,
"totalMs": 1189.2999999821186,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": true,
"imagesPerSecond": 3.01023479842229,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "good",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-menu-4",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "Open 8:00-18:00",
"actual": "Open 8:00-18:00",
"neuralExecuted": true,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 870.9000000059605,
"inferenceMs": 318.5,
"totalMs": 1189.4000000059605,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": true,
"imagesPerSecond": 3.1397174254317113,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "good",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-menu-4-blur",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "Open 8:00-18:00",
"actual": "Open 8:00-18:00",
"neuralExecuted": true,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 852,
"inferenceMs": 296.10000002384186,
"totalMs": 1148.1000000238419,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": true,
"imagesPerSecond": 3.3772374195186776,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "good",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-menu-5",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "Free Wi-Fi inside",
"actual": "Free Wi-Fi inside",
"neuralExecuted": true,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 867.5999999940395,
"inferenceMs": 375.69999998807907,
"totalMs": 1243.2999999821186,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": true,
"imagesPerSecond": 2.661698163512723,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "good",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-faint-line",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "LocalAI Tools",
"actual": "LocalAI Tools",
"neuralExecuted": true,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 840.8000000119209,
"inferenceMs": 298.19999998807907,
"totalMs": 1139,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": true,
"imagesPerSecond": 3.3534540578134684,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "good",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-blank-white",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "",
"actual": "",
"neuralExecuted": false,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 830.8999999761581,
"inferenceMs": 9.700000017881393,
"totalMs": 840.5999999940395,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": false,
"imagesPerSecond": 103.092783315109,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "uniform-shortcut",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-blank-gray",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "",
"actual": "",
"neuralExecuted": false,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 844.6999999880791,
"inferenceMs": 10.200000017881393,
"totalMs": 854.9000000059605,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": false,
"imagesPerSecond": 98.03921551440413,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "uniform-shortcut",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-blank-color",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "",
"actual": "",
"neuralExecuted": false,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 844,
"inferenceMs": 9.5,
"totalMs": 853.5,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": false,
"imagesPerSecond": 105.26315789473684,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "uniform-shortcut",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "fail",
"quality": {
"caseId": "rec-webgpu-menu-0",
"role": "rec",
"device": "webgpu",
"passed": false,
"reason": "Error: Can't create a session. ERROR_CODE: 1, ERROR_MESSAGE: This session contains graph nodes that are assigned to the default CPU EP, but fallback to CPU EP has been explicitly disabled by the user.\n at worker.onmessage (eval at evaluate (:291:30), <anonymous>:1:484)",
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"grade": "unsupported",
"requestedDevice": "webgpu",
"actualDevice": null,
"reason": "Error: Can't create a session. ERROR_CODE: 1, ERROR_MESSAGE: This session contains graph nodes that are assigned to the default CPU EP, but fallback to CPU EP has been explicitly disabled by the user.\n at worker.onmessage (eval at evaluate (:291:30), <anonymous>:1:484)"
},
"failure": "Error: Can't create a session. ERROR_CODE: 1, ERROR_MESSAGE: This session contains graph nodes that are assigned to the default CPU EP, but fallback to CPU EP has been explicitly disabled by the user.\n at worker.onmessage (eval at evaluate (:291:30), <anonymous>:1:484)"
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-faint-holdout-0",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "Invoice 4821",
"actual": "Invoice 4821",
"neuralExecuted": true,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 873.5999999940395,
"inferenceMs": 301.2000000178814,
"totalMs": 1174.800000011921,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": true,
"imagesPerSecond": 3.3200531206528314,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "good",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"caseId": "rec-wasm-faint-holdout-1",
"role": "rec",
"device": "wasm",
"passed": true,
"metric": "characterErrorRate",
"score": 0,
"expected": "Backup complete",
"actual": "Backup complete",
"neuralExecuted": true,
"gibberish": false,
"scope": "Frozen English synthetic/menu fixtures; no broad language or app performance certification."
},
"runtime": {
"loadMs": 852.5,
"inferenceMs": 307.80000001192093,
"totalMs": 1160.300000011921,
"device": "wasm",
"actualDevice": "wasm",
"precision": "fp32",
"fallback": false,
"strictGpuNoCpuFallback": false,
"adapterInfo": null,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"requestedDevice": "wasm",
"neuralExecuted": true,
"imagesPerSecond": 3.2488628978598784,
"qualityMetric": "characterErrorRate",
"qualityScore": 0,
"grade": "good",
"loadScope": "Fresh module worker and checksum-verified locally served snapshot,not an internet transfer benchmark."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"metric": "exactText",
"score": 1,
"expected": "Corner Cafe"
},
"runtime": {
"loadMs": 671.8000000119209,
"inferenceMs": 298.09999999403954,
"totalMs": 969.9000000059605,
"device": "wasm",
"actualDevice": "wasm",
"fallback": false,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"neuralExecuted": true,
"imagesPerSecond": 3.354579000402532,
"qualityMetric": "exactText",
"qualityScore": 1,
"grade": "good",
"loadScope": "Public API init after a verified local-snapshot download."
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"metric": "exactText",
"score": 1,
"expected": "Espresso 3.20"
},
"runtime": {
"loadMs": 0,
"inferenceMs": 261.2000000178814,
"totalMs": 261.2000000178814,
"device": "wasm",
"actualDevice": "wasm",
"fallback": false,
"runtimeVersion": "1.24.3",
"artifactRevision": "b70df217f4fd99d14f970bad092cebe7d74cc4d1",
"neuralExecuted": true,
"imagesPerSecond": 3.828483920105442,
"qualityMetric": "exactText",
"qualityScore": 1,
"grade": "good",
"loadScope": "Public API init after a verified local-snapshot download."
}
}
]