EttinX NLI XS — 32M Cross-Encoder
58 configurations testedQuality criteria
All six frozen cases must have the expected label and per-case minimum margin with finite normalized complete three-class probabilities. XS original three-case evidence reused with original times, plus independent three added cases; no sibling precision/backend inheritance.
| Precision | Device | Result | |
|---|---|---|---|
Q4 | GPU · WebGPUAccelerated | Passed | Very fast |
FP32 | GPU · WebGPUAccelerated | Passed | Very fast |
Q4 | GPU · WebGPUAccelerated | Passed | Very fast |
BNB4 | GPU · WebGPUAccelerated | Passed | Very fast |
BNB4 | GPU · WebGPUAccelerated | Passed | Very fast |
FP32 | GPU · WebGPUAccelerated | Passed | Very fast |
UINT8 | GPU · WebGPUAccelerated | Passed | Very fast |
Q8 | GPU · WebGPUAccelerated | Passed | Very fast |
INT8 | GPU · WebGPUAccelerated | Passed | Very fast |
UINT8 | GPU · WebGPUAccelerated | Passed | Very fast |
Q8 | GPU · WebGPUAccelerated | Passed | Very fast |
INT8 | GPU · WebGPUAccelerated | Passed | Very fast |
FP32 | GPU · WebGPUAccelerated | Passed | Very fast |
BNB4 | GPU · WebGPUAccelerated | Passed | Very fast |
Q4 | GPU · WebGPUAccelerated | Passed | Very fast |
BNB4 | GPU · WebGPUAccelerated | Passed | Fast |
Q8 | GPU · WebGPUAccelerated | Passed | Fast |
INT8 | GPU · WebGPUAccelerated | Passed | Fast |
UINT8 | GPU · WebGPUAccelerated | Passed | Fast |
Q8 | GPU · WebGPUAccelerated | Passed | Fast |
UINT8 | GPU · WebGPUAccelerated | Passed | Fast |
Q4 | GPU · WebGPUAccelerated | Passed | Fast |
INT8 | GPU · WebGPUAccelerated | Passed | Fast |
FP32 | GPU · WebGPUAccelerated | Passed | Fast |
FP32 | CPU · WASM | Passed | Very fast |
FP32 | CPU · WASM | Passed | Very fast |
Q8 | CPU · WASM | Passed | Very fast |
INT8 | CPU · WASM | Passed | Very fast |
Q8 | CPU · WASM | Passed | Very fast |
UINT8 | CPU · WASM | Passed | Very fast |
INT8 | CPU · WASM | Passed | Very fast |
Q8 | CPU · WASM | Passed | Very fast |
Q8 | CPU · WASM | Passed | Very fast |
Q8 | CPU · WASM | Passed | Very fast |
UINT8 | CPU · WASM | Passed | Very fast |
Q8 | CPU · WASM | Passed | Very fast |
Q8 | CPU · WASM | Passed | Very fast |
BNB4 | CPU · WASM | Passed | Very fast |
BNB4 | CPU · WASM | Passed | Very fast |
Q4 | CPU · WASM | Passed | Very fast |
Q4 | CPU · WASM | Passed | Very fast |
FP32 | CPU · WASM | Passed | Very fast |
Q8 | CPU · WASM | Passed | Very fast |
Q8 | CPU · WASM | Passed | Very fast |
INT8 | CPU · WASM | Passed | Very fast |
UINT8 | CPU · WASM | Passed | Very fast |
Q4 | CPU · WASM | Passed | Very fast |
BNB4 | CPU · WASM | Passed | Very fast |
Q8 | CPU · WASM | Passed | Very fast |
FP32 | CPU · WASM | Passed | Very fast |
INT8 | CPU · WASM | Passed | Very fast |
UINT8 | CPU · WASM | Passed | Very fast |
BNB4 | CPU · WASM | Passed | Fast |
Q4 | CPU · WASM | Passed | Fast |
FP16 | CPU · WASM | Failed | — |
FP16 | GPU · WebGPU | Failed | — |
Q4F16 | CPU · WASM | Failed | — |
Q4F16 | GPU · WebGPU | Failed | — |
Test setup & sample details
- Quality gate
- All six frozen cases must have the expected label and per-case minimum margin with finite normalized complete three-class probabilities. XS original three-case evidence reused with original times, plus independent three added cases; no sibling precision/backend inheritance.
- Tested
- 2026-09-15T12:52:16.195Z
- Environment
- Isolated headless Chrome on macOS; Transformers.js 4.2.0
- Runtime
- Transformers.js 4.2.0
- Artifact revision
- cf5995a136f48843191ba185bd3430e18b0ffc2f
- Fixture
{ "file": "model-samples/browser/ettinx-nli/quality-gates.json", "url": "https://assets.localai.tools/samples/v1/files/85a255ca69a81f4851b7bb6dfe9bdd69be90fb6e769760ea6de1d3b4e2ddc207/ettinx-nli-quality-gates.json", "size": 2264, "sha1": "5fb79c4a66c12418e28abe244fd155713e8bd891", "sha256": "85a255ca69a81f4851b7bb6dfe9bdd69be90fb6e769760ea6de1d3b4e2ddc207" }- Rerun when
- artifact revision change; runtime version change; fixture or quality-gate change; browser/backend behavior change
Per-case quality & failure details
[
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"cases": [
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9348974209781319,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "contradiction",
"actual": "contradiction",
"finite": true,
"margin": 0.9997462882602456,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9828258181954121,
"minimumMargin": 0.15,
"pass": true
}
],
"grade": "nli-pass"
},
"runtime": {
"loadMs": 9634.600000023842,
"inferenceMs": 69.7999999821186,
"totalMs": 9798,
"grade": "excellent",
"pairsPerSecond": 42.97994270442036
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "pass",
"quality": {
"cases": [
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9348975111783933,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "contradiction",
"actual": "contradiction",
"finite": true,
"margin": 0.9997462882143131,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.982825985060039,
"minimumMargin": 0.15,
"pass": true
}
],
"grade": "nli-pass"
},
"runtime": {
"loadMs": 10304.59999999404,
"inferenceMs": 317.80000001192093,
"totalMs": 10722.09999999404,
"grade": "good",
"pairsPerSecond": 9.43989930738662
}
},
{
"precision": "fp16",
"device": "wasm",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": 5229.300000011921,
"grade": "failed"
},
"failure": "Error: Can't create a session. ERROR_CODE: 1, ERROR_MESSAGE: Type Error: Type (tensor(float16)) of output arg (/Cast_1_output_0) of node (/Cast_1) does not match expected type (tensor(float))."
},
{
"precision": "fp16",
"device": "webgpu",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": 5037.899999976158,
"grade": "failed"
},
"failure": "Error: Can't create a session. ERROR_CODE: 1, ERROR_MESSAGE: Type Error: Type (tensor(float16)) of output arg (/Cast_1_output_0) of node (/Cast_1) does not match expected type (tensor(float))."
},
{
"precision": "int8",
"device": "wasm",
"status": "pass",
"quality": {
"cases": [
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9421899525497719,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "contradiction",
"actual": "contradiction",
"finite": true,
"margin": 0.9992550970649349,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9708945043089913,
"minimumMargin": 0.15,
"pass": true
}
],
"grade": "nli-pass"
},
"runtime": {
"loadMs": 4269.4000000059605,
"inferenceMs": 71,
"totalMs": 4523.299999982119,
"grade": "excellent",
"pairsPerSecond": 42.25352112676056
}
},
{
"precision": "int8",
"device": "webgpu",
"status": "pass",
"quality": {
"cases": [
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9421898516197966,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "contradiction",
"actual": "contradiction",
"finite": true,
"margin": 0.9992550970649349,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9708945164656116,
"minimumMargin": 0.15,
"pass": true
}
],
"grade": "nli-pass"
},
"runtime": {
"loadMs": 4394.0999999940395,
"inferenceMs": 289.90000000596046,
"totalMs": 4780.5,
"grade": "good",
"pairsPerSecond": 10.348395998407447
}
},
{
"precision": "uint8",
"device": "wasm",
"status": "pass",
"quality": {
"cases": [
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9077236297893401,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "contradiction",
"actual": "contradiction",
"finite": true,
"margin": 0.999403466661996,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9694324941237493,
"minimumMargin": 0.15,
"pass": true
}
],
"grade": "nli-pass"
},
"runtime": {
"loadMs": 4948.199999988079,
"inferenceMs": 71.5,
"totalMs": 5160.0999999940395,
"grade": "excellent",
"pairsPerSecond": 41.95804195804196
}
},
{
"precision": "uint8",
"device": "webgpu",
"status": "pass",
"quality": {
"cases": [
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9077237456137992,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "contradiction",
"actual": "contradiction",
"finite": true,
"margin": 0.9994034653005069,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9684620590199027,
"minimumMargin": 0.15,
"pass": true
}
],
"grade": "nli-pass"
},
"runtime": {
"loadMs": 4201.5,
"inferenceMs": 206.90000000596046,
"totalMs": 4581.100000023842,
"grade": "good",
"pairsPerSecond": 14.499758336943328
}
},
{
"precision": "q8",
"device": "wasm",
"status": "pass",
"quality": {
"cases": [
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9421899525497719,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "contradiction",
"actual": "contradiction",
"finite": true,
"margin": 0.9992550970649349,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9708945043089913,
"minimumMargin": 0.15,
"pass": true
}
],
"grade": "nli-pass"
},
"runtime": {
"loadMs": 4252.800000011921,
"inferenceMs": 68.80000001192093,
"totalMs": 4427.699999988079,
"grade": "excellent",
"pairsPerSecond": 43.604651155235345
}
},
{
"precision": "q8",
"device": "webgpu",
"status": "pass",
"quality": {
"cases": [
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9421898516197966,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "contradiction",
"actual": "contradiction",
"finite": true,
"margin": 0.9992550970649349,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9708945164656116,
"minimumMargin": 0.15,
"pass": true
}
],
"grade": "nli-pass"
},
"runtime": {
"loadMs": 4123.800000011921,
"inferenceMs": 202.19999998807907,
"totalMs": 4412.5,
"grade": "good",
"pairsPerSecond": 14.836795253100238
}
},
{
"precision": "q4",
"device": "wasm",
"status": "pass",
"quality": {
"cases": [
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.8586604888902396,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "contradiction",
"actual": "contradiction",
"finite": true,
"margin": 0.9995681430491034,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.931758709483561,
"minimumMargin": 0.15,
"pass": true
}
],
"grade": "nli-pass"
},
"runtime": {
"loadMs": 25404.70000001788,
"inferenceMs": 106.19999998807907,
"totalMs": 29241,
"grade": "excellent",
"pairsPerSecond": 28.248587573792367
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"cases": [
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.8586619773056341,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "contradiction",
"actual": "contradiction",
"finite": true,
"margin": 0.9995681412532368,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9317590704410151,
"minimumMargin": 0.15,
"pass": true
}
],
"grade": "nli-pass"
},
"runtime": {
"loadMs": 18898.5,
"inferenceMs": 288.40000000596046,
"totalMs": 21088.30000001192,
"grade": "good",
"pairsPerSecond": 10.402219139868231
}
},
{
"precision": "q4f16",
"device": "wasm",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": 16376.5,
"grade": "failed"
},
"failure": "Error: Can't create a session. ERROR_CODE: 1, ERROR_MESSAGE: Type Error: Type (tensor(float16)) of output arg (/Cast_1_output_0) of node (/Cast_1) does not match expected type (tensor(float))."
},
{
"precision": "q4f16",
"device": "webgpu",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": 15038.299999982119,
"grade": "failed"
},
"failure": "Error: Can't create a session. ERROR_CODE: 1, ERROR_MESSAGE: Type Error: Type (tensor(float16)) of output arg (/Cast_1_output_0) of node (/Cast_1) does not match expected type (tensor(float))."
},
{
"precision": "bnb4",
"device": "wasm",
"status": "pass",
"quality": {
"cases": [
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.8586604888902396,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "contradiction",
"actual": "contradiction",
"finite": true,
"margin": 0.9995681430491034,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.931758709483561,
"minimumMargin": 0.15,
"pass": true
}
],
"grade": "nli-pass"
},
"runtime": {
"loadMs": 19335.40000000596,
"inferenceMs": 105.40000000596046,
"totalMs": 20751.19999998808,
"grade": "excellent",
"pairsPerSecond": 28.462998100857185
}
},
{
"precision": "bnb4",
"device": "webgpu",
"status": "pass",
"quality": {
"cases": [
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.8586619773056341,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "contradiction",
"actual": "contradiction",
"finite": true,
"margin": 0.9995681412532368,
"minimumMargin": 0.3,
"pass": true
},
{
"expected": "entailment",
"actual": "entailment",
"finite": true,
"margin": 0.9317590704410151,
"minimumMargin": 0.15,
"pass": true
}
],
"grade": "nli-pass"
},
"runtime": {
"loadMs": 20732.09999999404,
"inferenceMs": 113.90000000596046,
"totalMs": 22250.600000023842,
"grade": "excellent",
"pairsPerSecond": 26.338893765083476
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9937077714710486,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1908.9000000059605,
"inferenceMs": 31.099999994039536,
"totalMs": 1940,
"pairsPerSecond": 32.154340842175394,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "fp32",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "contradiction",
"actualLabel": "contradiction",
"margin": 0.9895947333553039,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1908.9000000059605,
"inferenceMs": 10.299999982118607,
"totalMs": 1919.199999988079,
"pairsPerSecond": 97.08737880932598,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "fp32",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9861083342119112,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1908.9000000059605,
"inferenceMs": 10.5,
"totalMs": 1919.4000000059605,
"pairsPerSecond": 95.23809523809524,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "fp32",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9937077683645019,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1958.9000000059605,
"inferenceMs": 85.09999999403954,
"totalMs": 2044,
"pairsPerSecond": 11.750881316921747,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "fp32",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "contradiction",
"actualLabel": "contradiction",
"margin": 0.9895947361146832,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1958.9000000059605,
"inferenceMs": 28.600000023841858,
"totalMs": 1987.5000000298023,
"pairsPerSecond": 34.96503493588701,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "fp32",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9861082919168208,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1958.9000000059605,
"inferenceMs": 21.099999994039536,
"totalMs": 1980,
"pairsPerSecond": 47.393364942297936,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "fp32",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "int8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9763729440912856,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 916.3999999761581,
"inferenceMs": 34.29999998211861,
"totalMs": 950.6999999582767,
"pairsPerSecond": 29.154518965636253,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "int8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "int8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "contradiction",
"actualLabel": "contradiction",
"margin": 0.9838062331929446,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 916.3999999761581,
"inferenceMs": 11.5,
"totalMs": 927.8999999761581,
"pairsPerSecond": 86.95652173913044,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "int8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "int8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.978804847561861,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 916.3999999761581,
"inferenceMs": 11.799999982118607,
"totalMs": 928.1999999582767,
"pairsPerSecond": 84.74576284028579,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "int8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "int8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9762451474043524,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 960,
"inferenceMs": 125.69999998807907,
"totalMs": 1085.699999988079,
"pairsPerSecond": 7.955449483650249,
"grade": "good",
"actualDevice": "webgpu",
"actualPrecision": "int8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "int8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "contradiction",
"actualLabel": "contradiction",
"margin": 0.9838062150472762,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 960,
"inferenceMs": 58.30000001192093,
"totalMs": 1018.3000000119209,
"pairsPerSecond": 17.152658658585324,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "int8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "int8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9788048684966374,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 960,
"inferenceMs": 52.900000005960464,
"totalMs": 1012.9000000059605,
"pairsPerSecond": 18.90359168028971,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "int8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "uint8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9933587858816335,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1007.6999999880791,
"inferenceMs": 34.900000005960464,
"totalMs": 1042.5999999940395,
"pairsPerSecond": 28.65329512404622,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "uint8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "uint8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "contradiction",
"actualLabel": "contradiction",
"margin": 0.9152550423406945,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1007.6999999880791,
"inferenceMs": 11.600000023841858,
"totalMs": 1019.3000000119209,
"pairsPerSecond": 86.20689637454029,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "uint8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "uint8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9879818206585347,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1007.6999999880791,
"inferenceMs": 12.299999982118607,
"totalMs": 1019.9999999701977,
"pairsPerSecond": 81.30081312632291,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "uint8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "uint8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9933588004329331,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 981.1999999880791,
"inferenceMs": 129.2999999821186,
"totalMs": 1110.4999999701977,
"pairsPerSecond": 7.733952050566851,
"grade": "good",
"actualDevice": "webgpu",
"actualPrecision": "uint8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "uint8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "contradiction",
"actualLabel": "contradiction",
"margin": 0.9152550219595118,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 981.1999999880791,
"inferenceMs": 53.60000002384186,
"totalMs": 1034.800000011921,
"pairsPerSecond": 18.65671640961174,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "uint8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "uint8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9879818161374019,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 981.1999999880791,
"inferenceMs": 49.29999998211861,
"totalMs": 1030.4999999701977,
"pairsPerSecond": 20.283975666586322,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "uint8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "q8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9763729440912856,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 906.4000000059605,
"inferenceMs": 34.099999994039536,
"totalMs": 940.5,
"pairsPerSecond": 29.325513201606853,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "q8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "q8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "contradiction",
"actualLabel": "contradiction",
"margin": 0.9838062331929446,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 906.4000000059605,
"inferenceMs": 11.599999994039536,
"totalMs": 918,
"pairsPerSecond": 86.2068965960201,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "q8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "q8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.978804847561861,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 906.4000000059605,
"inferenceMs": 12.100000023841858,
"totalMs": 918.5000000298023,
"pairsPerSecond": 82.64462793633045,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "q8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "q8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9762451474043524,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 976.7999999821186,
"inferenceMs": 122,
"totalMs": 1098.7999999821186,
"pairsPerSecond": 8.19672131147541,
"grade": "good",
"actualDevice": "webgpu",
"actualPrecision": "q8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "q8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "contradiction",
"actualLabel": "contradiction",
"margin": 0.9838062150472762,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 976.7999999821186,
"inferenceMs": 56.599999994039536,
"totalMs": 1033.3999999761581,
"pairsPerSecond": 17.667844524828773,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "q8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "q8",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9788048684966374,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 976.7999999821186,
"inferenceMs": 50.30000001192093,
"totalMs": 1027.0999999940395,
"pairsPerSecond": 19.880715701053745,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "q8",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "q4",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9835005489209638,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1197.2999999821186,
"inferenceMs": 46.19999998807907,
"totalMs": 1243.4999999701977,
"pairsPerSecond": 21.645021650606683,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "q4",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "q4",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "contradiction",
"actualLabel": "contradiction",
"margin": 0.9875688396313099,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1197.2999999821186,
"inferenceMs": 23.80000001192093,
"totalMs": 1221.0999999940395,
"pairsPerSecond": 42.01680670164372,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "q4",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "q4",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9772584155029673,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1197.2999999821186,
"inferenceMs": 23.69999998807907,
"totalMs": 1220.9999999701977,
"pairsPerSecond": 42.19409284822754,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "q4",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9835005392242704,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1233.5999999940395,
"inferenceMs": 87.30000001192093,
"totalMs": 1320.9000000059605,
"pairsPerSecond": 11.454753721230798,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "q4",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "contradiction",
"actualLabel": "contradiction",
"margin": 0.9875688545978833,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1233.5999999940395,
"inferenceMs": 22.5,
"totalMs": 1256.0999999940395,
"pairsPerSecond": 44.44444444444444,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "q4",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9772584580200314,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1233.5999999940395,
"inferenceMs": 19.400000005960464,
"totalMs": 1253,
"pairsPerSecond": 51.54639173674018,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "q4",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "bnb4",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9835005489209638,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1262.199999988079,
"inferenceMs": 46.20000001788139,
"totalMs": 1308.4000000059605,
"pairsPerSecond": 21.64502163664409,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "bnb4",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "bnb4",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "contradiction",
"actualLabel": "contradiction",
"margin": 0.9875688396313099,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1262.199999988079,
"inferenceMs": 23.099999994039536,
"totalMs": 1285.2999999821186,
"pairsPerSecond": 43.290043301213366,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "bnb4",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "bnb4",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9772584155029673,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1262.199999988079,
"inferenceMs": 23.599999994039536,
"totalMs": 1285.7999999821186,
"pairsPerSecond": 42.37288136663398,
"grade": "excellent",
"actualDevice": "wasm",
"actualPrecision": "bnb4",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": false,
"acceleration": "cpu"
}
},
{
"precision": "bnb4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9835005392242704,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1301.300000011921,
"inferenceMs": 85.90000000596046,
"totalMs": 1387.2000000178814,
"pairsPerSecond": 11.641443538191055,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "bnb4",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "bnb4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "contradiction",
"actualLabel": "contradiction",
"margin": 0.9875688545978833,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1301.300000011921,
"inferenceMs": 26.5,
"totalMs": 1327.800000011921,
"pairsPerSecond": 37.735849056603776,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "bnb4",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "bnb4",
"device": "webgpu",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"allLabels": true,
"expectedLabel": "neutral",
"actualLabel": "neutral",
"margin": 0.9772584580200314,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1301.300000011921,
"inferenceMs": 22.69999998807907,
"totalMs": 1324,
"pairsPerSecond": 44.05286345925776,
"grade": "excellent",
"actualDevice": "webgpu",
"actualPrecision": "bnb4",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f",
"gpuForwardQueueObserved": true,
"acceleration": "partial"
}
},
{
"precision": "q8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"actualLabel": "entailment",
"expectedLabel": "entailment",
"margin": 0.9421899525497719,
"minimumMargin": 0.3
},
"runtime": {
"loadMs": 1296.7999999821186,
"inferenceMs": 33.099999994039536,
"totalMs": 1329.8999999761581,
"pairsPerSecond": 30.21148036797808,
"grade": "excellent",
"actualPrecision": "q8",
"actualDevice": "wasm",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f"
}
},
{
"precision": "q8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"actualLabel": "contradiction",
"expectedLabel": "contradiction",
"margin": 0.9992550970649349,
"minimumMargin": 0.3
},
"runtime": {
"loadMs": 1296.7999999821186,
"inferenceMs": 19.099999994039536,
"totalMs": 1315.8999999761581,
"pairsPerSecond": 52.356020958746925,
"grade": "excellent",
"actualPrecision": "q8",
"actualDevice": "wasm",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f"
}
},
{
"precision": "q8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"actualLabel": "entailment",
"expectedLabel": "entailment",
"margin": 0.9708945043089913,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1296.7999999821186,
"inferenceMs": 13.100000023841858,
"totalMs": 1309.9000000059605,
"pairsPerSecond": 76.33587772366495,
"grade": "excellent",
"actualPrecision": "q8",
"actualDevice": "wasm",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f"
}
},
{
"precision": "q8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"actualLabel": "neutral",
"expectedLabel": "neutral",
"margin": 0.9763729440912856,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1296.7999999821186,
"inferenceMs": 11.900000005960464,
"totalMs": 1308.699999988079,
"pairsPerSecond": 84.03361340328745,
"grade": "excellent",
"actualPrecision": "q8",
"actualDevice": "wasm",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f"
}
},
{
"precision": "q8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"actualLabel": "contradiction",
"expectedLabel": "contradiction",
"margin": 0.9838062331929446,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1296.7999999821186,
"inferenceMs": 11,
"totalMs": 1307.7999999821186,
"pairsPerSecond": 90.9090909090909,
"grade": "excellent",
"actualPrecision": "q8",
"actualDevice": "wasm",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f"
}
},
{
"precision": "q8",
"device": "wasm",
"status": "pass",
"quality": {
"passed": true,
"finite": true,
"actualLabel": "neutral",
"expectedLabel": "neutral",
"margin": 0.978804847561861,
"minimumMargin": 0.15
},
"runtime": {
"loadMs": 1296.7999999821186,
"inferenceMs": 12.099999994039536,
"totalMs": 1308.8999999761581,
"pairsPerSecond": 82.64462813988433,
"grade": "excellent",
"actualPrecision": "q8",
"actualDevice": "wasm",
"fallback": false,
"version": "4.2.0",
"artifactRevision": "cf5995a136f48843191ba185bd3430e18b0ffc2f"
}
}
]