Granite Embedding 97M Multilingual R2
16 configurations testedQuality criteria
Three finite, distinct, normalized 384d vectors; an English query must rank its Japanese answer above an unrelated passage by at least 0.1 cosine.
| Precision | Device | Result | |
|---|---|---|---|
Q4 | GPU · WebGPUAccelerated | Passed | Fast |
BNB4 | GPU · WebGPUAccelerated | Passed | Fast |
FP32 | GPU · WebGPUAccelerated | Passed | Fast |
Q8 | GPU · WebGPUAccelerated | Passed | Fast |
INT8 | GPU · WebGPUAccelerated | Passed | Fast |
UINT8 | GPU · WebGPUAccelerated | Passed | Fast |
FP32 | CPU · WASM | Passed | Very fast |
UINT8 | CPU · WASM | Passed | Very fast |
Q8 | CPU · WASM | Passed | Very fast |
INT8 | CPU · WASM | Passed | Very fast |
Q4 | CPU · WASM | Passed | Fast |
BNB4 | CPU · WASM | Passed | Fast |
FP16 | CPU · WASM | Failed | — |
FP16 | GPU · WebGPU | Failed | — |
Q4F16 | CPU · WASM | Failed | — |
Q4F16 | GPU · WebGPU | Failed | — |
Test setup & sample details
- Quality gate
- Three finite, distinct, normalized 384d vectors; an English query must rank its Japanese answer above an unrelated passage by at least 0.1 cosine.
- Tested
- 2026-09-15T16:25:44.372Z
- Environment
- Isolated headless Chrome on macOS; Transformers.js 4.2.0
- Runtime
- Transformers.js 4.2.0
- Artifact revision
- 536a9f241cb3f02a9c5995a1e708c784bd274859
- Fixture
{ "file": "granite-embedding-97m-multilingual-r2.json", "url": "https://f005.backblazeb2.com/file/hubters-webai-public/assets/benchmarks/granite-embedding-97m-multilingual-r2.json", "size": 735, "sha1": "721d37a672010fd5ec9cdf564c697319d598e342" }- Rerun when
- artifact revision change; runtime version change; fixture or quality-gate change; browser/backend behavior change
Per-case quality & failure details
[
{
"precision": "fp32",
"device": "wasm",
"status": "pass",
"quality": {
"shape": true,
"finite": true,
"normalized": true,
"distinct": true,
"positive": 0.9259532456887626,
"negative": 0.6901542851041528,
"margin": 0.2357989605846098,
"minimumMargin": 0.1,
"grade": "semantic-retrieval-pass"
},
"runtime": {
"loadMs": 44256.80000001192,
"inferenceMs": 88.90000000596046,
"totalMs": 45639.09999999404,
"grade": "excellent",
"embeddingsPerSecond": 33.745781775015296
}
},
{
"precision": "fp32",
"device": "webgpu",
"status": "pass",
"quality": {
"shape": true,
"finite": true,
"normalized": true,
"distinct": true,
"positive": 0.9259531192134285,
"negative": 0.6901542423153498,
"margin": 0.23579887689807866,
"minimumMargin": 0.1,
"grade": "semantic-retrieval-pass"
},
"runtime": {
"loadMs": 44079.79999998212,
"inferenceMs": 122,
"totalMs": 45423,
"grade": "excellent",
"embeddingsPerSecond": 24.59016393442623
}
},
{
"precision": "fp16",
"device": "wasm",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": 29303.29999998212,
"grade": "failed"
},
"failure": "Error: Can't create a session. ERROR_CODE: 1, ERROR_MESSAGE: Type Error: Type (tensor(float16)) of output arg (/layers.0/attn/rotary_emb/Cast_1_output_0) of node (/layers.0/attn/rotary_emb/Cast_1) does not match expected type (tensor(float))."
},
{
"precision": "fp16",
"device": "webgpu",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": 27575.29999998212,
"grade": "failed"
},
"failure": "Error: Can't create a session. ERROR_CODE: 1, ERROR_MESSAGE: Type Error: Type (tensor(float16)) of output arg (/layers.0/attn/rotary_emb/Cast_1_output_0) of node (/layers.0/attn/rotary_emb/Cast_1) does not match expected type (tensor(float))."
},
{
"precision": "int8",
"device": "wasm",
"status": "pass",
"quality": {
"shape": true,
"finite": true,
"normalized": true,
"distinct": true,
"positive": 0.8823177193527786,
"negative": 0.5999481015019146,
"margin": 0.282369617850864,
"minimumMargin": 0.1,
"grade": "semantic-retrieval-pass"
},
"runtime": {
"loadMs": 18007.59999999404,
"inferenceMs": 97.80000001192093,
"totalMs": 19350.29999998212,
"grade": "excellent",
"embeddingsPerSecond": 30.674846622027886
}
},
{
"precision": "int8",
"device": "webgpu",
"status": "pass",
"quality": {
"shape": true,
"finite": true,
"normalized": true,
"distinct": true,
"positive": 0.8823176872969928,
"negative": 0.5999480410167466,
"margin": 0.2823696462802462,
"minimumMargin": 0.1,
"grade": "semantic-retrieval-pass"
},
"runtime": {
"loadMs": 18234.59999999404,
"inferenceMs": 184.19999998807907,
"totalMs": 19668.100000023842,
"grade": "good",
"embeddingsPerSecond": 16.286644952194095
}
},
{
"precision": "uint8",
"device": "wasm",
"status": "pass",
"quality": {
"shape": true,
"finite": true,
"normalized": true,
"distinct": true,
"positive": 0.8717737596771854,
"negative": 0.6119479442542025,
"margin": 0.2598258154229829,
"minimumMargin": 0.1,
"grade": "semantic-retrieval-pass"
},
"runtime": {
"loadMs": 19023.90000000596,
"inferenceMs": 96.69999998807907,
"totalMs": 20419.09999999404,
"grade": "excellent",
"embeddingsPerSecond": 31.02378490558255
}
},
{
"precision": "uint8",
"device": "webgpu",
"status": "pass",
"quality": {
"shape": true,
"finite": true,
"normalized": true,
"distinct": true,
"positive": 0.8693715912393482,
"negative": 0.6193572420061288,
"margin": 0.2500143492332193,
"minimumMargin": 0.1,
"grade": "semantic-retrieval-pass"
},
"runtime": {
"loadMs": 20835.90000000596,
"inferenceMs": 252.30000001192093,
"totalMs": 22369,
"grade": "good",
"embeddingsPerSecond": 11.890606420365648
}
},
{
"precision": "q8",
"device": "wasm",
"status": "pass",
"quality": {
"shape": true,
"finite": true,
"normalized": true,
"distinct": true,
"positive": 0.8823177193527786,
"negative": 0.5999481015019146,
"margin": 0.282369617850864,
"minimumMargin": 0.1,
"grade": "semantic-retrieval-pass"
},
"runtime": {
"loadMs": 19433,
"inferenceMs": 97.5,
"totalMs": 20740.79999998212,
"grade": "excellent",
"embeddingsPerSecond": 30.769230769230766
}
},
{
"precision": "q8",
"device": "webgpu",
"status": "pass",
"quality": {
"shape": true,
"finite": true,
"normalized": true,
"distinct": true,
"positive": 0.8823176872969928,
"negative": 0.5999480410167466,
"margin": 0.2823696462802462,
"minimumMargin": 0.1,
"grade": "semantic-retrieval-pass"
},
"runtime": {
"loadMs": 19048.29999998212,
"inferenceMs": 183.7000000178814,
"totalMs": 20532.20000001788,
"grade": "good",
"embeddingsPerSecond": 16.33097441321709
}
},
{
"precision": "q4",
"device": "wasm",
"status": "pass",
"quality": {
"shape": true,
"finite": true,
"normalized": true,
"distinct": true,
"positive": 0.9234198756914827,
"negative": 0.6984499903196258,
"margin": 0.2249698853718568,
"minimumMargin": 0.1,
"grade": "semantic-retrieval-pass"
},
"runtime": {
"loadMs": 35297.89999997616,
"inferenceMs": 125.40000000596046,
"totalMs": 36728.09999999404,
"grade": "excellent",
"embeddingsPerSecond": 23.923444974939436
}
},
{
"precision": "q4",
"device": "webgpu",
"status": "pass",
"quality": {
"shape": true,
"finite": true,
"normalized": true,
"distinct": true,
"positive": 0.92342002894741,
"negative": 0.6984500657764473,
"margin": 0.22496996317096274,
"minimumMargin": 0.1,
"grade": "semantic-retrieval-pass"
},
"runtime": {
"loadMs": 35796.59999999404,
"inferenceMs": 107.40000000596046,
"totalMs": 37176.40000000596,
"grade": "excellent",
"embeddingsPerSecond": 27.932960892304532
}
},
{
"precision": "q4f16",
"device": "wasm",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": 23773,
"grade": "failed"
},
"failure": "Error: Can't create a session. ERROR_CODE: 1, ERROR_MESSAGE: Type Error: Type (tensor(float16)) of output arg (/layers.0/attn/rotary_emb/Cast_1_output_0) of node (/layers.0/attn/rotary_emb/Cast_1) does not match expected type (tensor(float))."
},
{
"precision": "q4f16",
"device": "webgpu",
"status": "fail",
"quality": "Not embedded",
"runtime": {
"loadMs": null,
"inferenceMs": null,
"totalMs": 24294.90000000596,
"grade": "failed"
},
"failure": "Error: Can't create a session. ERROR_CODE: 1, ERROR_MESSAGE: Type Error: Type (tensor(float16)) of output arg (/layers.0/attn/rotary_emb/Cast_1_output_0) of node (/layers.0/attn/rotary_emb/Cast_1) does not match expected type (tensor(float))."
},
{
"precision": "bnb4",
"device": "wasm",
"status": "pass",
"quality": {
"shape": true,
"finite": true,
"normalized": true,
"distinct": true,
"positive": 0.9234198756914827,
"negative": 0.6984499903196258,
"margin": 0.2249698853718568,
"minimumMargin": 0.1,
"grade": "semantic-retrieval-pass"
},
"runtime": {
"loadMs": 37582.90000000596,
"inferenceMs": 129.59999999403954,
"totalMs": 39018.89999997616,
"grade": "excellent",
"embeddingsPerSecond": 23.14814814921276
}
},
{
"precision": "bnb4",
"device": "webgpu",
"status": "pass",
"quality": {
"shape": true,
"finite": true,
"normalized": true,
"distinct": true,
"positive": 0.92342002894741,
"negative": 0.6984500657764473,
"margin": 0.22496996317096274,
"minimumMargin": 0.1,
"grade": "semantic-retrieval-pass"
},
"runtime": {
"loadMs": 34449.20000001788,
"inferenceMs": 107.7999999821186,
"totalMs": 35773.5,
"grade": "excellent",
"embeddingsPerSecond": 27.829313548215463
}
}
]